From 4a8a6567ca309fa6220a3945140c1d42751d44ff Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Sat, 24 Oct 2026 14:44:50 -0700 Subject: [PATCH 01/23] Add the shape overlay: typed JavaScript from analysis-proven facts (opt-in) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The shape overlay (DESIGN.md, approved route) rewrites the JavaScript of opted-in npm packages with JSDoc types the shape analysis proves: record typedefs for non-escaping literal sites (optional fields for spread sources), class field, parameter, return and local types, receiver casts, and branches the analysis proved unreachable replaced by checks. The lowering trusts only annotations carrying the overlay's marker; package JSDoc stays a hint (its type tags are neutralized in overlaid files). - Inferred-fact checks: every checked conversion lowered in an overlaid file records its site, and a failure throws a distinct internal error ("scriptc internal error: an inferred type was contradicted at … This is a bug in scriptc's shape analysis, not in the program's data") naming the opt-outs; SCRIPTC_SHAPE_CHECK_LOG makes checks log instead (validation mode). - Opt-outs: `--no-shape-overlay`, and `"scriptc": {"packages": {"": {"shapes": false}}}` per package. - Build report: SC6103 counts the declarations typed, the checks inserted and the branches replaced. - `x instanceof Promise` on plain data folds to false. The overlay runs only with SCRIPTC_SHAPE_OVERLAY=1 while it is new. Co-Authored-By: Claude Opus 5.5 --- packages/cli/src/bootstrap.ts | 6 + .../compiler/src/backend/llvm/expr-async.ts | 14 +- packages/compiler/src/cli/command.ts | 1 + packages/compiler/src/cli/usage.ts | 5 + packages/compiler/src/compile-types.ts | 4 + .../compiler/src/diagnostics/diagnostic.ts | 5 +- .../src/executable/executable-cache.ts | 3 + .../src/frontend/lowering/call-signatures.ts | 3 + .../lowering/expressions/relational.ts | 33 + .../frontend/lowering/generic-functions.ts | 6 + .../src/frontend/lowering/lower-classes.ts | 20 +- .../compiler/src/frontend/lowering/lowerer.ts | 12 + .../compiler/src/frontend/package-shims.ts | 18 + packages/compiler/src/frontend/pipeline.ts | 5 +- packages/compiler/src/frontend/program.ts | 27 +- .../src/frontend/shape-inference/analyze.ts | 160 ++++- .../src/frontend/shape-inference/domain.ts | 2 + .../src/frontend/shape-inference/index.ts | 144 +++- .../src/frontend/shape-inference/overlay.ts | 615 ++++++++++++++++++ packages/compiler/src/index.ts | 12 + .../src/ir/function-references.test.ts | 3 + packages/compiler/src/ir/ir.ts | 9 + packages/runtime/src/scr_dyn.h | 4 + packages/runtime/src/scr_json.c | 46 ++ tests/harness/shape-overlay.test.ts | 62 ++ 25 files changed, 1208 insertions(+), 11 deletions(-) create mode 100644 packages/compiler/src/frontend/shape-inference/overlay.ts create mode 100644 tests/harness/shape-overlay.test.ts diff --git a/packages/cli/src/bootstrap.ts b/packages/cli/src/bootstrap.ts index f14866a9c..485d62528 100644 --- a/packages/cli/src/bootstrap.ts +++ b/packages/cli/src/bootstrap.ts @@ -141,6 +141,11 @@ async function tryFastPath(): Promise { ...(values["detached-exit"] ? { detachedExit: true as const } : {}), ...(checker === undefined ? {} : { checker }), npmStatic, + // Mirrors the full compiler's key: shape inference's overlay is opt-in + // (SCRIPTC_SHAPE_OVERLAY=1) and --no-shape-overlay turns it off. + ...(process.env["SCRIPTC_SHAPE_OVERLAY"] === "1" && values["shape-overlay"] !== false + ? { shapeOverlay: true as const } + : {}), ffiProfile: ffiPath === null ? null : { path: ffiPath, bytes: ffiBytes! }, target: `${process.env["SCRIPTC_TARGET"] ?? "native"}:${buildPlatform}:${process.arch}:${ helperObjectRoute ? "runtime-pack" : "driver-tu" @@ -212,6 +217,7 @@ async function coverageFastPath( values["provenance-sources"] || values["emit-ir"] || (values["npm-static"] ?? []).length > 0 || + values["shape-overlay"] === false || (values["external-types"] ?? []).length > 0 ) return null; diff --git a/packages/compiler/src/backend/llvm/expr-async.ts b/packages/compiler/src/backend/llvm/expr-async.ts index ee9ff2983..0e0191af4 100644 --- a/packages/compiler/src/backend/llvm/expr-async.ts +++ b/packages/compiler/src/backend/llvm/expr-async.ts @@ -394,12 +394,24 @@ export function emitSerializationExpr( } const dynV = emitBorrowedInput(host, e.value); const helper = host.dyn.dynCheckHelper(e.type, e.preserveRefs); - if (isDynTypedRefType(e.type)) return emitClassCheck(host, e.type, dynV.name, helper); + // A check of an analysis-proven fact (shape inference's overlay) + // names its site, so a failure reports a compiler bug. + const site = e.inferredSite !== undefined ? host.cstr(e.inferredSite) : null; + if (site !== null) { + host.declare(`declare void @scr_dyn_inferred_site(ptr)`); + B.line(`call void @scr_dyn_inferred_site(ptr ${site})`); + } + if (isDynTypedRefType(e.type)) { + const checked = emitClassCheck(host, e.type, dynV.name, helper); + if (site !== null) B.line(`call void @scr_dyn_inferred_site(ptr null)`); + return checked; + } const ty = host.llType(e.type); const t = B.tmp(); B.line( `${t} = call ${ty === "i1" ? "zeroext i1" : ty} @${helper}(ptr ${dynV.name}, ptr null)`, ); + if (site !== null) B.line(`call void @scr_dyn_inferred_site(ptr null)`); const out = host.own({ name: t, type: e.type }); host.emitPendingCheck(); return out; diff --git a/packages/compiler/src/cli/command.ts b/packages/compiler/src/cli/command.ts index 57b15349e..a57885ca8 100644 --- a/packages/compiler/src/cli/command.ts +++ b/packages/compiler/src/cli/command.ts @@ -534,6 +534,7 @@ async function main(args: string[], host: CliHost): Promise { ? {} : { profileUse: resolve(values["profile-use"]) }), ...(npmStatic !== undefined ? { npmStatic } : {}), + ...(values["shape-overlay"] === false ? { shapeOverlay: false } : {}), ...(ffiProfilePath !== undefined ? { ffiProfilePath } : {}), ...(printNativeLinkInfo ? { nativeLinkInfo: true } : {}), ...(tier !== undefined ? { tier } : {}), diff --git a/packages/compiler/src/cli/usage.ts b/packages/compiler/src/cli/usage.ts index 01afbdd41..292755981 100644 --- a/packages/compiler/src/cli/usage.ts +++ b/packages/compiler/src/cli/usage.ts @@ -87,6 +87,10 @@ Options: "pkg..." also names every package pkg depends on). Packages that fail preflight are refused with a diagnostic. Eligible dependencies compile automatically. + --no-shape-overlay + compile npm packages' JavaScript without the types shape + inference proves (per package: "scriptc": {"packages": + {"": {"shapes": false}}} in package.json) --provenance-sources EXPERIMENTAL: compile npm dependencies from their provenance-attested SOURCE (fetched at the attested @@ -138,6 +142,7 @@ export const CLI_OPTIONS = { sanitize: { type: "boolean", default: false }, ffi: { type: "string" }, "npm-static": { type: "string", multiple: true }, + "shape-overlay": { type: "boolean" }, "provenance-sources": { type: "boolean", default: false }, "external-types": { type: "string", multiple: true }, tier: { type: "string" }, diff --git a/packages/compiler/src/compile-types.ts b/packages/compiler/src/compile-types.ts index c50931c6a..2c9ba44d7 100644 --- a/packages/compiler/src/compile-types.ts +++ b/packages/compiler/src/compile-types.ts @@ -55,6 +55,10 @@ export interface CompileBaseOptions { /** Eligible packages compile automatically. Explicit names override automatic selection heuristics. */ npmStatic?: readonly string[] | "auto"; + /** Shape inference's typed overlay for opted-in packages' JavaScript. + * False disables it (`--no-shape-overlay`); a package opts out with + * `"scriptc": { "packages": { "": { "shapes": false } } }`. */ + shapeOverlay?: boolean; /** Outbound native FFI manifest. Its signature-only TypeScript bindings * lower to direct C ABI calls. Source outputs retain those declarations; * archive/system-library inputs join only an executable link. */ diff --git a/packages/compiler/src/diagnostics/diagnostic.ts b/packages/compiler/src/diagnostics/diagnostic.ts index 2fd8b6fb0..34657008d 100644 --- a/packages/compiler/src/diagnostics/diagnostic.ts +++ b/packages/compiler/src/diagnostics/diagnostic.ts @@ -64,7 +64,10 @@ * parses date strings with V8's grammar. SC61xx are performance * notes, also warnings (coverage --fail-on=dynamic opts in): * likely-hot functions that use untyped values (SC6101, - * diagnostics/dynamic-tier.ts) + * diagnostics/dynamic-tier.ts), package files whose projected + * declaration types were dropped (SC6102), and shape inference's + * overlay report: what it typed and how many inferred facts the + * build checks (SC6103, frontend/shape-inference) * SC9xxx internal compiler errors (still source-anchored) */ import type { SrcLoc } from "../ir/ir.js"; diff --git a/packages/compiler/src/executable/executable-cache.ts b/packages/compiler/src/executable/executable-cache.ts index 754c20648..2bfed90f3 100644 --- a/packages/compiler/src/executable/executable-cache.ts +++ b/packages/compiler/src/executable/executable-cache.ts @@ -152,6 +152,9 @@ export interface EarlyExecutableCacheOptions { /** Omitted unless SCRIPTC_YOUNG_NURSERY=1 (backend/young-nursery.ts). */ youngNursery?: true; npmStatic: readonly string[] | "auto" | null; + /** Omitted unless shape inference's overlay compiles the packages + * (frontend/shape-inference). */ + shapeOverlay?: true; /** Raw manifest identity: path and bytes. Native archives remain under the * stricter native cache's independent dependency validation. */ ffiProfile: { path: string; bytes: Uint8Array } | null; diff --git a/packages/compiler/src/frontend/lowering/call-signatures.ts b/packages/compiler/src/frontend/lowering/call-signatures.ts index c9fd954b5..9d8096c62 100644 --- a/packages/compiler/src/frontend/lowering/call-signatures.ts +++ b/packages/compiler/src/frontend/lowering/call-signatures.ts @@ -1,3 +1,4 @@ +import { shapeOverlayAnnotated } from "../shape-inference/index.js"; import { typedArgumentsCandidate, typedArgumentsShapes, @@ -181,6 +182,8 @@ export function checkedJsDocumentedParameter( lowerer: Lowerer, param: ts.ParameterDeclaration, ): boolean { + if (param.parent !== undefined && shapeOverlayAnnotated(param.parent) && documentedParameter(param)) + return true; return lowerer.checkedJsFile(ts.getSourceFileOfNode(param)) && documentedParameter(param); } diff --git a/packages/compiler/src/frontend/lowering/expressions/relational.ts b/packages/compiler/src/frontend/lowering/expressions/relational.ts index 47030777a..9477fb4b3 100644 --- a/packages/compiler/src/frontend/lowering/expressions/relational.ts +++ b/packages/compiler/src/frontend/lowering/expressions/relational.ts @@ -420,6 +420,22 @@ export function lowerInstanceOf(lowerer: Lowerer, expr: ts.BinaryExpression, loc loc, }; } + // Plain data (records, arrays, strings, numbers, booleans, and unions + // of them) is never a promise: false after the operand's evaluation. + const neverPromise = (t: IrType): boolean => + NEVER_PROMISE_KINDS.has(t.kind) || + (t.kind === "union" && (lowerer.unions.get(t.unionId)?.arms ?? [t]).every(neverPromise)); + if (neverPromise(left.type)) { + const no: IrExpr = { kind: "boolLit", value: false, type: BOOL, loc }; + if (isSafeToDiscard(left)) return no; + return { + kind: "seqExpr", + stmts: [{ kind: "exprStmt", expr: left, loc }], + result: no, + type: BOOL, + loc, + }; + } lowerer.unsupported( "SC1090", expr, @@ -2271,3 +2287,20 @@ function nodeDeclaredMember(lowerer: Lowerer, t: ts.Type, key: string): boolean ); }); } + +const NEVER_PROMISE_KINDS: ReadonlySet = new Set([ + "record", + "array", + "string", + "f64", + "bool", + "bigint", + "date", + "map", + "set", + "regex", + "bytes", + "symbol", + "nullT", + "undefinedT", +]); diff --git a/packages/compiler/src/frontend/lowering/generic-functions.ts b/packages/compiler/src/frontend/lowering/generic-functions.ts index 7896da578..b60cb4a60 100644 --- a/packages/compiler/src/frontend/lowering/generic-functions.ts +++ b/packages/compiler/src/frontend/lowering/generic-functions.ts @@ -1,3 +1,5 @@ +import { shapeOverlayAnnotated } from "../shape-inference/index.js"; +import { documentedParameter } from "./call-signatures.js"; import { staticNameBindings } from "./static-function-names.js"; import { positionTag } from "./structural-positions.js"; import { keyedHelperName } from "./stable-names.js"; @@ -1855,10 +1857,14 @@ export function implicitAnyParamSymbolsOf( // The variadic-`arguments` form keeps its dynRest story whole. if (bodyReadsArguments(decl)) return null; let hasImplicitParams = false; + const overlayTyped = shapeOverlayAnnotated(decl); const out = decl.parameters.nodes.map((param): ts.Symbol | null => { if (!ts.isIdentifier(param.name)) return null; if (param.dotDotDotToken || param.questionToken || param.initializer) return null; if (param.name.text === "this") return null; + // A parameter typed by shape inference's overlay is a proven type, + // not an implicit type parameter. + if (overlayTyped && documentedParameter(param)) return null; const t = lowerer.typeOf(param.name); const broadFunction = lowerer.checker.typeToString(t) === "Function"; const broadArray = lowerer.checkerAnyArrayType(t); diff --git a/packages/compiler/src/frontend/lowering/lower-classes.ts b/packages/compiler/src/frontend/lowering/lower-classes.ts index b829b6ddc..573bb0c69 100644 --- a/packages/compiler/src/frontend/lowering/lower-classes.ts +++ b/packages/compiler/src/frontend/lowering/lower-classes.ts @@ -1,4 +1,5 @@ import { lowerAsyncLocalStorageOptions } from "./builtins/async-context.js"; +import { shapeOverlayAnnotated } from "../shape-inference/index.js"; import { DATE_TIME_FIELD, dateRootOf, @@ -1152,7 +1153,11 @@ function jsFieldInitializerType(lowerer: Lowerer, type: IrType, expression: ts.E ) return DYN; } - if (type.kind === "array" || type.kind === "record") return DYN; + // Shape inference's overlay types a field as a record or array only when + // the analysis proves its values never reach code that could observe a + // copy, so those fields keep their native layout. + if ((type.kind === "array" || type.kind === "record") && !shapeOverlayAnnotated(expression.parent?.parent ?? expression)) + return DYN; // JavaScript documentation cannot guarantee that a call or an indexed // read returns a present number. Numeric fields receiving those values // must retain undefined, so later arithmetic performs JS coercion (NaN). @@ -3316,6 +3321,7 @@ export function collectClassShapeInner( !functionLocalClass(decl) && ts.isIdentifier(member.name) && !lowerer.virtualJsMethods.has(member) && + !lowerer.checkedJsOverrideMethods.has(member) && !lowerer.prototypeMethodAccesses.has(member.name.text) && inst === undefined && decl.typeParameters === undefined && @@ -7597,6 +7603,16 @@ export function collectVirtualJsMethods(lowerer: Lowerer, files: readonly ts.Sou if (!ts.isIdentifier(member.name)) continue; const property = lowerer.checker.getPropertyOfType(base, member.name.text); for (const declaration of property ? property.declarations : []) { + if ( + ts.isMethodDeclaration(declaration) && + isJsSourceFile(ts.getSourceFileOfNode(declaration)) && + checkedJsOverride(lowerer, declaration, member) + ) { + // A checked override pair dispatches through the vtable, + // so neither side may become an implicit generic. + lowerer.checkedJsOverrideMethods.add(declaration); + lowerer.checkedJsOverrideMethods.add(member); + } if ( ts.isMethodDeclaration(declaration) && isJsSourceFile(ts.getSourceFileOfNode(declaration)) && @@ -7683,7 +7699,7 @@ function checkedJsOverride( ): boolean { return [base, member].every( (method) => - lowerer.checkedJsFile(ts.getSourceFileOfNode(method)) && + (lowerer.checkedJsFile(ts.getSourceFileOfNode(method)) || shapeOverlayAnnotated(method)) && method.parameters.nodes.every(documentedParameter), ); } diff --git a/packages/compiler/src/frontend/lowering/lowerer.ts b/packages/compiler/src/frontend/lowering/lowerer.ts index 988b4b804..f8ad0a559 100644 --- a/packages/compiler/src/frontend/lowering/lowerer.ts +++ b/packages/compiler/src/frontend/lowering/lowerer.ts @@ -1,3 +1,5 @@ +import { markInferredFactChecks, shapeOverlayFileNames } from "../shape-inference/index.js"; +import { npmStaticPackageOfPath } from "../npm-static.js"; import { isDeclBoundaryFile } from "../decl-projection.js"; import { relieveHeapPressure } from "../../heap-pressure.js"; import { moduleNamespaceObject } from "./module-namespace-elements.js"; @@ -3095,6 +3097,8 @@ export class Lowerer { /** Inferred JS methods participating in an override chain keep a vtable * ABI instead of call-site specialization. Filled before class collection. */ readonly virtualJsMethods = new Set(); + /** Methods of checked-JavaScript override pairs (vtable dispatch). */ + readonly checkedJsOverrideMethods = new Set(); /** Descendant writes must widen the original JS field before collecting * any class layout; base and derived instances share its storage ABI. */ readonly inheritedJsFieldWrites = new Map< @@ -7885,6 +7889,7 @@ export class Lowerer { // every producer and consumer agrees. Programs with no unregistered // reference are untouched — byte-stability holds. checkRequireBindingAccesses(this, functions); + markInferredFactChecks(functions, this.program); const module: IrModule | null = this.diags.length > 0 ? null @@ -7900,6 +7905,13 @@ export class Lowerer { ...(this.usesWorkers ? { workers: true } : {}), ...(this.usesLifetimeRegions ? { lifetimeRegions: true as const } : {}), ...(this.declBoundary ? { declBoundary: true as const } : {}), + ...(shapeOverlayFileNames().length > 0 + ? { + inferredFactFiles: shapeOverlayFileNames() + .map((file) => ({ file, pkg: npmStaticPackageOfPath(file) ?? "" })) + .sort((a, b) => (a.file < b.file ? -1 : a.file > b.file ? 1 : 0)), + } + : {}), ...(this.threadSlotKeys.size > 0 ? { threadSlotKeys: [...this.threadSlotKeys].sort() } : {}), diff --git a/packages/compiler/src/frontend/package-shims.ts b/packages/compiler/src/frontend/package-shims.ts index 10beeaae8..d26579719 100644 --- a/packages/compiler/src/frontend/package-shims.ts +++ b/packages/compiler/src/frontend/package-shims.ts @@ -202,6 +202,7 @@ let unloadedPackages: ReadonlySet = new Set(); * map of the entry's nearest package.json. Called once per program load. */ export function configurePackageShims(entryPath: string): void { const packages = new Set(); + const noShapes = new Set(); for (let dir = dirname(resolve(entryPath)); ;) { const text = trackedReadFile(join(dir, "package.json")); if (text !== null) { @@ -222,6 +223,14 @@ export function configurePackageShims(entryPath: string): void { if (map !== null && typeof map === "object") { for (const [name, kind] of Object.entries(map as Record)) { if (kind === "unloaded") packages.add(name); + // `{ "shapes": false }`: compile the package without shape + // inference's typed overlay. + if ( + kind !== null && + typeof kind === "object" && + (kind as Record)["shapes"] === false + ) + noShapes.add(name); } } break; @@ -231,6 +240,15 @@ export function configurePackageShims(entryPath: string): void { dir = parent; } unloadedPackages = packages; + shapelessPackages = noShapes; +} + +let shapelessPackages: ReadonlySet = new Set(); + +/** Packages the project compiles without shape inference's overlay + * (`"scriptc": { "packages": { "": { "shapes": false } } }`). */ +export function isShapelessPackage(name: string): boolean { + return shapelessPackages.has(name); } /** True for a package name the registry implements or the project marks diff --git a/packages/compiler/src/frontend/pipeline.ts b/packages/compiler/src/frontend/pipeline.ts index c2f528b25..92bafe54d 100644 --- a/packages/compiler/src/frontend/pipeline.ts +++ b/packages/compiler/src/frontend/pipeline.ts @@ -1,3 +1,4 @@ +import { shapeOverlayReport } from "./shape-inference/index.js"; /** Shared frontend orchestration. Hosts supply a program loader so Node * and native callers use the same npm selection, preflight and lowering. */ import type { ScrDiagnostic } from "../diagnostics/diagnostic.js"; @@ -699,8 +700,8 @@ function loadFrontend( }); // The declaration-typed boundary's notes (SC6102) join the // performance warnings; they never fail a build. - const notes = finalLoad.declBoundaryNotes; - if (notes === undefined) return lowered; + const notes = [...(finalLoad.declBoundaryNotes ?? []), ...shapeOverlayReport(finalLoad.entry.fileName)]; + if (notes.length === 0) return lowered; return { ...lowered, performance: [...notes, ...(lowered.performance ?? [])] }; }, npmStatic: statuses, diff --git a/packages/compiler/src/frontend/program.ts b/packages/compiler/src/frontend/program.ts index 8e582af89..daeb61ded 100644 --- a/packages/compiler/src/frontend/program.ts +++ b/packages/compiler/src/frontend/program.ts @@ -140,7 +140,12 @@ import { } from "./type-only-imports.js"; import { moduleSourceCandidates } from "./module-source-candidates.js"; import { planDeclProjection, type DeclProjection } from "./decl-projection.js"; -import { reportShapes } from "./shape-inference/index.js"; +import { + reportShapes, + resetShapeOverlay, + shapeOverlayEnabled, + shapeOverlayer, +} from "./shape-inference/index.js"; import { inferredJsDiagnosticSuppressed } from "./inferred-js-diagnostics.js"; import { containsJsx, @@ -857,6 +862,9 @@ function loadProgram7( /** The declaration-typed boundary's projection (decl-projection.ts): * answers rewritten user files for the lowering world, or null. */ projector?: (program: ts.Program) => DeclProjection | null, + /** Shape inference's typed overlay (shape-inference/overlay.ts): + * rewritten package JavaScript for the lowering world, or null. */ + overlayer?: (program: ts.Program) => ReadonlyMap | null, ): LoadResult & { disposeAll: () => void } { const config = adoptProjectConfig7(host, entryPath); const configuredPaths = config.options["paths"]; @@ -998,6 +1006,22 @@ function loadProgram7( } } } + // Shape inference's overlay: package files typed from proven facts. The + // tsc gate keeps reading the files as written. + // A load without the overlay must not see an earlier load's overlay + // files (the registry is per process). + resetShapeOverlay(); + if (overlayer !== undefined && jsx.refusal === null) { + const overlay = overlayer(program); + if (overlay !== null && overlay.size > 0) { + const before = program; + for (const [file, text] of overlay) host.addVirtualFile(file, text); + program = host.createProgram([...programRoots, overridesDtsPath()], options); + if (tscWorld === undefined) + tscWorld = { program: before, diagnostics: ts.getPreEmitDiagnostics(before), projectWorld: null }; + else if (tscWorld.program !== before) before.dispose(); + } + } const entry = program.getSourceFile(entryPath); if (!entry) throw new Error(`could not load ${entryPath}`); const externalTypeSpecifiersByFile = externalTypeFileClosure7(program, externalTypes); @@ -1260,6 +1284,7 @@ export function loadProgram( externalTypes, npmStaticPackages.length > 0, projector, + npmStaticPackages.length > 0 && shapeOverlayEnabled() ? shapeOverlayer : undefined, ); // Shape inference, stage S1: a report only (SCRIPTC_DEBUG_SHAPES). const shapes = process.env["SCRIPTC_DEBUG_SHAPES"]; diff --git a/packages/compiler/src/frontend/shape-inference/analyze.ts b/packages/compiler/src/frontend/shape-inference/analyze.ts index 8b1101da2..fe87a8c53 100644 --- a/packages/compiler/src/frontend/shape-inference/analyze.ts +++ b/packages/compiler/src/frontend/shape-inference/analyze.ts @@ -70,6 +70,8 @@ export interface LiteralSite { readonly kind: "object" | "array"; /** Static keys in creation order; null when a spread or computed key makes them unknown. */ readonly keys: string[] | null; + /** The literal's own static keys (without spread contributions). */ + readonly ownKeys: string[]; readonly hasAccessor: boolean; readonly hasProto: boolean; readonly hasComputedKey: boolean; @@ -129,6 +131,15 @@ export interface ShapeAnalysis { readonly escapedFunctions: ReadonlySet; /** ES5 constructors: each function's default prototype object. */ readonly protoOf: ReadonlyMap; + /** Every value a declaration (parameter, variable, binding element) holds, + * merged over the analysis contexts it was analyzed in. */ + declValues(decl: Node): Set; + /** The classes a class declaration's chain contains, base last. */ + classChain(decl: Node): Node[]; + /** `if` statements in walked code whose condition is known: the branch + * whose condition values never allowed it is absent from `walkedBranches`. */ + readonly branchIfs: ReadonlySet; + readonly walkedBranches: ReadonlySet; } /** Array.prototype members: other names on an array are own properties. */ @@ -358,6 +369,10 @@ class Analyzer { nodeLabel: this.nodeLabel, escapedFunctions: this.escapedFunctionCalls, protoOf: this.protoOf, + declValues: (decl) => this.declValues(decl), + classChain: (decl) => this.classChain(decl), + branchIfs: this.branchIfs, + walkedBranches: this.walkedBranches, }; } @@ -450,6 +465,78 @@ class Analyzer { return p; } + /** A boolean literal: labeled `boolean` like any boolean, but its + * truthiness is known, so `if` statements on it walk one branch. */ + boolLiteral(value: boolean): Atom { + return this.atoms.intern(`prim:boolean:${value}`, () => ({ + kind: "prim", + label: "boolean", + literal: value, + })); + } + + /** The analysis environment a deferred walk restores. */ + private env(): WalkEnv { + return { + ctx: this.ctx, + ctxRoot: this.ctxRoot, + currentFn: this.currentFn, + loopDepth: this.loopDepth, + }; + } + + private inEnv(env: WalkEnv, walk: () => void): void { + const saved = this.env(); + this.ctx = env.ctx; + this.ctxRoot = env.ctxRoot; + this.currentFn = env.currentFn; + this.loopDepth = env.loopDepth; + try { + walk(); + } finally { + this.ctx = saved.ctx; + this.ctxRoot = saved.ctxRoot; + this.currentFn = saved.currentFn; + this.loopDepth = saved.loopDepth; + } + } + + readonly branchIfs = new Set(); + readonly walkedBranches = new Set(); + /** Ifs whose branches wait for condition values; at the fixpoint, an + * `if` whose condition node stayed empty walks every waiting branch. */ + private readonly pendingIfs: PendingIf[] = []; + + /** An `if` walks a branch once a condition value allows it: the then + * branch on a value that may be truthy, the else branch on one that may + * be falsy. A branch no value allows is never walked, so its effects + * (returns, calls, writes) stay out of the facts; the overlay turns it + * into a checked internal error. */ + private ifStatement(s: ts.IfStatement): void { + const cond = this.expr(s.expression); + this.branchIfs.add(s); + const pending: PendingIf = { + s, + cond, + env: this.env(), + thenDone: false, + elseDone: s.elseStatement === undefined, + }; + this.pendingIfs.push(pending); + this.on(cond, (a) => { + if (!pending.thenDone && mayBeTruthy(a)) this.walkBranch(pending, "then"); + if (!pending.elseDone && mayBeFalsy(a)) this.walkBranch(pending, "else"); + }); + } + + private walkBranch(pending: PendingIf, which: "then" | "else"): void { + const branch = which === "then" ? pending.s.thenStatement : pending.s.elseStatement!; + if (which === "then") pending.thenDone = true; + else pending.elseDone = true; + this.walkedBranches.add(branch); + this.inEnv(pending.env, () => this.stmt(branch)); + } + builtin(path: string, receiver?: Atom): Atom { const key = receiver === undefined ? `builtin:${path}` : `builtin:${path}@${receiver.id}`; return this.atoms.intern(key, () => ({ kind: "builtin", label: path, path, receiver })); @@ -718,6 +805,24 @@ class Analyzer { return `${inside ? this.ctx : ""}|${this.nodeKey(node)}`; } + private declIndex: Map | null = null; + declValues(decl: Node): Set { + if (this.declIndex === null) { + this.declIndex = new Map(); + for (const [key, id] of this.bindings) { + const k = Number(key.slice(key.lastIndexOf("|") + 1)); + let l = this.declIndex.get(k); + if (l === undefined) this.declIndex.set(k, (l = [])); + l.push(id); + } + } + const out = new Set(); + const k = this.nodeKeys.get(decl); + if (k === undefined) return out; + for (const id of this.declIndex.get(k) ?? []) for (const a of this.nodes[id]!.atoms) out.add(a); + return out; + } + receiverCtx(atom: Atom): string { return atom.kind === "instance" || atom.kind === "class" ? `C${atom.id}` : ""; } @@ -1223,6 +1328,10 @@ class Analyzer { this.expr(s.expression); return; } + if (ts.isIfStatement(s) && this.currentFn !== null) { + this.ifStatement(s); + return; + } // Blocks, ifs, switches, labels: walk children, evaluating expressions. ts.forEachChild(s, (child) => { if (ts.isExpression(child)) this.expr(child); @@ -1317,8 +1426,9 @@ class Analyzer { case ts.SyntaxKind.BigIntLiteral: return this.constNode(this.prim("bigint")); case ts.SyntaxKind.TrueKeyword: + return this.constNode(this.boolLiteral(true)); case ts.SyntaxKind.FalseKeyword: - return this.constNode(this.prim("boolean")); + return this.constNode(this.boolLiteral(false)); case ts.SyntaxKind.NullKeyword: return this.constNode(this.prim("null")); case ts.SyntaxKind.RegularExpressionLiteral: @@ -1750,6 +1860,7 @@ class Analyzer { node: e, kind: "object", keys: keysKnown ? keys : null, + ownKeys: keys, hasAccessor, hasProto, hasComputedKey, @@ -1916,6 +2027,7 @@ class Analyzer { node: e, kind: "array", keys: null, + ownKeys: [], hasAccessor: false, hasProto: hasHole, hasComputedKey: false, @@ -3202,6 +3314,21 @@ class Analyzer { const before = this.steps; for (const c of this.lateCopies) c(); this.solve(); + // A condition with no values at the fixpoint is one the analysis + // could not evaluate: walk both of its branches. + let walked = false; + for (const p of this.pendingIfs) { + if (this.nodes[p.cond]!.atoms.size > 0) continue; + if (!p.thenDone) { + this.walkBranch(p, "then"); + walked = true; + } + if (!p.elseDone) { + this.walkBranch(p, "else"); + walked = true; + } + } + if (walked) continue; if (this.steps === before || this.budgetExceeded) return; } } @@ -3223,4 +3350,35 @@ function cjsExportName(decl: Node): string | null { ) return decl.left.name.text; return null; + } + +interface WalkEnv { + readonly ctx: string; + readonly ctxRoot: Node | null; + readonly currentFn: FunctionInfo | null; + readonly loopDepth: number; +} + +interface PendingIf { + readonly s: ts.IfStatement; + readonly cond: number; + readonly env: WalkEnv; + thenDone: boolean; + elseDone: boolean; +} + +/** Whether a value of the atom may be truthy (only `null`, `undefined` and + * the literal `false` never are; ⊤ may be anything). */ +function mayBeTruthy(a: Atom): boolean { + if (a.kind !== "prim") return true; + if (a.label === "null" || a.label === "undefined") return false; + return a.literal !== false; +} + +/** Whether a value of the atom may be falsy (objects, arrays, functions, + * classes, promises and collections never are). */ +function mayBeFalsy(a: Atom): boolean { + if (a.kind === "top") return true; + if (a.kind !== "prim") return false; + return a.literal !== true; } diff --git a/packages/compiler/src/frontend/shape-inference/domain.ts b/packages/compiler/src/frontend/shape-inference/domain.ts index d90a83cf2..21e52346c 100644 --- a/packages/compiler/src/frontend/shape-inference/domain.ts +++ b/packages/compiler/src/frontend/shape-inference/domain.ts @@ -38,6 +38,8 @@ export interface Atom { /** function atoms: the analysis context of the function's clone (a * method's receiver class), "" for the context-insensitive default. */ readonly ctx?: string | undefined; + /** prim `boolean` atoms of a literal: its value. */ + readonly literal?: boolean | undefined; } export class AtomTable { diff --git a/packages/compiler/src/frontend/shape-inference/index.ts b/packages/compiler/src/frontend/shape-inference/index.ts index 402a788b1..e2f7add04 100644 --- a/packages/compiler/src/frontend/shape-inference/index.ts +++ b/packages/compiler/src/frontend/shape-inference/index.ts @@ -1,7 +1,13 @@ /* Shape inference for untyped npm JavaScript - * (results/design-shape-inference/DESIGN.md). Stage S1: report only. */ + * (results/design-shape-inference/DESIGN.md): the report (S1) and the typed + * overlay (overlay.ts). */ import type * as ts from "../ts7/adapter.js"; -import { analyzeShapes } from "./analyze.js"; +import type { IrFunction } from "../../ir/ir.js"; +import type { ScrDiagnostic } from "../../diagnostics/diagnostic.js"; +import { npmStaticPackageOfPath } from "../npm-static.js"; +import { isShapelessPackage } from "../package-shims.js"; +import { analyzeShapes, isPackageJsFile } from "./analyze.js"; +import { OVERLAY_MARK, planShapeOverlay } from "./overlay.js"; import { buildShapeReport } from "./report.js"; import { computeConstantGraphs, constantGraphReport } from "./constant-graphs.js"; @@ -10,7 +16,8 @@ export { buildShapeReport } from "./report.js"; /** SCRIPTC_DEBUG_SHAPES: `1` reports package code in hot functions, * `verbose` every package site, `all` user files too, any other value only - * files whose path contains it. Writes to stderr; changes nothing else. */ + * files whose path contains it. Writes to stderr; changes nothing else. The + * build cache does not key on the variable: use SCRIPTC_NO_CACHE=1. */ export function reportShapes(program: ts.Program, filter: string): void { const start = performance.now(); const analysis = analyzeShapes(program, program.getTypeChecker()); @@ -22,3 +29,134 @@ export function reportShapes(program: ts.Program, filter: string): void { const ms = (performance.now() - start).toFixed(0); process.stderr.write(`[shapes] ${ms} ms\n${lines.map((l) => `[shapes] ${l}`).join("\n")}\n`); } + +/* The overlay's files for the current load: the lowering treats them as + * checked JavaScript (their JSDoc types are the analysis's proven facts). */ +let overlayFiles: ReadonlySet = new Set(); +let overlayAnnotations = 0; +let overlayDeadBranches = 0; + +export function isShapeOverlayFile(fileName: string): boolean { + return overlayFiles.has(fileName); +} + +/** Whether a statement or declaration carries an overlay annotation. */ +export function shapeOverlayAnnotated(node: ts.Node): boolean { + const sf = node.getSourceFile(); + if (!overlayFiles.has(sf.fileName)) return false; + return sf.text.slice(node.pos, node.getStart(sf)).includes(OVERLAY_MARK); +} + +export function shapeOverlayFileNames(): readonly string[] { + return [...overlayFiles]; +} + +export function shapeOverlayAnnotationCount(): number { + return overlayAnnotations; +} + +export function resetShapeOverlay(): void { + overlayFiles = new Set(); + overlayAnnotations = 0; + overlayDeadBranches = 0; + inferredFactChecks = 0; +} + +/** Whether the overlay runs. While it is new it is opt-in + * (SCRIPTC_SHAPE_OVERLAY=1); `--no-shape-overlay` (the `shapeOverlay: + * false` build option) sets the variable to 0 for the build. */ +export function shapeOverlayEnabled(): boolean { + return process.env["SCRIPTC_SHAPE_OVERLAY"] === "1"; +} + +/** loadProgram7's overlayer: analyze, plan, and register the overlay files. */ +export function shapeOverlayer(program: ts.Program): ReadonlyMap | null { + resetShapeOverlay(); + const analysis = analyzeShapes(program, program.getTypeChecker()); + if (analysis.budgetExceeded) return null; + const report = buildShapeReport(analysis, "1"); + const packageFiles = analysis.files.filter((sf) => { + if (!isPackageJsFile(sf)) return false; + const pkg = npmStaticPackageOfPath(sf.fileName); + return pkg === null || !isShapelessPackage(pkg); + }); + const overlay = planShapeOverlay(analysis, report.sites, packageFiles); + overlayFiles = new Set(overlay.files.keys()); + overlayAnnotations = overlay.annotations; + overlayDeadBranches = overlay.deadBranches; + if (process.env["SCRIPTC_DEBUG_SHAPE_OVERLAY"] !== undefined) { + for (const [file, text] of overlay.files) + process.stderr.write(`[shape-overlay] ${file}\n${text}\n`); + } + return overlay.files; +} + +/** Marks the checked conversions lowered in overlay files: each checks an + * analysis-proven fact, so its failure names the site as an analysis bug. */ +export function markInferredFactChecks( + functions: readonly IrFunction[], + program: ts.Program, +): number { + if (overlayFiles.size === 0) return 0; + let count = 0; + const lines = new Map number>(); + const lineOf = (file: string, pos: number): number => { + let f = lines.get(file); + if (f === undefined) { + const sf = program.getSourceFile(file); + f = sf === undefined ? () => 0 : (p: number) => sf.getLineAndCharacterOfPosition(p).line + 1; + lines.set(file, f); + } + return f(pos); + }; + const visit = (x: unknown): void => { + if (Array.isArray(x)) { + for (const v of x) visit(v); + return; + } + if (x === null || typeof x !== "object") return; + const node = x as { + kind?: unknown; + loc?: { file: string; start: number }; + inferredSite?: string; + }; + if (node.kind === "dynCheck" && node.loc !== undefined && overlayFiles.has(node.loc.file)) { + const pkg = npmStaticPackageOfPath(node.loc.file) ?? ""; + const rel = node.loc.file.split("/node_modules/").pop() ?? node.loc.file; + node.inferredSite = `${pkg} ${rel}:${lineOf(node.loc.file, node.loc.start)}`; + count++; + } + for (const v of Object.values(x)) if (v !== null && typeof v === "object") visit(v); + }; + for (const fn of functions) visit(fn.body); + inferredFactChecks = count; + return count; +} + +let inferredFactChecks = 0; + +/** Inferred-fact checks inserted by the last build (the build report). */ +export function shapeInferredFactCheckCount(): number { + return inferredFactChecks; +} + +/** SC6103, the overlay's build report: what the overlay typed and how many + * checks of inferred facts the build inserted. A note, never a failure. */ +export function shapeOverlayReport(entryFile: string): ScrDiagnostic[] { + if (overlayFiles.size === 0) return []; + const packages = new Set([...overlayFiles].map((f) => npmStaticPackageOfPath(f) ?? f)); + return [ + { + code: "SC6103", + message: + `shape inference typed ${overlayAnnotations} declarations in ${overlayFiles.size} files of ${packages.size} ` + + `package${packages.size === 1 ? "" : "s"} (${[...packages].sort().join(", ")}); the build checks ` + + `${inferredFactChecks} inferred fact${inferredFactChecks === 1 ? "" : "s"} at run time` + + (overlayDeadBranches > 0 + ? ` and replaces ${overlayDeadBranches} branch${overlayDeadBranches === 1 ? "" : "es"} it proved unreachable with checks` + : ""), + loc: { file: entryFile, start: 0, end: 0 }, + hint: "a failing check reports a scriptc analysis bug; --no-shape-overlay compiles without the overlay", + }, + ]; +} diff --git a/packages/compiler/src/frontend/shape-inference/overlay.ts b/packages/compiler/src/frontend/shape-inference/overlay.ts new file mode 100644 index 000000000..28b5336cf --- /dev/null +++ b/packages/compiler/src/frontend/shape-inference/overlay.ts @@ -0,0 +1,615 @@ +/* Shape inference: the typed overlay (results/design-shape-inference, + * maintainer decision 2026-10-23). The analysis's proven facts become JSDoc + * types in a rewritten copy of each package JavaScript file, and the + * lowering treats the rewritten file like a checked JavaScript file, so the + * static tier compiles the package's hot code from those types. + * + * Only proven facts are written: a position whose value set contains ⊤ + * (anything from islands, JSON.parse, the network or the file system, + * reflection, globals, `process`, caught exceptions, or code the analysis + * cannot see) stays untyped. A check the lowering inserts where a value + * meets an inferred type therefore fails only when the analysis is wrong; + * such checks carry the inferred-fact failure message. + * + * The package's own JSDoc is a hint, not a declaration: its tags are + * neutralized in the rewritten file, so only the analysis's types remain. + * + * Every edit is inline: no line breaks are added or removed, so positions + * keep their lines. */ +import * as ts from "../ts7/adapter.js"; +import type { Atom } from "./domain.js"; +import type { ShapeAnalysis } from "./analyze.js"; +import type { SiteVerdict } from "./report.js"; + +export interface ShapeOverlay { + /** Rewritten text per package file. */ + readonly files: Map; + /** Annotations written, for the build report. */ + readonly annotations: number; + /** Branches the analysis proved unreachable, replaced by checks. */ + readonly deadBranches: number; +} + +interface Edit { + readonly pos: number; + readonly text: string; + /** Replaces pos..end instead of inserting at pos. */ + readonly end?: number; +} + +/** Marks the overlay's own comments (an unknown JSDoc tag, ignored). */ +export const OVERLAY_MARK = "@scriptcShape"; +const MARK = OVERLAY_MARK; + +/** Above 1, values of several record key sets type as one merged record + * with optional fields (SCRIPTC_SHAPE_RECORD_ARMS; experimental). */ +const MAX_RECORD_ARMS = Number(process.env.SCRIPTC_SHAPE_RECORD_ARMS ?? 1); + +/** The most class or record arms an inferred union may have. */ +const MAX_ARMS = 6; + +function list(arr: { readonly nodes: readonly T[] } | undefined): readonly T[] { + return arr?.nodes ?? []; +} + +let debugIndex = 0; + +export function planShapeOverlay( + analysis: ShapeAnalysis, + verdicts: readonly SiteVerdict[], + packageFiles: readonly ts.SourceFile[], +): ShapeOverlay { + const pSites = new Map(); + const siteVerdicts = new Set(verdicts.map((v) => v.site.atom)); + // Records and arrays print only for sites whose objects never escape: a + // native record that reached untyped code would be seen as a copy. + const siteKeys = new Map }>(); + for (const v of verdicts) { + if (v.verdict !== "P-local") continue; + if (v.site.kind === "array") { + pSites.set(v.site.atom, v); + continue; + } + const keys = keysOfSite(analysis, v); + if (keys === null) continue; + siteKeys.set(v.site.atom, keys); + pSites.set(v.site.atom, v); + } + const files = new Map(); + let annotations = 0; + let deadBranches = 0; + debugIndex = 0; + // `if` branches no condition value allowed, per file: the overlay + // replaces each with a throwing check, so the checker and the lowering + // see the code the analysis saw. + const deadByFile = new Map(); + if (process.env["SCRIPTC_SHAPE_DEAD_BRANCHES"] !== "0") + for (const s of analysis.branchIfs) { + for (const branch of [s.thenStatement, s.elseStatement]) { + if (branch === undefined || analysis.walkedBranches.has(branch)) continue; + const sf = s.getSourceFile(); + let list = deadByFile.get(sf); + if (list === undefined) deadByFile.set(sf, (list = [])); + list.push(branch); + } + } + for (const sf of packageFiles) { + const typedefs = new Map(); + const typedefText: string[] = []; + const edits: Edit[] = []; + + const className = (decl: ts.Node): string | null => { + if (decl.getSourceFile() !== sf) return null; + const own = (decl as ts.ClassLikeDeclaration).name; + if (own !== undefined && ts.isClassDeclaration(decl) && decl.parent === sf) return own.text; + const p = decl.parent; + if ( + p !== undefined && + ts.isVariableDeclaration(p) && + ts.isIdentifier(p.name) && + p.parent?.parent?.parent === sf + ) + return p.name.text; + return null; + }; + + // `inner`: inside a record field, where ⊤ and unprintable values are `any`. + const typeOf = (atoms: ReadonlySet, inner: boolean, depth: number): string | null => { + if (atoms.size === 0) return inner ? "any" : null; + const arms = new Set(); + let objects = 0; + // Every array arm becomes one Array<…> of the joined element values. + const elements = new Set(); + let hasArray = false; + const promised = new Set(); + let hasPromise = false; + const recordSites: Atom[] = []; + for (const a of atoms) { + let t: string | null = null; + switch (a.kind) { + case "prim": + t = + a.label === "null" || + a.label === "undefined" || + a.label === "string" || + a.label === "number" || + a.label === "boolean" || + a.label === "bigint" + ? a.label + : null; + break; + case "instance": + t = a.node !== undefined ? className(a.node) : null; + objects++; + break; + case "object": { + objects++; + const v = pSites.get(a); + if (v === undefined || depth >= 4 || !siteKeys.has(a)) return inner ? "any" : null; + recordSites.push(a); + continue; + } + case "array": { + objects++; + if (analysis.escaped.has(a) || (siteVerdicts.has(a) && !pSites.has(a))) + return inner ? "any" : null; + const el = analysis.elements.get(a); + if (el !== undefined) for (const x of analysis.nodes[el]!.atoms) elements.add(x); + hasArray = true; + continue; + } + case "promise": { + // One promise arm: the values it resolves to, joined. + const value = analysis.fields.get(a)?.get("value"); + if (value !== undefined) for (const x of analysis.nodes[value]!.atoms) promised.add(x); + hasPromise = true; + continue; + } + default: + t = null; + } + if (t === null) return inner ? "any" : null; + arms.add(t); + } + // Records of one key set print as their typedef; records of several + // key sets merge into one record whose fields missing from some sites + // are optional (unions of overlapping records would need the lowering + // to pick an arm per value). + if (recordSites.length > 0) { + const keySets = new Set(recordSites.map(keySetOf)); + if (keySets.size === 1) arms.add(recordType(recordSites[0]!, depth)); + else if (MAX_RECORD_ARMS > 1) arms.add(mergedRecordType(recordSites, depth)); + else return inner ? "any" : null; + } + if (hasArray) { + const inside = depth < 4 ? typeOf(elements, true, depth + 1) : "any"; + arms.add(`Array<${inside ?? "any"}>`); + } + // A promise beside other values cannot cross into untyped code + // (maybe-promise unions stay typed-only), so it stays untyped here. + if (hasPromise) return inner ? "any" : null; + if (objects > MAX_ARMS) return inner ? "any" : null; + if (arms.has("any")) return inner ? "any" : null; + return [...arms].join(" | "); + }; + + // One typedef per key set: sites with the same keys intern to the same + // structural record, so their field types are joined. + const keySetOf = (site: Atom): string => { + const k = siteKeys.get(site)!; + return k.keys + .map((key) => (k.optional.has(key) ? `${key}?` : key)) + .sort() + .join(","); + }; + const sitesByKeys = new Map(); + for (const site of pSites.keys()) { + if (pSites.get(site)!.site.kind !== "object") continue; + const k = keySetOf(site); + let l = sitesByKeys.get(k); + if (l === undefined) sitesByKeys.set(k, (l = [])); + l.push(site); + } + const recordType = (site: Atom, depth: number): string => { + const keySet = keySetOf(site); + const existing = typedefs.get(keySet); + if (existing !== undefined) return existing; + const name = `__shape_${typedefs.size}`; + typedefs.set(keySet, name); + const { keys, optional } = siteKeys.get(site)!; + const parts = keys.map((k) => { + const atoms = new Set(); + for (const other of sitesByKeys.get(keySet) ?? [site]) { + const id = analysis.fields.get(other)?.get(k); + if (id !== undefined) for (const a of analysis.nodes[id]!.atoms) atoms.add(a); + } + const t = typeOf(atoms, true, depth + 1); + return `${/^[A-Za-z_$][\w$]*$/.test(k) ? k : JSON.stringify(k)}${optional.has(k) ? "?" : ""}: ${t ?? "any"}`; + }); + typedefText.push(`/** @typedef {{${parts.join(", ")}}} ${name} ${MARK} */`); + return name; + }; + + const mergedRecordType = (sites: readonly Atom[], depth: number): string => { + const all = new Set(); + for (const site of sites) + for (const other of sitesByKeys.get(keySetOf(site)) ?? [site]) all.add(other); + const keys: string[] = []; + const optional = new Set(); + for (const site of all) { + const k = siteKeys.get(site)!; + for (const key of k.keys) { + if (!keys.includes(key)) keys.push(key); + if (k.optional.has(key)) optional.add(key); + } + } + for (const site of all) { + const own = siteKeys.get(site)!.keys; + for (const key of keys) if (!own.includes(key)) optional.add(key); + } + const keySet = `merged:${keys + .map((k) => (optional.has(k) ? `${k}?` : k)) + .sort() + .join(",")}`; + const existing = typedefs.get(keySet); + if (existing !== undefined) return existing; + const name = `__shape_${typedefs.size}`; + typedefs.set(keySet, name); + const parts = keys.map((k) => { + const atoms = new Set(); + for (const site of all) { + const id = analysis.fields.get(site)?.get(k); + if (id !== undefined) for (const a of analysis.nodes[id]!.atoms) atoms.add(a); + } + const t = typeOf(atoms, true, depth + 1); + return `${/^[A-Za-z_$][\w$]*$/.test(k) ? k : JSON.stringify(k)}${optional.has(k) ? "?" : ""}: ${t ?? "any"}`; + }); + typedefText.push(`/** @typedef {{${parts.join(", ")}}} ${name} ${MARK} */`); + return name; + }; + + // Return values merged over every analysis context of each function. + const returns = new Map>(); + for (const info of analysis.functions.values()) { + if (info.node.getSourceFile() !== sf || info.isAsync || info.isGenerator) continue; + let set = returns.get(info.node); + if (set === undefined) returns.set(info.node, (set = new Set())); + for (const a of analysis.nodes[info.ret]!.atoms) set.add(a); + } + + // Receivers each method runs on, merged over its analysis contexts. + const thisAtoms = new Map>(); + for (const info of analysis.functions.values()) { + if (info.node.getSourceFile() !== sf || !ts.isMethodDeclaration(info.node)) continue; + let set = thisAtoms.get(info.node); + if (set === undefined) thisAtoms.set(info.node, (set = new Set())); + for (const a of analysis.nodes[info.thisNode]!.atoms) set.add(a); + } + const declaresMember = (cls: ts.Node, name: string): boolean => + analysis.classChain(cls).some((c) => + (c as ts.ClassLikeDeclaration).members.nodes.some((m) => { + const n = (m as { name?: ts.Node }).name; + return ( + n !== undefined && (ts.isIdentifier(n) || ts.isPrivateIdentifier(n)) && n.text === name + ); + }), + ); + // `this.m(…)` in an inherited method whose class does not declare `m` + // (zod's ZodType._parseSync calls the subclasses' _parse): the receiver + // is cast to the classes the method runs on, so the call dispatches over + // them instead of through an untyped call. + const receiverCasts = (method: ts.MethodDeclaration): void => { + const cls = method.parent; + const receivers = thisAtoms.get(method); + if (cls === undefined || receivers === undefined || receivers.size === 0) return; + if (![...receivers].every((a) => a.kind === "instance")) return; + const t = typeOf(receivers, false, 0); + if (t === null) return; + const scan = (node: ts.Node): void => { + if (ts.isFunctionLike(node) && !ts.isArrowFunction(node)) return; + if ( + ts.isCallExpression(node) && + ts.isPropertyAccessExpression(node.expression) && + node.expression.expression.kind === ts.SyntaxKind.ThisKeyword && + ts.isIdentifier(node.expression.name) && + !declaresMember(cls, node.expression.name.text) + ) { + const self = node.expression.expression; + edits.push({ pos: ts.tokenStartOfNode(self), text: `/** @type {${t}} ${MARK} */ (` }); + edits.push({ pos: self.end, text: ")" }); + annotations++; + } + ts.forEachChild(node, (c) => { + scan(c); + return undefined; + }); + }; + if (method.body !== undefined) for (const st of method.body.statements.nodes) scan(st); + }; + + // Same-named instance methods of the file's classes form a family: a + // call on a union of those classes needs one signature, so each member's + // parameters and result join the whole family's values. + const families = new Map(); + const familyVisit = (node: ts.Node): void => { + if ( + ts.isMethodDeclaration(node) && + ts.isIdentifier(node.name) && + !list((node as { modifiers?: { nodes: readonly ts.Node[] } }).modifiers).some( + (m) => m.kind === ts.SyntaxKind.StaticKeyword, + ) + ) { + let l = families.get(node.name.text); + if (l === undefined) families.set(node.name.text, (l = [])); + l.push(node); + } + ts.forEachChild(node, (c) => { + familyVisit(c); + return undefined; + }); + }; + familyVisit(sf); + // Members the program runs (they have analysis contexts); members it + // never reaches would only widen the family's types. + const reached = new Set(); + for (const info of analysis.functions.values()) + if (info.node.getSourceFile() === sf) reached.add(info.node); + const familyOf = (fn: ts.FunctionLikeDeclaration): readonly ts.MethodDeclaration[] => { + if (!ts.isMethodDeclaration(fn) || !ts.isIdentifier(fn.name)) + return [fn as ts.MethodDeclaration]; + const members = (families.get(fn.name.text) ?? [fn]).filter((m) => reached.has(m)); + return members.length > 0 ? members : [fn]; + }; + const paramValues = (fn: ts.FunctionLikeDeclaration, index: number): Set => { + const out = new Set(); + for (const member of familyOf(fn)) { + const p = list(member.parameters)[index]; + if (p === undefined) continue; + for (const a of analysis.declValues(p)) out.add(a); + } + return out; + }; + const returnValues = (fn: ts.FunctionLikeDeclaration): Set | undefined => { + let out: Set | undefined; + for (const member of familyOf(fn)) { + const r = returns.get(member); + if (r === undefined) continue; + out ??= new Set(); + for (const a of r) out.add(a); + } + return out; + }; + + const paramTags = (fn: ts.FunctionLikeDeclaration): string | null => { + const tags: string[] = []; + if (!ts.isConstructorDeclaration(fn)) { + const ret = returnValues(fn); + const t = ret !== undefined && ret.size > 0 ? typeOf(ret, false, 0) : null; + const why = process.env["SCRIPTC_DEBUG_SHAPE_OVERLAY_WHY"]; + if ( + why !== undefined && + t === null && + ret !== undefined && + ts.isMethodDeclaration(fn) && + ts.isIdentifier(fn.name) && + fn.name.text === why + ) + process.stderr.write( + `[shape-overlay-why] ${fn.name.text}: ${[...ret].map((a) => `${a.kind}:${a.label}${analysis.escaped.has(a) ? "(escaped)" : ""}`).join(" | ")}\n`, + ); + if (t !== null) tags.push(`@returns {${t}}`); + } + list(fn.parameters).forEach((p, index) => { + if ( + !ts.isIdentifier(p.name) || + p.dotDotDotToken !== undefined || + p.initializer !== undefined || + p.questionToken !== undefined + ) + return; + const t = typeOf(paramValues(fn, index), false, 0); + if (t !== null) tags.push(`@param {${t}} ${p.name.text}`); + }); + return tags.length > 0 ? `${tags.join(" ")} ${MARK}` : null; + }; + + const visit = (node: ts.Node): void => { + if (ts.isMethodDeclaration(node)) receiverCasts(node); + if ( + ts.isMethodDeclaration(node) || + ts.isConstructorDeclaration(node) || + ts.isFunctionDeclaration(node) + ) { + if (node.body !== undefined) { + const tags = paramTags(node); + if (tags !== null) { + edits.push({ pos: ts.tokenStartOfNode(node), text: `/** ${tags} */ ` }); + annotations++; + } + } + // `this.f = …` fields of a constructor's top-level statements. + if (ts.isConstructorDeclaration(node) && node.body !== undefined) { + const cls = node.parent; + const seen = new Set(); + for (const st of list(node.body.statements)) { + if (!ts.isExpressionStatement(st) || !ts.isBinaryExpression(st.expression)) continue; + const b = st.expression; + if ( + b.operatorToken.kind !== ts.SyntaxKind.EqualsToken || + !ts.isPropertyAccessExpression(b.left) + ) + continue; + if ( + b.left.expression.kind !== ts.SyntaxKind.ThisKeyword || + !ts.isIdentifier(b.left.name) + ) + continue; + const name = b.left.name.text; + if (seen.has(name)) continue; + seen.add(name); + const t = cls !== undefined ? fieldType(cls, name) : null; + if (t !== null) { + edits.push({ pos: ts.tokenStartOfNode(st), text: `/** @type {${t}} ${MARK} */ ` }); + annotations++; + } + } + } + } + // Locals only: module-level bindings are initialized once (and a + // record-typed module constant can turn a return type into a union of + // overlapping records, which the lowering width-coerces). + if ( + ts.isVariableStatement(node) && + (node.parent !== sf || process.env["SCRIPTC_SHAPE_OVERLAY_MODULE_VARS"] === "1") && + node.declarationList.declarations.nodes.length === 1 + ) { + const d = node.declarationList.declarations.nodes[0]!; + const init = d.initializer; + if (ts.isIdentifier(d.name) && init !== undefined && !literalInitializer(init)) { + const t = typeOf(analysis.declValues(d), false, 0); + if (t !== null) { + edits.push({ pos: ts.tokenStartOfNode(node), text: `/** @type {${t}} ${MARK} */ ` }); + annotations++; + } + } + } + ts.forEachChild(node, (c) => { + visit(c); + return undefined; + }); + }; + + const fieldType = (cls: ts.Node, name: string): string | null => { + const atoms = new Set(); + for (const a of analysis.atoms.atoms) { + if ( + a.kind !== "instance" || + a.node === undefined || + !analysis.classChain(a.node).includes(cls) + ) + continue; + const id = analysis.fields.get(a)?.get(name); + if (id !== undefined) for (const x of analysis.nodes[id]!.atoms) atoms.add(x); + } + return typeOf(atoms, false, 0); + }; + + visit(sf); + // Dead branches: outermost first; edits inside a replaced range drop. + const dead = (deadByFile.get(sf) ?? []).sort((a, b) => a.pos - b.pos); + const ranges: [number, number][] = []; + for (const branch of dead) { + const start = ts.tokenStartOfNode(branch); + if (ranges.some(([lo, hi]) => start >= lo && start < hi)) continue; + ranges.push([start, branch.end]); + } + if (ranges.length > 0) { + const kept = edits.filter((e) => !ranges.some(([lo, hi]) => e.pos > lo && e.pos < hi)); + edits.length = 0; + edits.push(...kept); + const pkg = packageNameOf(sf.fileName); + const rel = sf.fileName.split("/node_modules/").pop() ?? sf.fileName; + for (const [lo, hi] of ranges) { + const line = sf.getLineAndCharacterOfPosition(lo).line + 1; + const message = + `scriptc internal error: shape inference proved the branch at ${rel}:${line} unreachable, but it ran. ` + + `This is a bug in scriptc's shape analysis, not in the program. Rebuild with --no-shape-overlay, ` + + `or set "scriptc": {"packages": {"${pkg}": {"shapes": false}}} in package.json.`; + edits.push({ pos: lo, end: hi, text: `{ throw new Error(${JSON.stringify(message)}); }` }); + deadBranches++; + } + } + // Debugging: SCRIPTC_SHAPE_OVERLAY_ONLY=a-b keeps the annotations + // numbered a..b (program order across files), to bisect a miscompile. + const only = process.env["SCRIPTC_SHAPE_OVERLAY_ONLY"]; + if (only !== undefined) { + const [lo, hi] = only.split("-").map(Number) as [number, number]; + edits.sort((a, b) => a.pos - b.pos); + const kept = edits.filter(() => { + const n = debugIndex++; + return n >= lo && n <= hi; + }); + edits.length = 0; + edits.push(...kept); + } + if (edits.length === 0) continue; + files.set(sf.fileName, neutralizeDocTags(applyEdits(sf.text, edits)) + typedefText.join("")); + } + return { files, annotations, deadBranches }; +} + +function packageNameOf(fileName: string): string { + const parts = (fileName.split("/node_modules/").pop() ?? "").split("/"); + return parts[0]!.startsWith("@") ? `${parts[0]}/${parts[1]}` : parts[0]!; +} + +function literalInitializer(e: ts.Expression): boolean { + return ( + ts.isObjectLiteralExpression(e) || + (ts.isArrayLiteralExpression(e) && e.elements.nodes.length > 0) || + ts.isStringLiteral(e) || + ts.isNumericLiteral(e) || + ts.isNoSubstitutionTemplateLiteral(e) || + ts.isArrowFunction(e) || + ts.isFunctionExpression(e) || + e.kind === ts.SyntaxKind.TrueKeyword || + e.kind === ts.SyntaxKind.FalseKeyword + ); +} + +function applyEdits(text: string, edits: readonly Edit[]): string { + const sorted = [...edits].sort((a, b) => a.pos - b.pos); + let out = ""; + let at = 0; + for (const e of sorted) { + out += text.slice(at, e.pos) + e.text; + at = e.end ?? e.pos; + } + return out + text.slice(at); +} + +/** The package's JSDoc type tags become plain text: package JSDoc is a + * hint, not a declaration. The overlay's own comments carry MARK and are + * kept. Lengths are unchanged. */ +export function neutralizeDocTags(text: string): string { + return text.replace(/\/\*\*[\s\S]*?\*\//g, (comment) => { + if (comment.includes(MARK)) return comment; + return comment.replace( + /@(param|type|typedef|returns?|template|this|callback|property|prop|arg|argument|enum|implements|extends|augments|satisfies|overload|import)\b/g, + "-$1", + ); + }); +} + +/** A record site's keys: its static keys, or for a literal with spreads its + * own keys plus every key a spread source contributes (optional unless + * every value of that source has it). Null when a computed key or an + * untracked spread source makes the key set unknown. */ +function keysOfSite( + analysis: ShapeAnalysis, + v: SiteVerdict, +): { keys: string[]; optional: Set } | null { + const site = v.site; + if (site.keys !== null) return { keys: site.keys, optional: new Set() }; + if (site.hasComputedKey) return null; + const fields = analysis.fields.get(site.atom); + const any = fields?.get("*"); + if (any !== undefined && analysis.nodes[any]!.atoms.size > 0) return null; + const required = new Set(site.ownKeys); + for (const spread of site.spreads) { + const sets: Set[] = []; + for (const a of analysis.nodes[spread]!.atoms) { + if (a.kind === "prim" && (a.label === "undefined" || a.label === "null")) continue; + if (a.kind !== "object") return null; + const own = new Set(); + for (const k of analysis.fields.get(a)?.keys() ?? []) if (k !== "*") own.add(k); + sets.push(own); + } + const first = sets[0]; + if (first === undefined) continue; + for (const k of first) if (sets.every((set) => set.has(k))) required.add(k); + } + const all = [...site.ownKeys]; + for (const k of fields?.keys() ?? []) if (k !== "*" && !all.includes(k)) all.push(k); + return { keys: all, optional: new Set(all.filter((k) => !required.has(k))) }; +} diff --git a/packages/compiler/src/index.ts b/packages/compiler/src/index.ts index e2d1c97cc..c0f0e0649 100644 --- a/packages/compiler/src/index.ts +++ b/packages/compiler/src/index.ts @@ -1,3 +1,4 @@ +import { shapeOverlayEnabled } from "./frontend/shape-inference/index.js"; import { optimizationField, type NativeOptimization } from "./backend/optimization.js"; import { targetCpuField, targetCpuTargetRefusal } from "./backend/target-cpu.js"; import { compilationTiming, type CompilationTiming, type TimingClock } from "./timing.js"; @@ -442,6 +443,10 @@ export async function compile( opts: CompileRequestOptions, ): Promise { clearCompileSessionCaches(); + // `shapeOverlay: false` reaches the frontend (and its child process) + // through the environment the overlay reads. + const overlayEnv = process.env["SCRIPTC_SHAPE_OVERLAY"]; + if (opts.shapeOverlay === false) process.env["SCRIPTC_SHAPE_OVERLAY"] = "0"; const frontendInputs = new FrontendInputTracker(); const warnings: CompileWarningSink = { warnings: [], sourceTexts: null }; // compileTracked starts it for a build with no earlier build of the same invocation. @@ -454,6 +459,10 @@ export async function compile( } finally { setLibDiagnosticsStore(null); frontendProcess?.release(); + if (opts.shapeOverlay === false) { + if (overlayEnv === undefined) delete process.env["SCRIPTC_SHAPE_OVERLAY"]; + else process.env["SCRIPTC_SHAPE_OVERLAY"] = overlayEnv; + } } } @@ -1190,6 +1199,8 @@ function configureLibDiagnostics(root: string | null): void { export async function runFrontendProcess(requestPath: string, resultPath: string): Promise { const request = JSON.parse(await readFile(requestPath, "utf8")) as FrontendProcessRequest; + // The process started before the build's options were known. + if (request.opts.shapeOverlay === false) process.env["SCRIPTC_SHAPE_OVERLAY"] = "0"; clearCompileSessionCaches(); setProvenanceSources(request.provenance); // The compile that started this process prepared the cache root. @@ -1520,6 +1531,7 @@ async function compileTracked( ...checkerField(), ...(youngNurseryRequested() ? { youngNursery: true as const } : {}), npmStatic: opts.npmStatic ?? null, + ...(shapeOverlayEnabled() ? { shapeOverlay: true as const } : {}), ffiProfile: opts.ffiProfilePath === undefined || ffiProfileBytes === null ? null diff --git a/packages/compiler/src/ir/function-references.test.ts b/packages/compiler/src/ir/function-references.test.ts index db872a525..6c13d3916 100644 --- a/packages/compiler/src/ir/function-references.test.ts +++ b/packages/compiler/src/ir/function-references.test.ts @@ -100,6 +100,7 @@ const EXPR_FIELDS: Classified = { unionIsTag: { unionId: "other" }, unionEq: { unionId: "other" }, jsonStringify: { indent: "other" }, + dynCheck: { inferredSite: "other" }, }; const STMT_FIELDS: Classified = { @@ -222,6 +223,8 @@ const MODULE_FIELDS: Record>, Role> = { "wasmUntranslated[].reason": "other", "threadSlotKeys[]": "other", "freezableArrays[]": "other", + "inferredFactFiles[].file": "other", + "inferredFactFiles[].pkg": "other", }; const loc = { file: "references.ts", start: 0, end: 0 }; diff --git a/packages/compiler/src/ir/ir.ts b/packages/compiler/src/ir/ir.ts index 3818433f8..25fce8364 100644 --- a/packages/compiler/src/ir/ir.ts +++ b/packages/compiler/src/ir/ir.ts @@ -1061,6 +1061,11 @@ export interface IrModule { * closures boxed for package code return records and arrays as live * capsules, so a value that comes back keeps its identity. */ declBoundary?: true; + /** Package files compiled from shape inference's typed overlay + * (frontend/shape-inference/overlay.ts), each with its package name: a + * checked conversion in them checks an analysis-proven fact, so its + * failure reports a compiler analysis bug instead of a type error. */ + inferredFactFiles?: { file: string; pkg: string }[]; /** The program constructs LifetimeRegion (@scriptc/experimental): every * ProgramLifetime instance lives in a counted region (the region.* * intrinsics; backend/llvm/regions.ts). */ @@ -3984,6 +3989,10 @@ export type IrExpr = preserveRefs?: true; adopt?: true; exact?: true; + /** Shape inference's overlay: this conversion checks an + * analysis-proven fact (" :"); a failure is a + * compiler analysis bug and reports itself as one. */ + inferredSite?: string; type: IrType; loc: SrcLoc; }; diff --git a/packages/runtime/src/scr_dyn.h b/packages/runtime/src/scr_dyn.h index d3997ebd9..5e31d2aa9 100644 --- a/packages/runtime/src/scr_dyn.h +++ b/packages/runtime/src/scr_dyn.h @@ -1541,6 +1541,10 @@ void scr_dyn_check_fail(const ScrDynPath *path, const char *want, const ScrDyn * /* The Date object a checked-dynamic Date handle holds, +1, or NULL with * the check failure pending. */ ScrDate *scr_dyn_native_date_check(const ScrDyn *value, const ScrDynPath *path); +/* Names the analysis-proven fact the next checked conversion checks + * (shape inference's overlay), or NULL: its failure then reports a compiler + * analysis bug (" :"). */ +void scr_dyn_inferred_site(const char *site); /* Matchers and builders of recursive shapes (records and arrays whose type * reaches itself) track the dyn values they visit, so cyclic values match diff --git a/packages/runtime/src/scr_json.c b/packages/runtime/src/scr_json.c index 45ba766a1..39e930cf1 100644 --- a/packages/runtime/src/scr_json.c +++ b/packages/runtime/src/scr_json.c @@ -10782,7 +10782,53 @@ static const char *scr_dyn_kind_name(const ScrDyn *d) { return "unknown"; } +/* Shape inference's overlay (compiler frontend/shape-inference): a checked + * conversion of an analysis-proven fact names its site (" + * :") while it runs. A failure there is a compiler analysis + * bug, so it reports itself as one instead of as a type error. */ +static SCR_TL const char *scr_dyn_inferred_site_name; + +void scr_dyn_inferred_site(const char *site) { scr_dyn_inferred_site_name = site; } + +static void scr_dyn_inferred_fact_fail(const char *site, const ScrDynPath *path, const char *want, + const ScrDyn *got) { + const char *space = strchr(site, ' '); + size_t pkg_len = space ? (size_t)(space - site) : 0; + const char *where = space ? space + 1 : site; + ScrJsonBuf b; + scr_jb_init(&b); + scr_jb_puts(&b, "scriptc internal error: an inferred type was contradicted at "); + scr_jb_puts(&b, where); + scr_jb_puts(&b, " (expected "); + scr_jb_puts(&b, want); + scr_jb_puts(&b, " at "); + scr_dyn_path_render(&b, path); + scr_jb_puts(&b, ", got "); + scr_jb_puts(&b, scr_dyn_kind_name(got)); + scr_jb_puts(&b, "). This is a bug in scriptc's shape analysis, not in the program's data. " + "Rebuild with --no-shape-overlay, or set \"scriptc\": {\"packages\": {\""); + for (size_t i = 0; i < pkg_len; i++) scr_jb_putc(&b, site[i]); + scr_jb_puts(&b, "\": {\"shapes\": false}}} in package.json, to compile the package without it."); + /* Validation mode: every hit is logged, also when the program catches it. */ + const char *log = getenv("SCRIPTC_SHAPE_CHECK_LOG"); + if (log && *log) { + FILE *f = fopen(log, "a"); + if (f) { + fwrite(b.data, 1, b.len, f); + fputc('\n', f); + fclose(f); + } + } + scr_throw_error(SCR_ERR_ERROR, scr_jb_finish(&b)); +} + void scr_dyn_check_fail(const ScrDynPath *path, const char *want, const ScrDyn *got) { + const char *site = scr_dyn_inferred_site_name; + if (site) { + scr_dyn_inferred_site_name = NULL; + scr_dyn_inferred_fact_fail(site, path, want, got); + return; + } ScrJsonBuf b; scr_jb_init(&b); scr_jb_puts(&b, "expected "); diff --git a/tests/harness/shape-overlay.test.ts b/tests/harness/shape-overlay.test.ts new file mode 100644 index 000000000..ad1988238 --- /dev/null +++ b/tests/harness/shape-overlay.test.ts @@ -0,0 +1,62 @@ +/* The shape overlay (shape-inference/overlay.ts): with SCRIPTC_SHAPE_OVERLAY=1 + * the zod3-boundary npm case compiles zod's JavaScript with analysis-proven JSDoc + * types, reports what it typed and checks (SC6103), and prints what Node + * prints; the shapeOverlay: false option (the CLI's --no-shape-overlay) + * turns it off. */ +import { execFile } from "node:child_process"; +import { mkdirSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { promisify } from "node:util"; +import { expect, test } from "vitest"; +import { compile } from "@scriptc/compiler"; + +const execFileAsync = promisify(execFile); +const repoRoot = join(import.meta.dirname, "../.."); +const entry = join(repoRoot, "tests/fixtures/npm/cases/zod3-boundary/main.ts"); +const outDir = join(repoRoot, "node_modules/.cache/scriptc-tests/shape-overlay"); +const sanitize = process.env["SCRIPTC_SAN"] === "1"; + +async function withOverlay(run: () => Promise): Promise { + const saved = process.env["SCRIPTC_SHAPE_OVERLAY"]; + process.env["SCRIPTC_SHAPE_OVERLAY"] = "1"; + try { + return await run(); + } finally { + if (saved === undefined) delete process.env["SCRIPTC_SHAPE_OVERLAY"]; + else process.env["SCRIPTC_SHAPE_OVERLAY"] = saved; + } +} + +test( + "the overlay types zod's JavaScript and keeps Node's output", + { timeout: 900_000 }, + async () => { + mkdirSync(outDir, { recursive: true }); + const result = await withOverlay(() => + compile(entry, { outPath: join(outDir, "on"), outDir, sanitize }), + ); + if (!result.ok) + throw new Error(result.diagnostics.map((d) => `${d.code}: ${d.message}`).join("\n")); + const report = (result.warnings ?? []).find((w) => w.code === "SC6103"); + expect(report?.message).toMatch( + /shape inference typed \d+ declarations in \d+ files? of 1 package \(zod\)/, + ); + const run = execFileAsync(result.binaryPath, [], { cwd: dirname(entry), encoding: "utf8" }); + run.child.stdin?.end(); + const node = await execFileAsync(process.execPath, [entry], { + cwd: dirname(entry), + encoding: "utf8", + }); + expect((await run).stdout).toBe(node.stdout); + }, +); + +test("shapeOverlay: false compiles without the overlay", { timeout: 900_000 }, async () => { + mkdirSync(outDir, { recursive: true }); + const result = await withOverlay(() => + compile(entry, { outPath: join(outDir, "off"), outDir, sanitize, shapeOverlay: false }), + ); + if (!result.ok) + throw new Error(result.diagnostics.map((d) => `${d.code}: ${d.message}`).join("\n")); + expect((result.warnings ?? []).some((w) => w.code === "SC6103")).toBe(false); +}); From 688c1e03a15b69cf9dd0693a9199e45b08170d6d Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Sat, 24 Oct 2026 22:52:52 -0500 Subject: [PATCH 02/23] Keep undefined results, literal text and hoisted vars in the shape overlay - A function's results include undefined when a call can run off the end of its body or reach a bare return, so the overlay's return type keeps undefined and a caller's branch on it stays reachable. `new` of an ES5 constructor ignores primitive results, and functions called with `new` get no @returns tag. - Package JSDoc tags are neutralized only in comments: string, template and regular expression literals and JSX text keep their text. - A branch replaced as unreachable keeps the var names it declares that its scope declares nowhere else, so references outside it read undefined. A branch that declares a function in a block is kept. - The shape-overlay-facts npm case covers all three; four files of the overlay change are formatted. --- .../src/frontend/lowering/call-signatures.ts | 6 +- .../src/frontend/lowering/lower-classes.ts | 5 +- packages/compiler/src/frontend/pipeline.ts | 5 +- packages/compiler/src/frontend/program.ts | 6 +- .../src/frontend/shape-inference/analyze.ts | 64 +++++- .../frontend/shape-inference/overlay.test.ts | 32 +++ .../src/frontend/shape-inference/overlay.ts | 182 ++++++++++++++++-- .../cases/shape-overlay-facts/main.ts.json | 6 + .../npm/cases/shape-overlay-facts/main.ts | 10 + .../node_modules/shape-facts/index.d.ts | 4 + .../node_modules/shape-facts/index.js | 55 ++++++ .../node_modules/shape-facts/package.json | 1 + .../cases/shape-overlay-facts/package.json | 1 + tests/harness/shape-overlay.test.ts | 30 +++ 14 files changed, 384 insertions(+), 23 deletions(-) create mode 100644 packages/compiler/src/frontend/shape-inference/overlay.test.ts create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/shape-overlay-facts/main.ts.json create mode 100644 tests/fixtures/npm/cases/shape-overlay-facts/main.ts create mode 100644 tests/fixtures/npm/cases/shape-overlay-facts/node_modules/shape-facts/index.d.ts create mode 100644 tests/fixtures/npm/cases/shape-overlay-facts/node_modules/shape-facts/index.js create mode 100644 tests/fixtures/npm/cases/shape-overlay-facts/node_modules/shape-facts/package.json create mode 100644 tests/fixtures/npm/cases/shape-overlay-facts/package.json diff --git a/packages/compiler/src/frontend/lowering/call-signatures.ts b/packages/compiler/src/frontend/lowering/call-signatures.ts index 9d8096c62..3b2a4bd39 100644 --- a/packages/compiler/src/frontend/lowering/call-signatures.ts +++ b/packages/compiler/src/frontend/lowering/call-signatures.ts @@ -182,7 +182,11 @@ export function checkedJsDocumentedParameter( lowerer: Lowerer, param: ts.ParameterDeclaration, ): boolean { - if (param.parent !== undefined && shapeOverlayAnnotated(param.parent) && documentedParameter(param)) + if ( + param.parent !== undefined && + shapeOverlayAnnotated(param.parent) && + documentedParameter(param) + ) return true; return lowerer.checkedJsFile(ts.getSourceFileOfNode(param)) && documentedParameter(param); } diff --git a/packages/compiler/src/frontend/lowering/lower-classes.ts b/packages/compiler/src/frontend/lowering/lower-classes.ts index 573bb0c69..b2036dc8a 100644 --- a/packages/compiler/src/frontend/lowering/lower-classes.ts +++ b/packages/compiler/src/frontend/lowering/lower-classes.ts @@ -1156,7 +1156,10 @@ function jsFieldInitializerType(lowerer: Lowerer, type: IrType, expression: ts.E // Shape inference's overlay types a field as a record or array only when // the analysis proves its values never reach code that could observe a // copy, so those fields keep their native layout. - if ((type.kind === "array" || type.kind === "record") && !shapeOverlayAnnotated(expression.parent?.parent ?? expression)) + if ( + (type.kind === "array" || type.kind === "record") && + !shapeOverlayAnnotated(expression.parent?.parent ?? expression) + ) return DYN; // JavaScript documentation cannot guarantee that a call or an indexed // read returns a present number. Numeric fields receiving those values diff --git a/packages/compiler/src/frontend/pipeline.ts b/packages/compiler/src/frontend/pipeline.ts index 92bafe54d..ad64fb8d5 100644 --- a/packages/compiler/src/frontend/pipeline.ts +++ b/packages/compiler/src/frontend/pipeline.ts @@ -700,7 +700,10 @@ function loadFrontend( }); // The declaration-typed boundary's notes (SC6102) join the // performance warnings; they never fail a build. - const notes = [...(finalLoad.declBoundaryNotes ?? []), ...shapeOverlayReport(finalLoad.entry.fileName)]; + const notes = [ + ...(finalLoad.declBoundaryNotes ?? []), + ...shapeOverlayReport(finalLoad.entry.fileName), + ]; if (notes.length === 0) return lowered; return { ...lowered, performance: [...notes, ...(lowered.performance ?? [])] }; }, diff --git a/packages/compiler/src/frontend/program.ts b/packages/compiler/src/frontend/program.ts index daeb61ded..f44efb228 100644 --- a/packages/compiler/src/frontend/program.ts +++ b/packages/compiler/src/frontend/program.ts @@ -1018,7 +1018,11 @@ function loadProgram7( for (const [file, text] of overlay) host.addVirtualFile(file, text); program = host.createProgram([...programRoots, overridesDtsPath()], options); if (tscWorld === undefined) - tscWorld = { program: before, diagnostics: ts.getPreEmitDiagnostics(before), projectWorld: null }; + tscWorld = { + program: before, + diagnostics: ts.getPreEmitDiagnostics(before), + projectWorld: null, + }; else if (tscWorld.program !== before) before.dispose(); } } diff --git a/packages/compiler/src/frontend/shape-inference/analyze.ts b/packages/compiler/src/frontend/shape-inference/analyze.ts index fe87a8c53..cbdfef081 100644 --- a/packages/compiler/src/frontend/shape-inference/analyze.ts +++ b/packages/compiler/src/frontend/shape-inference/analyze.ts @@ -131,6 +131,8 @@ export interface ShapeAnalysis { readonly escapedFunctions: ReadonlySet; /** ES5 constructors: each function's default prototype object. */ readonly protoOf: ReadonlyMap; + /** ES5 constructor instances: the function atom whose `new` made each. */ + readonly fnOfInstance: ReadonlyMap; /** Every value a declaration (parameter, variable, binding element) holds, * merged over the analysis contexts it was analyzed in. */ declValues(decl: Node): Set; @@ -249,6 +251,47 @@ function isLoop(node: Node): boolean { ); } +/** Whether control can reach the end of `stmts`. False only when every + * path ends in a return or throw, or in a `while (true)`, `do … while + * (true)` or `for (;;)` loop with no break inside; anything else is + * assumed to complete. */ +function canComplete(stmts: readonly Node[]): boolean { + return stmts.every(canCompleteStatement); +} + +function canCompleteStatement(s: Node): boolean { + if (ts.isReturnStatement(s) || ts.isThrowStatement(s)) return false; + if (ts.isBlock(s)) return canComplete(list(s.statements)); + if (ts.isIfStatement(s)) + return ( + s.elseStatement === undefined || + canCompleteStatement(s.thenStatement) || + canCompleteStatement(s.elseStatement) + ); + if (ts.isTryStatement(s)) { + if (s.finallyBlock !== undefined && !canCompleteStatement(s.finallyBlock)) return false; + return ( + canCompleteStatement(s.tryBlock) || + (s.catchClause !== undefined && canCompleteStatement(s.catchClause.block)) + ); + } + const endless = + ((ts.isWhileStatement(s) || ts.isDoStatement(s)) && + s.expression.kind === ts.SyntaxKind.TrueKeyword) || + (ts.isForStatement(s) && s.condition === undefined); + return !endless || containsBreak(s); +} + +/** Whether a break statement occurs in `node`'s body, outside nested + * functions. */ +function containsBreak(node: Node): boolean { + return ( + ts.forEachChild(node, (c) => + ts.isBreakStatement(c) || (!ts.isFunctionLike(c) && containsBreak(c)) ? true : undefined, + ) === true + ); +} + const LOOP_CALLBACK_METHODS: ReadonlySet = new Set([ "map", "forEach", @@ -369,6 +412,7 @@ class Analyzer { nodeLabel: this.nodeLabel, escapedFunctions: this.escapedFunctionCalls, protoOf: this.protoOf, + fnOfInstance: this.fnOfInstance, declValues: (decl) => this.declValues(decl), classChain: (decl) => this.classChain(decl), branchIfs: this.branchIfs, @@ -918,8 +962,11 @@ class Analyzer { } const body = fn.body; if (body !== undefined) { - if (ts.isBlock(body)) this.stmts(list(body.statements)); - else this.edge(this.expr(body as ts.Expression), info.ret); + if (ts.isBlock(body)) { + this.stmts(list(body.statements)); + // A call that runs off the end of the body returns undefined. + if (canComplete(list(body.statements))) this.add(info.ret, this.prim("undefined")); + } else this.edge(this.expr(body as ts.Expression), info.ret); } this.currentFn = saved; this.loopDepth = savedLoop; @@ -1253,8 +1300,9 @@ class Analyzer { return; } if (ts.isReturnStatement(s)) { - if (s.expression !== undefined && this.currentFn !== null) - this.edge(this.expr(s.expression), this.currentFn.ret); + if (this.currentFn === null) return; + if (s.expression !== undefined) this.edge(this.expr(s.expression), this.currentFn.ret); + else this.add(this.currentFn.ret, this.prim("undefined")); return; } if (ts.isFunctionDeclaration(s)) { @@ -2667,7 +2715,11 @@ class Analyzer { this.activate(info); this.add(info.thisNode, inst); this.bindArgs(info, args, spread); - this.edge(info.ret, dst); + // `new` ignores a primitive result (and the undefined of a + // body that runs off its end) and yields the instance. + this.on(info.ret, (r) => { + if (r.kind !== "prim") this.add(dst, r); + }); this.edgeFor(e, info); } return; @@ -3350,7 +3402,7 @@ function cjsExportName(decl: Node): string | null { ) return decl.left.name.text; return null; - } +} interface WalkEnv { readonly ctx: string; diff --git a/packages/compiler/src/frontend/shape-inference/overlay.test.ts b/packages/compiler/src/frontend/shape-inference/overlay.test.ts new file mode 100644 index 000000000..f0effae77 --- /dev/null +++ b/packages/compiler/src/frontend/shape-inference/overlay.test.ts @@ -0,0 +1,32 @@ +import { afterAll, expect, test } from "vitest"; +import { closeSourceParser, parseSourceFile } from "../ts7/source-parser-node.js"; +import { neutralizeDocTags, OVERLAY_MARK } from "./overlay.js"; + +afterAll(closeSourceParser); + +test("neutralizeDocTags rewrites JSDoc comments and keeps literals", () => { + const source = [ + "/** @param {number} a */", + "function f(a) {", + ' const s = "/** @type {number} */";', + " const t = `/** @returns {string} ${a} */`;", + " const r = /\\/\\*\\* @param \\*\\//;", + " // /** @type {string} */", + ` /** @type {string} ${OVERLAY_MARK} */ const u = s + t;`, + " return a / 2 /** @returns {number} */;", + "}", + ].join("\n"); + const out = neutralizeDocTags(parseSourceFile("overlay.js", source, "js")); + expect(out.length).toBe(source.length); + expect(out.split("\n")).toEqual([ + "/** -param {number} a */", + "function f(a) {", + ' const s = "/** @type {number} */";', + " const t = `/** @returns {string} ${a} */`;", + " const r = /\\/\\*\\* @param \\*\\//;", + " // /** @type {string} */", + ` /** @type {string} ${OVERLAY_MARK} */ const u = s + t;`, + " return a / 2 /** -returns {number} */;", + "}", + ]); +}); diff --git a/packages/compiler/src/frontend/shape-inference/overlay.ts b/packages/compiler/src/frontend/shape-inference/overlay.ts index 28b5336cf..7747fac20 100644 --- a/packages/compiler/src/frontend/shape-inference/overlay.ts +++ b/packages/compiler/src/frontend/shape-inference/overlay.ts @@ -268,6 +268,10 @@ export function planShapeOverlay( return name; }; + // Functions called with `new` yield their instance, whatever they return. + const constructed = new Set( + [...analysis.fnOfInstance.values()].map((a) => a.node), + ); // Return values merged over every analysis context of each function. const returns = new Map>(); for (const info of analysis.functions.values()) { @@ -382,7 +386,7 @@ export function planShapeOverlay( const paramTags = (fn: ts.FunctionLikeDeclaration): string | null => { const tags: string[] = []; - if (!ts.isConstructorDeclaration(fn)) { + if (!ts.isConstructorDeclaration(fn) && !constructed.has(fn)) { const ret = returnValues(fn); const t = ret !== undefined && ret.size > 0 ? typeOf(ret, false, 0) : null; const why = process.env["SCRIPTC_DEBUG_SHAPE_OVERLAY_WHY"]; @@ -498,9 +502,13 @@ export function planShapeOverlay( // Dead branches: outermost first; edits inside a replaced range drop. const dead = (deadByFile.get(sf) ?? []).sort((a, b) => a.pos - b.pos); const ranges: [number, number][] = []; + const hoisted = new Map(); for (const branch of dead) { const start = ts.tokenStartOfNode(branch); if (ranges.some(([lo, hi]) => start >= lo && start < hi)) continue; + const names = hoistedNames(branch); + if (names === null) continue; + hoisted.set(start, names); ranges.push([start, branch.end]); } if (ranges.length > 0) { @@ -515,7 +523,14 @@ export function planShapeOverlay( `scriptc internal error: shape inference proved the branch at ${rel}:${line} unreachable, but it ran. ` + `This is a bug in scriptc's shape analysis, not in the program. Rebuild with --no-shape-overlay, ` + `or set "scriptc": {"packages": {"${pkg}": {"shapes": false}}} in package.json.`; - edits.push({ pos: lo, end: hi, text: `{ throw new Error(${JSON.stringify(message)}); }` }); + // The branch's own var names stay declared in its scope, undefined. + const names = hoisted.get(lo)!; + const vars = names.length > 0 ? `var ${names.join(", ")}; ` : ""; + edits.push({ + pos: lo, + end: hi, + text: `{ ${vars}throw new Error(${JSON.stringify(message)}); }`, + }); deadBranches++; } } @@ -533,7 +548,7 @@ export function planShapeOverlay( edits.push(...kept); } if (edits.length === 0) continue; - files.set(sf.fileName, neutralizeDocTags(applyEdits(sf.text, edits)) + typedefText.join("")); + files.set(sf.fileName, applyEdits(neutralizeDocTags(sf), edits) + typedefText.join("")); } return { files, annotations, deadBranches }; } @@ -568,17 +583,158 @@ function applyEdits(text: string, edits: readonly Edit[]): string { return out + text.slice(at); } -/** The package's JSDoc type tags become plain text: package JSDoc is a - * hint, not a declaration. The overlay's own comments carry MARK and are - * kept. Lengths are unchanged. */ -export function neutralizeDocTags(text: string): string { - return text.replace(/\/\*\*[\s\S]*?\*\//g, (comment) => { - if (comment.includes(MARK)) return comment; - return comment.replace( - /@(param|type|typedef|returns?|template|this|callback|property|prop|arg|argument|enum|implements|extends|augments|satisfies|overload|import)\b/g, - "-$1", - ); +/** The file's text with the JSDoc type tags of its comments made plain + * text: package JSDoc is a hint, not a declaration. String, template and + * regular expression literals and JSX text are copied unchanged, so + * comment-like text in them keeps its runtime value. Lengths are unchanged, + * so edit positions in the source text stay valid. */ +export function neutralizeDocTags(sf: ts.SourceFile): string { + const text = sf.text; + // Literal token ranges, in source order. + const literals: [number, number][] = []; + const visit = (node: ts.Node): void => { + if (ts.isJsxText(node)) { + literals.push([node.pos, node.end]); + return; + } + if ( + ts.isStringLiteral(node) || + ts.isNoSubstitutionTemplateLiteral(node) || + ts.isTemplateHead(node) || + ts.isTemplateMiddle(node) || + ts.isTemplateTail(node) || + ts.isRegularExpressionLiteral(node) + ) { + literals.push([ts.tokenStartOfNode(node), node.end]); + return; + } + ts.forEachChild(node, (c) => { + visit(c); + return undefined; + }); + }; + visit(sf); + literals.sort((a, b) => a[0] - b[0]); + let out = ""; + let copied = 0; + let next = 0; + let pos = 0; + // Outside literals, `//` and `/*` only start comments. + while (pos < text.length) { + while (next < literals.length && literals[next]![1] <= pos) next++; + const literal = literals[next]; + if (literal !== undefined && literal[0] <= pos) { + pos = literal[1]; + continue; + } + if (text.charCodeAt(pos) !== 47) { + pos++; + continue; + } + const second = text.charCodeAt(pos + 1); + if (second === 47) { + while (pos < text.length && !lineBreak(text.charCodeAt(pos))) pos++; + continue; + } + if (second !== 42) { + pos++; + continue; + } + const close = text.indexOf("*/", pos + 2); + const end = close < 0 ? text.length : close + 2; + const comment = text.slice(pos, end); + if (comment.startsWith("/**") && !comment.includes(MARK)) { + out += text.slice(copied, pos) + neutralizeComment(comment); + copied = end; + } + pos = end; + } + return out + text.slice(copied); +} + +function lineBreak(code: number): boolean { + return code === 10 || code === 13 || code === 0x2028 || code === 0x2029; +} + +function neutralizeComment(comment: string): string { + return comment.replace( + /@(param|type|typedef|returns?|template|this|callback|property|prop|arg|argument|enum|implements|extends|augments|satisfies|overload|import)\b/g, + "-$1", + ); +} + +/** The `var` names a dead branch declares that its function (or file) + * declares nowhere else: replacing the branch must keep them bound, as + * undefined. Null when the branch declares a function in a block, whose + * binding outside the block depends on the file's strictness; such a + * branch is kept. */ +function hoistedNames(branch: ts.Node): readonly string[] | null { + let scope: ts.Node | undefined = branch.parent; + while (scope !== undefined && !ts.isSourceFile(scope) && !functionScope(scope)) + scope = scope.parent; + if (scope === undefined) return null; + const own = new Set(); + let blockFunction = false; + ts.walkPreorder(branch, (n) => { + if (n !== branch && ts.isFunctionDeclaration(n)) { + blockFunction = true; + return "skip"; + } + if (n !== branch && (functionScope(n) || ts.isClassDeclaration(n) || ts.isClassExpression(n))) + return "skip"; + if (ts.isVariableDeclarationList(n) && (n.flags & ts.NodeFlags.BlockScoped) === 0) + for (const d of n.declarations.nodes) addBindingNames(d.name, own); + return undefined; }); + if (blockFunction) return null; + if (own.size === 0) return []; + // Names the scope binds outside the branch: parameters, a function + // expression's name, other var declarations and function declarations. + const elsewhere = new Set(); + if (!ts.isSourceFile(scope)) { + for (const p of list( + (scope as { parameters?: { readonly nodes: readonly ts.ParameterDeclaration[] } }).parameters, + )) + addBindingNames(p.name, elsewhere); + const name = (scope as { name?: ts.Node }).name; + if (name !== undefined && ts.isIdentifier(name)) elsewhere.add(name.text); + } + const body = ts.isSourceFile(scope) ? scope : (scope as { body?: ts.Node }).body; + if (body !== undefined) + ts.walkPreorder(body, (n) => { + if (n === branch) return "skip"; + if (n !== body && ts.isFunctionDeclaration(n) && n.name !== undefined) + elsewhere.add(n.name.text); + if (n !== body && (functionScope(n) || ts.isClassDeclaration(n) || ts.isClassExpression(n))) + return "skip"; + if (ts.isVariableDeclarationList(n) && (n.flags & ts.NodeFlags.BlockScoped) === 0) + for (const d of n.declarations.nodes) addBindingNames(d.name, elsewhere); + return undefined; + }); + return [...own].filter((name) => !elsewhere.has(name)); +} + +function functionScope(n: ts.Node): boolean { + return ( + ts.isFunctionDeclaration(n) || + ts.isFunctionExpression(n) || + ts.isArrowFunction(n) || + ts.isMethodDeclaration(n) || + ts.isGetAccessorDeclaration(n) || + ts.isSetAccessorDeclaration(n) || + ts.isConstructorDeclaration(n) || + ts.isClassStaticBlockDeclaration(n) + ); +} + +function addBindingNames(name: ts.BindingName, out: Set): void { + if (ts.isIdentifier(name)) { + out.add(name.text); + return; + } + for (const element of name.elements.nodes) + if (!ts.isOmittedExpression(element) && element.name !== undefined) + addBindingNames(element.name, out); } /** A record site's keys: its static keys, or for a literal with spreads its diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/shape-overlay-facts/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/shape-overlay-facts/main.ts.json new file mode 100644 index 000000000..f01de4ae5 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/shape-overlay-facts/main.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/fixtures/npm/cases/shape-overlay-facts/main.ts" + ], + "diags": [] +} diff --git a/tests/fixtures/npm/cases/shape-overlay-facts/main.ts b/tests/fixtures/npm/cases/shape-overlay-facts/main.ts new file mode 100644 index 000000000..3ed1cf5de --- /dev/null +++ b/tests/fixtures/npm/cases/shape-overlay-facts/main.ts @@ -0,0 +1,10 @@ +// The shape overlay's facts: results that include undefined, literals that +// hold comment-like text, and a var declared in a branch no call takes. +import { banner, hoisted, lookup, lookupOrReturn } from "shape-facts"; + +console.log(lookup("b")); +console.log(lookup("z")); +console.log(lookupOrReturn("a")); +console.log(lookupOrReturn("y")); +console.log(banner(7)); +console.log(hoisted("note:")); diff --git a/tests/fixtures/npm/cases/shape-overlay-facts/node_modules/shape-facts/index.d.ts b/tests/fixtures/npm/cases/shape-overlay-facts/node_modules/shape-facts/index.d.ts new file mode 100644 index 000000000..b1865415d --- /dev/null +++ b/tests/fixtures/npm/cases/shape-overlay-facts/node_modules/shape-facts/index.d.ts @@ -0,0 +1,4 @@ +export declare function lookup(key: string): string; +export declare function lookupOrReturn(key: string): string; +export declare function banner(n: number): string; +export declare function hoisted(label: string): string; diff --git a/tests/fixtures/npm/cases/shape-overlay-facts/node_modules/shape-facts/index.js b/tests/fixtures/npm/cases/shape-overlay-facts/node_modules/shape-facts/index.js new file mode 100644 index 000000000..505af75a0 --- /dev/null +++ b/tests/fixtures/npm/cases/shape-overlay-facts/node_modules/shape-facts/index.js @@ -0,0 +1,55 @@ +// Functions whose results include the undefined of a fall-through or a +// bare return, comment-like text in literals, and a branch no call takes +// that declares a var read outside it. + +const entries = [ + { key: "a", value: 1 }, + { key: "b", value: 2 }, +]; + +function find(key) { + for (const entry of entries) { + if (entry.key === key) return entry; + } +} + +function findOrReturn(key) { + for (const entry of entries) if (entry.key === key) return entry; + return; +} + +export function lookup(key) { + const hit = find(key); + if (hit) { + return "found " + hit.key + "=" + hit.value; + } else { + return "missing " + key; + } +} + +export function lookupOrReturn(key) { + const hit = findOrReturn(key); + if (hit) { + return "found " + hit.key + "=" + hit.value; + } else { + return "missing " + key; + } +} + +export function banner(n) { + const quoted = "/** @type {number} */"; + const template = `/** @returns {string} ${n} */`; + const pattern = /\/\*\* @param \*\//; + return quoted + " " + template + " " + pattern.source; +} + +function noteOf(extra) { + if (extra) { + var note = "set"; + } + return typeof note + " " + String(note); +} + +export function hoisted(label) { + return label + " " + noteOf(undefined); +} diff --git a/tests/fixtures/npm/cases/shape-overlay-facts/node_modules/shape-facts/package.json b/tests/fixtures/npm/cases/shape-overlay-facts/node_modules/shape-facts/package.json new file mode 100644 index 000000000..317269e91 --- /dev/null +++ b/tests/fixtures/npm/cases/shape-overlay-facts/node_modules/shape-facts/package.json @@ -0,0 +1 @@ +{ "name": "shape-facts", "version": "1.0.0", "type": "module", "main": "index.js", "types": "index.d.ts" } diff --git a/tests/fixtures/npm/cases/shape-overlay-facts/package.json b/tests/fixtures/npm/cases/shape-overlay-facts/package.json new file mode 100644 index 000000000..d15818a37 --- /dev/null +++ b/tests/fixtures/npm/cases/shape-overlay-facts/package.json @@ -0,0 +1 @@ +{ "name": "shape-overlay-facts-case", "private": true, "type": "module" } diff --git a/tests/harness/shape-overlay.test.ts b/tests/harness/shape-overlay.test.ts index ad1988238..0eddc391d 100644 --- a/tests/harness/shape-overlay.test.ts +++ b/tests/harness/shape-overlay.test.ts @@ -60,3 +60,33 @@ test("shapeOverlay: false compiles without the overlay", { timeout: 900_000 }, a throw new Error(result.diagnostics.map((d) => `${d.code}: ${d.message}`).join("\n")); expect((result.warnings ?? []).some((w) => w.code === "SC6103")).toBe(false); }); + +// shape-facts: results that include the undefined of a fall-through or a bare +// return keep the caller's falsy branch, comment-like text in literals keeps +// its value, and a replaced branch keeps its var declared. +const factsEntry = join(repoRoot, "tests/fixtures/npm/cases/shape-overlay-facts/main.ts"); + +test( + "the overlay keeps undefined results, literal text and hoisted vars", + { timeout: 900_000 }, + async () => { + mkdirSync(outDir, { recursive: true }); + const result = await withOverlay(() => + compile(factsEntry, { outPath: join(outDir, "facts"), outDir, sanitize }), + ); + if (!result.ok) + throw new Error(result.diagnostics.map((d) => `${d.code}: ${d.message}`).join("\n")); + const report = (result.warnings ?? []).find((w) => w.code === "SC6103"); + expect(report?.message).toMatch(/of 1 package \(shape-facts\)/); + expect(report?.message).toMatch(/replaces 1 branch it proved unreachable/); + const native = await execFileAsync(result.binaryPath, [], { + cwd: dirname(factsEntry), + encoding: "utf8", + }); + const node = await execFileAsync(process.execPath, [factsEntry], { + cwd: dirname(factsEntry), + encoding: "utf8", + }); + expect(native.stdout).toBe(node.stdout); + }, +); From 468ef7692b0af311dd9d4f3363757f40c83e3c04 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Sat, 24 Oct 2026 23:35:56 -0500 Subject: [PATCH 03/23] Drop the refused never-promise instanceof case from the maybe-promise fences instanceof Promise on values that are never promises now compiles to false after the operand is evaluated, so the diagnostics fixture no longer lists it as refused. 8326-never-promise-instanceof covers records, arrays, dates, collections, regular expressions, byte arrays and unions with null, and a call operand that still runs. --- .../8326-never-promise-instanceof.ts.json | 6 +++++ tests/corpus/8326-never-promise-instanceof.ts | 26 +++++++++++++++++++ tests/diagnostics/maybe-promise-fences.ts | 4 --- .../__snapshots__/maybe-promise-fences.ts.txt | 9 +------ .../8326-never-promise-instanceof.ts.json | 1 + 5 files changed, 34 insertions(+), 12 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8326-never-promise-instanceof.ts.json create mode 100644 tests/corpus/8326-never-promise-instanceof.ts create mode 100644 tests/harness/corpus-costs/8326-never-promise-instanceof.ts.json diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8326-never-promise-instanceof.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8326-never-promise-instanceof.ts.json new file mode 100644 index 000000000..d10bc2c18 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8326-never-promise-instanceof.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/8326-never-promise-instanceof.ts" + ], + "diags": [] +} diff --git a/tests/corpus/8326-never-promise-instanceof.ts b/tests/corpus/8326-never-promise-instanceof.ts new file mode 100644 index 000000000..6e01e3510 --- /dev/null +++ b/tests/corpus/8326-never-promise-instanceof.ts @@ -0,0 +1,26 @@ +// `instanceof Promise` on values that are never promises (records, arrays, +// dates, collections, regular expressions, byte arrays, null, and unions of +// them) is false, after the operand is evaluated. +let evaluated = 0; +function next(): { v: number } { + evaluated++; + return { v: evaluated }; +} + +const record = { v: 1 }; +console.log("record", record instanceof Promise); +console.log("call", next() instanceof Promise, evaluated); +console.log("array", [1, 2] instanceof Promise); +console.log("date", new Date(0) instanceof Promise); +console.log("map", new Map() instanceof Promise); +console.log("set", new Set() instanceof Promise); +console.log("regex", /a/ instanceof Promise); +console.log("bytes", new Uint8Array(2) instanceof Promise); + +function pick(i: number): { v: number } | number[] | null { + return i === 0 ? { v: 0 } : i === 1 ? [i] : null; +} +for (let i = 0; i < 3; i++) console.log("union", i, pick(i) instanceof Promise); + +const promised: unknown = Promise.resolve(1); +console.log("promise", promised instanceof Promise); diff --git a/tests/diagnostics/maybe-promise-fences.ts b/tests/diagnostics/maybe-promise-fences.ts index 1dcccaf08..e6fdf98fa 100644 --- a/tests/diagnostics/maybe-promise-fences.ts +++ b/tests/diagnostics/maybe-promise-fences.ts @@ -26,7 +26,3 @@ console.log(maybe(true)); // must be that arm's own promise type. const widened = Promise.resolve(maybe(false)); console.log(typeof widened); - -// `instanceof Promise` on a value that is never a promise. -const record = { v: 1 }; -console.log(record instanceof Promise); diff --git a/tests/harness/__snapshots__/maybe-promise-fences.ts.txt b/tests/harness/__snapshots__/maybe-promise-fences.ts.txt index 159e88f76..ab325aaaa 100644 --- a/tests/harness/__snapshots__/maybe-promise-fences.ts.txt +++ b/tests/harness/__snapshots__/maybe-promise-fences.ts.txt @@ -48,11 +48,4 @@ maybe-promise-fences.ts:27:50 - error SC2020: 'Promise.resolve over 'Promise Date: Sun, 25 Oct 2026 21:55:14 -0500 Subject: [PATCH 04/23] Store identical runtime artifacts once and strip the copied CLI executables Runtime packs: variants of a unit that do not read their defines, and flavors that compile a unit the same way, produce identical objects and bitcode. The pack build now points those manifest entries at one file (shareIdenticalArtifacts) and deletes the copies. Each link selects one flavor and one variant per unit, so no link names a shared file twice. The linux-x64-gnu pack keeps 1,191 of its 1,448 artifact files. Native CLI packages: - The comptime evaluator builds with -g0 and the executable strip flags, and the Windows build no longer ships its .pdb. - The copied tsc-ts executable is stripped with llvm-strip, which rewrites the linker-signed code signature of Mach-O executables. - On arm64 macOS, verify-native-cli.mjs --run checks the code signatures of the compiler, tsc-ts and the comptime evaluator. - SCRIPTC_NATIVE_KEEP_SYMBOLS=1 keeps the symbols of all three. - The copied runtime sources and declarations get LF line endings. A Windows checkout converts them to CRLF, so the Windows CLI shipped files that differed from @scriptc/runtime's. --- .../runtime-darwin-arm64/scripts/build.mjs | 2 + .../runtime-pack-common/scripts/build.mjs | 2 + .../scripts/share-artifacts.mjs | 48 +++++++++++ .../test/share-artifacts.test.ts | 79 +++++++++++++++++++ scripts/build-native-cli.mts | 14 +++- scripts/native-cli-assets.mts | 33 ++++++++ scripts/verify-native-cli.mjs | 7 ++ 7 files changed, 183 insertions(+), 2 deletions(-) create mode 100644 packages/runtime-pack-common/scripts/share-artifacts.mjs create mode 100644 packages/runtime-pack-common/test/share-artifacts.test.ts diff --git a/packages/runtime-darwin-arm64/scripts/build.mjs b/packages/runtime-darwin-arm64/scripts/build.mjs index 6006e73ac..92f6a432a 100644 --- a/packages/runtime-darwin-arm64/scripts/build.mjs +++ b/packages/runtime-darwin-arm64/scripts/build.mjs @@ -15,6 +15,7 @@ import { } from "../../runtime-pack-common/scripts/runtime-unit.mjs"; import { createDeterministicArchive } from "./archive.mjs"; import { installRuntimePack, withBuildLock } from "./build-state.mjs"; +import { shareIdenticalArtifacts } from "../../runtime-pack-common/scripts/share-artifacts.mjs"; const run = promisify(execFile); const packageRoot = dirname(dirname(fileURLToPath(import.meta.url))); @@ -365,6 +366,7 @@ async function build() { await copyFile(source, output); }), ); + await shareIdenticalArtifacts(flavors, buildRoot); const manifest = { schema: "scriptc.runtime-pack.v1", format: 1, diff --git a/packages/runtime-pack-common/scripts/build.mjs b/packages/runtime-pack-common/scripts/build.mjs index 357ba7337..8f2eb3bde 100644 --- a/packages/runtime-pack-common/scripts/build.mjs +++ b/packages/runtime-pack-common/scripts/build.mjs @@ -15,6 +15,7 @@ import { import { createDeterministicArchive } from "./archive.mjs"; import { buildImportBitcode, buildRuntimeUnit, resolveRuntimeUnitHelper } from "./runtime-unit.mjs"; import { installRuntimePack, withBuildLock } from "./build-state.mjs"; +import { shareIdenticalArtifacts } from "./share-artifacts.mjs"; import { parallelMap } from "./parallel-map.mjs"; const run = promisify(execFile); @@ -468,6 +469,7 @@ async function build() { await copyFile(source, output); }), ); + await shareIdenticalArtifacts(flavors, buildRoot); const manifest = { schema: "scriptc.runtime-pack.v1", format: 1, diff --git a/packages/runtime-pack-common/scripts/share-artifacts.mjs b/packages/runtime-pack-common/scripts/share-artifacts.mjs new file mode 100644 index 000000000..8eaf0707c --- /dev/null +++ b/packages/runtime-pack-common/scripts/share-artifacts.mjs @@ -0,0 +1,48 @@ +import { readdir, rm, rmdir } from "node:fs/promises"; +import { join } from "node:path"; + +/** Points the artifacts of one runtime unit that hold the same bytes at one + * file and deletes the copies. Variants whose defines a unit does not read, + * and flavors that compile a unit the same way, produce identical objects + * and bitcode. Each link selects one flavor and one variant of each unit, so + * no link names a shared file twice. `root` is the directory artifact paths + * are relative to; flavors keep the order they were built in, so the first + * flavor's path holds the shared file. */ +export async function shareIdenticalArtifacts(flavors, root) { + const firstPaths = new Map(); + for (const flavor of Object.values(flavors)) { + for (const unit of flavor.runtime_units) { + let paths = firstPaths.get(unit.source); + if (paths === undefined) { + paths = new Map(); + firstPaths.set(unit.source, paths); + } + for (const variant of unit.variants) { + for (const artifact of variant.bitcode === undefined + ? [variant] + : [variant, variant.bitcode]) { + const first = paths.get(artifact.sha256); + if (first === undefined) { + paths.set(artifact.sha256, artifact.path); + } else if (first !== artifact.path) { + await rm(join(root, artifact.path)); + artifact.path = first; + } + } + } + } + } + await removeEmptyDirectories(join(root, "artifacts")); +} + +async function removeEmptyDirectories(directory) { + const entries = await readdir(directory, { withFileTypes: true }); + let remaining = entries.length; + for (const entry of entries) { + if (entry.isDirectory() && (await removeEmptyDirectories(join(directory, entry.name)))) + remaining--; + } + if (remaining > 0) return false; + await rmdir(directory); + return true; +} diff --git a/packages/runtime-pack-common/test/share-artifacts.test.ts b/packages/runtime-pack-common/test/share-artifacts.test.ts new file mode 100644 index 000000000..a2cb68128 --- /dev/null +++ b/packages/runtime-pack-common/test/share-artifacts.test.ts @@ -0,0 +1,79 @@ +import { existsSync } from "node:fs"; +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { expect, test } from "vitest"; +import { shareIdenticalArtifacts } from "../scripts/share-artifacts.mjs"; + +type Artifact = { path: string; sha256: string; size: number; bitcode?: Artifact }; + +async function artifact(root: string, path: string, sha256: string): Promise { + await mkdir(dirname(join(root, path)), { recursive: true }); + await writeFile(join(root, path), sha256); + return { path, sha256, size: sha256.length }; +} + +test("identical artifacts of one unit share the first flavor's file", async () => { + const root = await mkdtemp(join(tmpdir(), "scriptc-share-artifacts-")); + try { + const unit = async (flavor: string, source: string, ids: Record) => ({ + source, + variants: await Promise.all( + Object.entries(ids).map(async ([id, sha]) => ({ + id, + ...(await artifact(root, `artifacts/${flavor}/runtime/${id}/${source}.o`, sha)), + })), + ), + }); + const release = { + runtime_units: [ + await unit("release", "scr_a", { default: "a1", "default-workers": "a1" }), + await unit("release", "scr_b", { default: "b1" }), + ], + }; + release.runtime_units[1]!.variants[0]!.bitcode = await artifact( + root, + "artifacts/release/runtime/default/scr_b.bc", + "b-bitcode", + ); + const speed = { + runtime_units: [ + await unit("speed", "scr_a", { default: "a2" }), + // The same bytes as another unit's object stay a separate file. + await unit("speed", "scr_b", { default: "b1", other: "a1" }), + ], + }; + speed.runtime_units[1]!.variants[0]!.bitcode = await artifact( + root, + "artifacts/speed/runtime/default/scr_b.bc", + "b-bitcode", + ); + const flavors = { release, speed }; + await shareIdenticalArtifacts(flavors, root); + const paths = (flavor: typeof release) => + flavor.runtime_units.map((unit) => + unit.variants.map((variant: Artifact) => [variant.path, variant.bitcode?.path]), + ); + expect(paths(release)).toEqual([ + [ + ["artifacts/release/runtime/default/scr_a.o", undefined], + ["artifacts/release/runtime/default/scr_a.o", undefined], + ], + [["artifacts/release/runtime/default/scr_b.o", "artifacts/release/runtime/default/scr_b.bc"]], + ]); + expect(paths(speed)).toEqual([ + [["artifacts/speed/runtime/default/scr_a.o", undefined]], + [ + ["artifacts/release/runtime/default/scr_b.o", "artifacts/release/runtime/default/scr_b.bc"], + ["artifacts/speed/runtime/other/scr_b.o", undefined], + ], + ]); + expect(existsSync(join(root, "artifacts/release/runtime/default-workers"))).toBe(false); + expect(existsSync(join(root, "artifacts/speed/runtime/default/scr_b.o"))).toBe(false); + expect(existsSync(join(root, "artifacts/speed/runtime/default/scr_b.bc"))).toBe(false); + expect(existsSync(join(root, "artifacts/speed/runtime/default/scr_a.o"))).toBe(true); + expect(existsSync(join(root, "artifacts/speed/runtime/other/scr_b.o"))).toBe(true); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); diff --git a/scripts/build-native-cli.mts b/scripts/build-native-cli.mts index 05f36cf68..c16e72777 100644 --- a/scripts/build-native-cli.mts +++ b/scripts/build-native-cli.mts @@ -6,7 +6,11 @@ import { basename, dirname, join, relative, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { gunzipSync } from "node:zlib"; import { compile } from "../packages/compiler/src/index.js"; -import { NATIVE_TARGETS, nativeCodegenTarget } from "../packages/compiler/src/backend/targets.js"; +import { + NATIVE_TARGETS, + executableStripLinkerArgs, + nativeCodegenTarget, +} from "../packages/compiler/src/backend/targets.js"; import { tsgoExecutable } from "../packages/compiler/src/frontend/ts7/rpc-api.js"; import { prebuiltLibDiagnosticsText } from "../packages/compiler/src/frontend/ts7/lib-diagnostics.js"; import { stageNativeCliAssets } from "./native-cli-assets.mjs"; @@ -94,12 +98,14 @@ const ffi = join(seed, "compiler.ffi.json"); writeFileSync(ffi, JSON.stringify({ ffi_format: 6, functions, libraries }, null, 2) + "\n"); const executable = join(bin, "scriptc" + target.outputSuffixes.exe); const comptime = join(lib, "scriptc-comptime" + target.outputSuffixes.exe); +const strip = process.env["SCRIPTC_NATIVE_KEEP_SYMBOLS"] !== "1"; execFileSync( compiler, [ ...compilerArgs, ...cFlags, "-O2", + ...(strip ? ["-g0", ...executableStripLinkerArgs(target.platform, true)] : []), "-I", join(root, "packages/runtime/vendor/quickjs-ng"), join(nativeSources, "comptime.c"), @@ -116,12 +122,15 @@ execFileSync( ], { stdio: "inherit" }, ); +// The Windows link writes the debug database beside the executable. +if (strip && target.platform === "win32") + rmSync(comptime.replace(/\.exe$/, ".pdb"), { force: true }); const result = await compile(join(root, "packages/compiler/src/native/cli.ts"), { outDir: seed, outPath: executable, backend: "llvm", optimization, - strip: process.env["SCRIPTC_NATIVE_KEEP_SYMBOLS"] !== "1", + strip, ffiProfilePath: ffi, sanitize, ...(profileGenerate === undefined ? {} : { profileGenerate }), @@ -167,6 +176,7 @@ const assets = stageNativeCliAssets({ target, ts7: tsgoExecutable(), compilerVersion, + strip, }); // A cross-libc seed may use Zig on the build host. The installed compiler // selects the destination platform's ordinary linker and SDK. diff --git a/scripts/native-cli-assets.mts b/scripts/native-cli-assets.mts index bdecc1da2..8162c9091 100644 --- a/scripts/native-cli-assets.mts +++ b/scripts/native-cli-assets.mts @@ -1,3 +1,4 @@ +import { execFileSync } from "node:child_process"; import { cpSync, existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from "node:fs"; import { basename, dirname, join, relative } from "node:path"; import type { NativeTargetSpec } from "../packages/compiler/src/backend/targets.js"; @@ -23,12 +24,41 @@ function copyPackage(source: string, destination: string, members: readonly stri } } +/** Removes the symbol table and debug sections of a copied executable. + * llvm-strip rewrites the linker-signed ad hoc code signature of a Mach-O + * executable, which arm64 macOS requires to run it. Windows executables keep + * their debug information in separate .pdb files, which are not copied. */ +function stripExecutable(path: string, platform: string): void { + if (platform === "win32") return; + execFileSync("llvm-strip", [path], { stdio: "inherit" }); +} + +/** Rewrites CRLF line endings to LF in the text files under `directory`. A + * Windows checkout converts the repository's LF text files to CRLF; this + * restores the files the other hosts ship. Text is what Git's conversion + * treats as text: no NUL byte in the first 8000 bytes. */ +function normalizeLineEndings(directory: string): void { + for (const entry of readdirSync(directory, { withFileTypes: true })) { + const path = join(directory, entry.name); + if (entry.isDirectory()) { + normalizeLineEndings(path); + continue; + } + if (!entry.isFile()) continue; + const bytes = readFileSync(path); + if (bytes.subarray(0, 8000).includes(0) || !bytes.includes("\r\n")) continue; + writeFileSync(path, bytes.toString("latin1").replaceAll("\r\n", "\n"), "latin1"); + } +} + export function stageNativeCliAssets(options: { repository: string; output: string; target: NativeTargetSpec; ts7: string; compilerVersion: string; + /** Strip the copied tsc-ts executable. */ + strip: boolean; }): Pick< NativeToolchainManifest, | "ts7" @@ -67,8 +97,10 @@ export function stageNativeCliAssets(options: { ); const tscTs = join(lib, "tsc-ts"); copyPackage(dirname(dirname(tscTsSource)), tscTs, ["lib"]); + if (options.strip) stripExecutable(join(tscTs, "lib", basename(tscTsSource)), target.platform); const sources = join(lib, "runtime-sources"); copyPackage(join(repository, "packages/runtime"), sources, ["src", "vendor"]); + normalizeLineEndings(sources); const declarations = join(lib, "declarations"); mkdirSync(declarations, { recursive: true }); for (const name of ["scriptc.d.ts", "scriptc-overrides.d.ts", "scriptc-node-fallback.d.ts"]) { @@ -78,6 +110,7 @@ export function stageNativeCliAssets(options: { cpSync(join(repository, "packages/compiler/shims"), join(declarations, "shims"), { recursive: true, }); + normalizeLineEndings(declarations); // This optional Node host runs the built WASI module, never the compiler. const wasi = join(lib, "wasi"); mkdirSync(join(wasi, "cli"), { recursive: true }); diff --git a/scripts/verify-native-cli.mjs b/scripts/verify-native-cli.mjs index 310cc04a9..6a18f9e70 100644 --- a/scripts/verify-native-cli.mjs +++ b/scripts/verify-native-cli.mjs @@ -111,5 +111,12 @@ if (process.argv.includes("--run")) { }).trim(); if (!tscTsVersion.startsWith("Version ")) throw new Error(`the distribution's tsc-ts reported ${JSON.stringify(tscTsVersion)}`); + // The release strips tsc-ts and the comptime evaluator; arm64 macOS runs + // only executables whose linker-generated code signature matches their + // contents. + if (process.platform === "darwin" && process.arch === "arm64") { + for (const executable of [binary, tscTs, resolve(dirname(binary), toolchain.comptime)]) + execFileSync("codesign", ["--verify", "--strict", executable], { stdio: "inherit" }); + } } console.log(`${identity.name}@${identity.version}: native distribution verified`); From ca3217fe9915d98e168aac8cf19033a27dd4208a Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Mon, 26 Oct 2026 02:48:03 -0500 Subject: [PATCH 05/23] Reuse the completion callback a stream passes to its option callbacks Each call of a typed or untyped write, writev, transform, flush, final, construct or destroy option callback allocated a closure and a box holding the stream for its `callback` argument. The emitted thunks now take it from scr_stream_done_closure, which keeps the last one on the stream state and hands it out again while the stream holds its only reference and it has no own properties. A callback the user kept (stored, or passed to setImmediate or a promise) is not reused: the next call gets a new closure, and the kept one keeps its own stream reference. The cached closure holds the stream, so the stream drops it when it emits 'close' and traces it, which lets the cycle collector reclaim a stream dropped before it closes. RUNTIME_ABI_VERSION goes from 46 to 47 for scr_stream_done_closure. --- packages/cli/test/native-link-info.test.ts | 4 +- .../src/backend/llvm/expr-stream-callbacks.ts | 19 +---- packages/compiler/src/backend/runtime-abi.ts | 4 +- .../src/backend/runtime-pack-targets.test.ts | 2 +- .../compiler/src/backend/runtime-pack.test.ts | 2 +- .../test/native-codegen-integration.test.ts | 4 +- .../corpus/7517-stream-kept-callbacks.ts.json | 6 ++ .../7518-stream-dropped-callbacks.ts.json | 6 ++ .../runtime-darwin-arm64/scripts/build.mjs | 2 +- .../runtime-pack-common/scripts/build.mjs | 2 +- packages/runtime/src/scr_console.c | 2 +- packages/runtime/src/scr_core.h | 2 +- packages/runtime/src/scr_stream.c | 30 +++++++ packages/runtime/src/scr_stream.h | 3 + tests/corpus/7517-stream-kept-callbacks.ts | 79 +++++++++++++++++++ tests/corpus/7518-stream-dropped-callbacks.ts | 54 +++++++++++++ .../7517-stream-kept-callbacks.ts.json | 1 + .../7518-stream-dropped-callbacks.ts.json | 1 + 18 files changed, 196 insertions(+), 27 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7517-stream-kept-callbacks.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7518-stream-dropped-callbacks.ts.json create mode 100644 tests/corpus/7517-stream-kept-callbacks.ts create mode 100644 tests/corpus/7518-stream-dropped-callbacks.ts create mode 100644 tests/harness/corpus-costs/7517-stream-kept-callbacks.ts.json create mode 100644 tests/harness/corpus-costs/7518-stream-dropped-callbacks.ts.json diff --git a/packages/cli/test/native-link-info.test.ts b/packages/cli/test/native-link-info.test.ts index 24db2c4c2..b838b3155 100644 --- a/packages/cli/test/native-link-info.test.ts +++ b/packages/cli/test/native-link-info.test.ts @@ -72,7 +72,7 @@ describe.runIf(supported)("macOS arm64 native link info", () => { minimum_os: "14.0", }, program: { object, entry_symbol: "main" }, - runtime_abi: { version: 46, marker: "scr_runtime_abi_v46" }, + runtime_abi: { version: 47, marker: "scr_runtime_abi_v47" }, runtime_pack: { kind: "precompiled", package: "@scriptc/runtime-darwin-arm64", @@ -165,6 +165,6 @@ describe.runIf(supported)("macOS arm64 native link info", () => { () => null, (failure: { stderr?: string }) => failure, ); - expect(error?.stderr).toContain("scr_runtime_abi_v46"); + expect(error?.stderr).toContain("scr_runtime_abi_v47"); }); }); diff --git a/packages/compiler/src/backend/llvm/expr-stream-callbacks.ts b/packages/compiler/src/backend/llvm/expr-stream-callbacks.ts index 285f9ca9a..7c5cba75b 100644 --- a/packages/compiler/src/backend/llvm/expr-stream-callbacks.ts +++ b/packages/compiler/src/backend/llvm/expr-stream-callbacks.ts @@ -684,22 +684,11 @@ export function streamCbThunkFor( passed.push(`ptr %sr`); } // The stream-owning completion closure (a 1-cap closure boxing the - // retained stream) — shared by the typed and dyn done shapes. + // retained stream) — shared by the typed and dyn done shapes. The + // stream reuses it while it holds the only reference. const doneClosure = (fnRef: string, tag: string): string => { - host.declare(`declare ptr @scr_closure_new(ptr, ${host.sizeType})`); - host.declare(`declare ptr @scr_box_new_obj(ptr, ptr, ptr)`); - host.declare(`declare void @scr_box_set_ref(ptr, ptr)`); - host.declare(`declare ptr @scr_stream_retain_v(ptr)`); - host.declare(`declare void @scr_stream_release_v(ptr)`); - host.declare(`declare void @scr_stream_trace(ptr, ptr, ptr)`); - d.push( - ` %${tag}clo = call ptr @scr_closure_new(ptr ${fnRef}, ${host.sizeType} 1)`, - ` %${tag}bx = call ptr @scr_box_new_obj(ptr @scr_stream_retain_v, ptr @scr_stream_release_v, ptr @scr_stream_trace)`, - ` %${tag}cp = getelementptr inbounds %ScrClosure, ptr %${tag}clo, i64 1`, - ` store ptr %${tag}bx, ptr %${tag}cp`, - ` %${tag}sr = call ptr @scr_stream_retain_v(ptr %s)`, - ` call void @scr_box_set_ref(ptr %${tag}bx, ptr %${tag}sr)`, - ); + host.declare(`declare ptr @scr_stream_done_closure(ptr, ptr)`); + d.push(` %${tag}clo = call ptr @scr_stream_done_closure(ptr %s, ptr ${fnRef})`); return `%${tag}clo`; }; for (let i = off; i < declared.length; i++) { diff --git a/packages/compiler/src/backend/runtime-abi.ts b/packages/compiler/src/backend/runtime-abi.ts index 892e9281e..985ba6203 100644 --- a/packages/compiler/src/backend/runtime-abi.ts +++ b/packages/compiler/src/backend/runtime-abi.ts @@ -2,8 +2,8 @@ * of the link contract: a program object keeps this symbol undefined and the * matching runtime defines it, so incompatible manual links fail before the * program can start. */ -export const RUNTIME_ABI_VERSION = 46 as const; -export const RUNTIME_ABI_MARKER = "scr_runtime_abi_v46" as const; +export const RUNTIME_ABI_VERSION = 47 as const; +export const RUNTIME_ABI_MARKER = "scr_runtime_abi_v47" as const; /** Object consumption is useful today, but the complete scr_* surface may * still change before 1.0. The versioned marker prevents accidental mixing; diff --git a/packages/compiler/src/backend/runtime-pack-targets.test.ts b/packages/compiler/src/backend/runtime-pack-targets.test.ts index 9c5b61f58..fcbe3f08e 100644 --- a/packages/compiler/src/backend/runtime-pack-targets.test.ts +++ b/packages/compiler/src/backend/runtime-pack-targets.test.ts @@ -52,7 +52,7 @@ async function fixture(target: NativeTargetSpec, workers = false) { object_format: target.objectFormat, minimum_os: target.minimumOs, }, - runtime_abi: { version: 46, marker: "scr_runtime_abi_v46" }, + runtime_abi: { version: 47, marker: "scr_runtime_abi_v47" }, compiler: { command: "fixture", identity: "fixture", target: target.llvmTriple }, macros: { executable: [], excluded: ["SCR_LIB"], sanitizer: "external-toolchain-required" }, flavors: { diff --git a/packages/compiler/src/backend/runtime-pack.test.ts b/packages/compiler/src/backend/runtime-pack.test.ts index 812fd9531..b6e3f0ef7 100644 --- a/packages/compiler/src/backend/runtime-pack.test.ts +++ b/packages/compiler/src/backend/runtime-pack.test.ts @@ -129,7 +129,7 @@ async function fixture(target: NativeTargetSpec = MACOS_ARM64_TARGET) { object_format: target.objectFormat, minimum_os: target.minimumOs, }, - runtime_abi: { version: 46, marker: "scr_runtime_abi_v46" }, + runtime_abi: { version: 47, marker: "scr_runtime_abi_v47" }, compiler: { command: "clang", identity: "fixture clang", diff --git a/packages/compiler/test/native-codegen-integration.test.ts b/packages/compiler/test/native-codegen-integration.test.ts index bdad62203..2ac6143b3 100644 --- a/packages/compiler/test/native-codegen-integration.test.ts +++ b/packages/compiler/test/native-codegen-integration.test.ts @@ -378,7 +378,7 @@ done: expect(await symbols(helperObject, ["-u"])).toEqual(await symbols(clangObject, ["-u"])); expect(await symbols(helperObject, ["-gU"])).toEqual(await symbols(clangObject, ["-gU"])); expect(await symbols(helperObject, ["-gU"])).toEqual(["0000000000000000 T _main"]); - expect(await symbols(helperObject, ["-u"])).toContain("_scr_runtime_abi_v46"); + expect(await symbols(helperObject, ["-u"])).toContain("_scr_runtime_abi_v47"); const clangExe = join(dir, "clang-program"); const linkDriver = join(dir, "clang-driver.c"); @@ -712,7 +712,7 @@ done: .split("\n"); expect(undefinedSymbols).toContain("_sf_scale"); expect(undefinedSymbols).toContain("_sf_callback_mix"); - expect(undefinedSymbols).toContain("_scr_runtime_abi_v46"); + expect(undefinedSymbols).toContain("_scr_runtime_abi_v47"); }); test("helper and clang object paths retain the same outbound FFI C ABI", async () => { diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7517-stream-kept-callbacks.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7517-stream-kept-callbacks.ts.json new file mode 100644 index 000000000..b3d1f420b --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7517-stream-kept-callbacks.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7517-stream-kept-callbacks.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7518-stream-dropped-callbacks.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7518-stream-dropped-callbacks.ts.json new file mode 100644 index 000000000..bf393beee --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7518-stream-dropped-callbacks.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7518-stream-dropped-callbacks.ts" + ], + "diags": [] +} diff --git a/packages/runtime-darwin-arm64/scripts/build.mjs b/packages/runtime-darwin-arm64/scripts/build.mjs index 6006e73ac..78cd3add0 100644 --- a/packages/runtime-darwin-arm64/scripts/build.mjs +++ b/packages/runtime-darwin-arm64/scripts/build.mjs @@ -371,7 +371,7 @@ async function build() { package: packageManifest.name, version: packageManifest.version, target: RUNTIME_PACK_MATRIX.target, - runtime_abi: { version: 46, marker: "scr_runtime_abi_v46" }, + runtime_abi: { version: 47, marker: "scr_runtime_abi_v47" }, compiler: { command: compiler, identity: compilerVersion, diff --git a/packages/runtime-pack-common/scripts/build.mjs b/packages/runtime-pack-common/scripts/build.mjs index 357ba7337..542b064d4 100644 --- a/packages/runtime-pack-common/scripts/build.mjs +++ b/packages/runtime-pack-common/scripts/build.mjs @@ -474,7 +474,7 @@ async function build() { package: packageManifest.name, version: packageManifest.version, target: matrix.target, - runtime_abi: { version: 46, marker: "scr_runtime_abi_v46" }, + runtime_abi: { version: 47, marker: "scr_runtime_abi_v47" }, compiler: { command: compiler, identity: compilerVersion, diff --git a/packages/runtime/src/scr_console.c b/packages/runtime/src/scr_console.c index 18c715cb2..f1fefa626 100644 --- a/packages/runtime/src/scr_console.c +++ b/packages/runtime/src/scr_console.c @@ -148,7 +148,7 @@ static void scr_enable_utf8_console_output(void) { } #endif -void scr_runtime_abi_v46(void) {} +void scr_runtime_abi_v47(void) {} void scr_init(void) { #ifdef SCR_WORKERS diff --git a/packages/runtime/src/scr_core.h b/packages/runtime/src/scr_core.h index 92392897a..f3c85e740 100644 --- a/packages/runtime/src/scr_core.h +++ b/packages/runtime/src/scr_core.h @@ -81,7 +81,7 @@ void scr_init(void); /* Program objects emitted by the bundled LLVM helper reference this symbol. * Its versioned spelling makes a mismatched manual runtime link fail before * the program can start. */ -void scr_runtime_abi_v46(void); +void scr_runtime_abi_v47(void); void scr_runtime_workers_v8(void); /* ── the trap funnel (scr_console.c; scr_library.c under -DSCR_LIB) ────── diff --git a/packages/runtime/src/scr_stream.c b/packages/runtime/src/scr_stream.c index fb416e5c3..b45d27528 100644 --- a/packages/runtime/src/scr_stream.c +++ b/packages/runtime/src/scr_stream.c @@ -246,6 +246,12 @@ struct ScrStreamState { ScrClosure *destroy_cb; ScrStreamErrInv destroy_inv; + /* The completion closure (`callback`) the emitted option-callback + * thunks last passed, owned: scr_stream_done_closure hands it out again + * while this is its only reference. It holds the stream, so it is + * dropped at 'close' and traced for a stream dropped before it closes. */ + ScrClosure *done_cache; + /* finished()/pipeline() watchers: fired once at the terminal point * (the 'close' tick — or a registration-time tick when already closed) * with the stream's finish status (NULL / the error / premature close). @@ -417,6 +423,7 @@ static void scr_stream_state_drop(ScrStreamState *st, bool gc) { if (st->transform_cb) scr_closure_release(st->transform_cb); if (st->flush_cb) scr_closure_release(st->flush_cb); if (st->destroy_cb) scr_closure_release(st->destroy_cb); + if (st->done_cache) scr_closure_release(st->done_cache); for (size_t i = 0; i < st->w.end_cbs_n; i++) scr_closure_release(st->w.end_cbs[i]); for (size_t i = 0; i < st->fin.n; i++) scr_closure_release(st->fin.cb[i]); for (size_t i = 0; i < st->pipes.n; i++) scr_stream_release(st->pipes.dst[i]); @@ -465,6 +472,7 @@ void scr_stream_st_trace(ScrStreamState *st, ScrTraceVisit visit, void *ctx) { if (st->transform_cb) visit(st->transform_cb, ctx); if (st->flush_cb) visit(st->flush_cb, ctx); if (st->destroy_cb) visit(st->destroy_cb, ctx); + if (st->done_cache) visit(st->done_cache, ctx); for (size_t i = 0; i < st->w.end_cbs_n; i++) visit(st->w.end_cbs[i], ctx); for (size_t i = 0; i < st->fin.n; i++) visit(st->fin.cb[i], ctx); for (size_t i = st->w.head; i < st->w.n; i++) { @@ -519,6 +527,27 @@ void scr_stream_release(ScrStream *s) { void *scr_stream_retain_v(void *s) { return scr_stream_retain(s); } void scr_stream_release_v(void *s) { scr_stream_release(s); } +/* The completion closure an option callback receives as `callback`: a + * 1-cap closure calling fn over a box that holds the stream. The stream + * keeps the last one and hands it out again while the cache holds its + * only reference and it has no own properties; otherwise a new closure + * replaces it, and one the user kept keeps its own stream reference. + * Nothing is cached after 'close'. Returns +1. */ +ScrClosure *scr_stream_done_closure(ScrStream *s, void *fn) { + ScrStreamState *st = s->st; + ScrClosure *c = st->done_cache; + if (c != NULL && c->rc == 1 && c->fn == fn && c->props == NULL) return scr_closure_retain(c); + c = scr_closure_new(fn, 1); + ScrBox *bx = scr_box_new_obj(scr_stream_retain_v, scr_stream_release_v, scr_stream_trace); + c->caps[0] = bx; + scr_box_set_ref(bx, scr_stream_retain(s)); + if (st->close_emitted) return c; + ScrClosure *old = st->done_cache; + st->done_cache = scr_closure_retain(c); + if (old) scr_closure_release(old); + return c; +} + /* ── the tick queue (the nextTick stand-in) ───────────────────────────── */ typedef enum { @@ -7960,6 +7989,7 @@ static void scr_stream_run_tick(ScrStreamTick *t) { if (st->transform_cb) { scr_closure_release(st->transform_cb); st->transform_cb = NULL; } if (st->flush_cb) { scr_closure_release(st->flush_cb); st->flush_cb = NULL; } if (st->destroy_cb) { scr_closure_release(st->destroy_cb); st->destroy_cb = NULL; } + if (st->done_cache) { scr_closure_release(st->done_cache); st->done_cache = NULL; } } break; } diff --git a/packages/runtime/src/scr_stream.h b/packages/runtime/src/scr_stream.h index c10978298..9b950d445 100644 --- a/packages/runtime/src/scr_stream.h +++ b/packages/runtime/src/scr_stream.h @@ -681,6 +681,9 @@ void scr_stream_release(ScrStream *s); void *scr_stream_retain_v(void *s); void scr_stream_release_v(void *s); void scr_stream_trace(void *obj, ScrTraceVisit visit, void *ctx); +/* The `callback` an option callback receives (+1), reused while the + * stream holds its only reference. */ +ScrClosure *scr_stream_done_closure(ScrStream *s, void *fn); void scr_stream_install(void); /* The loop's stream hook (scr_async.c): `pending` keeps the loop alive diff --git a/tests/corpus/7517-stream-kept-callbacks.ts b/tests/corpus/7517-stream-kept-callbacks.ts new file mode 100644 index 000000000..f721fab31 --- /dev/null +++ b/tests/corpus/7517-stream-kept-callbacks.ts @@ -0,0 +1,79 @@ +// Option callbacks that keep their `callback` argument: a Writable whose +// write() keeps every third callback and calls it later, a write that +// calls its callback and keeps it (`cb(); keep(cb)`) and calls it again +// after later writes (ERR_MULTIPLE_CALLBACK), and a Transform that calls +// back synchronously, from setImmediate, and from a promise. Every write +// and transform completes in Node's order. +import { Transform, Writable } from "node:stream"; + +type Done = (error?: Error | null) => void; + +const kept: Done[] = []; +const log: string[] = []; + +const deferred = new Writable({ + highWaterMark: 4, + write(chunk: Buffer, _encoding: string, callback: Done) { + const text = chunk.toString(); + if (Number(text) % 3 === 0) { + kept.push(callback); + log.push(`kept ${text}`); + setImmediate(() => { + const cb = kept.shift(); + log.push(`late ${text}`); + if (cb) cb(); + }); + return; + } + log.push(`write ${text}`); + callback(); + }, +}); +for (let i = 1; i <= 10; i++) { + deferred.write(String(i), () => log.push(`done ${i}`)); +} +deferred.end(() => { + console.log(log.join("\n")); + twice(); +}); + +function twice(): void { + let saved: Done | null = null; + const w = new Writable({ + write(chunk: Buffer, _encoding: string, callback: Done) { + const text = chunk.toString(); + console.log("twice write", text); + callback(); + if (text === "a") saved = callback; + }, + }); + w.on("error", (error: Error & { code?: string }) => { + console.log("twice error", error.code); + }); + w.write("a"); + w.write("b"); + w.write("c"); + setImmediate(() => { + const cb = saved; + if (cb) cb(); + setImmediate(transforms); + }); +} + +function transforms(): void { + let n = 0; + const t = new Transform({ + transform(chunk: Buffer, _encoding: string, callback: (error?: Error | null, data?: Buffer) => void) { + const k = n++; + const out = Buffer.from(`[${chunk.toString()}]`); + if (k % 3 === 0) callback(null, out); + else if (k % 3 === 1) setImmediate(() => callback(null, out)); + else Promise.resolve().then(() => callback(null, out)); + }, + }); + const parts: string[] = []; + t.on("data", (chunk: Buffer) => parts.push(chunk.toString())); + t.on("end", () => console.log("transform", parts.join(""))); + for (let i = 0; i < 9; i++) t.write(String(i)); + t.end(); +} diff --git a/tests/corpus/7518-stream-dropped-callbacks.ts b/tests/corpus/7518-stream-dropped-callbacks.ts new file mode 100644 index 000000000..faf273d1c --- /dev/null +++ b/tests/corpus/7518-stream-dropped-callbacks.ts @@ -0,0 +1,54 @@ +// Streams dropped before they end, after their option callbacks ran: a +// Writable, an object-mode Transform read in part, and a Writable whose +// destroy() never calls back. Each stream keeps the last `callback` it +// passed, which holds the stream, so dropping one leaves a cycle; the +// sanitized lane's exit audit checks that it is reclaimed. +import { Readable, Transform, Writable } from "node:stream"; + +type Done = (error?: Error | null) => void; + +let written = 0; +function writeSome(n: number): void { + const w = new Writable({ + write(_chunk: Buffer, _encoding: string, callback: Done) { + written++; + callback(); + }, + }); + for (let i = 0; i < n; i++) w.write("x"); +} + +let transformed = 0; +function transformSome(): void { + const t = new Transform({ + objectMode: true, + transform(chunk: number, _encoding: string, callback: (error?: Error | null, data?: number) => void) { + transformed++; + callback(null, chunk * 2); + }, + }); + Readable.from([1, 2, 3, 4, 5, 6]).pipe(t); + t.once("data", (value: number) => console.log("first", value)); +} + +function destroyNoCallback(): void { + const w = new Writable({ + write(_chunk: Buffer, _encoding: string, callback: Done) { + callback(); + }, + destroy(_error: Error | null, _callback: Done) { + console.log("destroy called"); + }, + }); + w.write("y", () => { + w.destroy(); + console.log("destroyed", w.destroyed, w.closed); + }); +} + +for (let i = 0; i < 20; i++) writeSome(5); +transformSome(); +destroyNoCallback(); +setImmediate(() => { + setImmediate(() => console.log("written", written, "transformed", transformed)); +}); diff --git a/tests/harness/corpus-costs/7517-stream-kept-callbacks.ts.json b/tests/harness/corpus-costs/7517-stream-kept-callbacks.ts.json new file mode 100644 index 000000000..ad7424fd8 --- /dev/null +++ b/tests/harness/corpus-costs/7517-stream-kept-callbacks.ts.json @@ -0,0 +1 @@ +{ "plain": 3300, "san": 4300 } diff --git a/tests/harness/corpus-costs/7518-stream-dropped-callbacks.ts.json b/tests/harness/corpus-costs/7518-stream-dropped-callbacks.ts.json new file mode 100644 index 000000000..db9671e02 --- /dev/null +++ b/tests/harness/corpus-costs/7518-stream-dropped-callbacks.ts.json @@ -0,0 +1 @@ +{ "plain": 2400, "san": 5700 } From c66ce24a73e85ccf1e60a34a8081b67780513dbd Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Mon, 26 Oct 2026 03:57:37 -0500 Subject: [PATCH 06/23] Fix Array.prototype calls on native arrays, typeof on ambient names, and net connections to Unix sockets Fixes #2116 Co-authored-by: 4ndual <44326858+4ndual@users.noreply.github.com> Fixes #2134 Co-authored-by: dolphin278 <1130473+dolphin278@users.noreply.github.com> Fixes #2142 --- .../ambient/scriptc-node-fallback.d.ts | 6 ++ .../compiler/src/backend/llvm/lib-network.ts | 16 ++++ .../compiler/src/backend/llvm/lib-shared.ts | 2 + .../frontend/lowering/completion-callbacks.ts | 1 + .../lowering/expressions/identifiers.ts | 10 +-- .../frontend/lowering/expressions/typeof.ts | 16 +++- .../src/frontend/lowering/lower-namespaces.ts | 24 ++++++ .../src/frontend/lowering/lower-server.ts | 77 +++++++++++++++++++ .../src/frontend/lowering/static-typeof.ts | 14 +++- packages/compiler/src/ir/builtin-effects.ts | 2 + .../compiler/src/ir/builtin-signatures.ts | 9 +++ ...array-prototype-call-native-arrays.ts.json | 6 ++ ...nix-socket-connect-default-import.mjs.json | 6 ++ .../corpus/net-unix-socket-connect.ts.json | 6 ++ .../typeof-ambient-undefined-names.ts.json | 6 ++ packages/runtime/src/scr_dyn_invoke.c | 5 +- packages/runtime/src/scr_net.c | 18 +++++ packages/runtime/src/scr_net.h | 4 + .../array-prototype-call-native-arrays.ts | 22 ++++++ ...net-unix-socket-connect-default-import.mjs | 22 ++++++ tests/corpus/net-unix-socket-connect.ts | 69 +++++++++++++++++ .../corpus/typeof-ambient-undefined-names.ts | 21 +++++ ...array-prototype-call-native-arrays.ts.json | 1 + ...nix-socket-connect-default-import.mjs.json | 1 + .../net-unix-socket-connect.ts.json | 1 + .../typeof-ambient-undefined-names.ts.json | 1 + 26 files changed, 352 insertions(+), 14 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/array-prototype-call-native-arrays.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/net-unix-socket-connect-default-import.mjs.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/net-unix-socket-connect.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/typeof-ambient-undefined-names.ts.json create mode 100644 tests/corpus/array-prototype-call-native-arrays.ts create mode 100644 tests/corpus/net-unix-socket-connect-default-import.mjs create mode 100644 tests/corpus/net-unix-socket-connect.ts create mode 100644 tests/corpus/typeof-ambient-undefined-names.ts create mode 100644 tests/harness/corpus-costs/array-prototype-call-native-arrays.ts.json create mode 100644 tests/harness/corpus-costs/net-unix-socket-connect-default-import.mjs.json create mode 100644 tests/harness/corpus-costs/net-unix-socket-connect.ts.json create mode 100644 tests/harness/corpus-costs/typeof-ambient-undefined-names.ts.json diff --git a/packages/compiler/ambient/scriptc-node-fallback.d.ts b/packages/compiler/ambient/scriptc-node-fallback.d.ts index 79305a8ba..2d8d25218 100644 --- a/packages/compiler/ambient/scriptc-node-fallback.d.ts +++ b/packages/compiler/ambient/scriptc-node-fallback.d.ts @@ -5827,6 +5827,12 @@ declare module "net" { }, connectListener?: () => void, ): Socket; + /* A Unix domain socket client (Node's IpcNetConnectOpts and path + * overloads). */ + export function connect(path: string, connectListener?: () => void): Socket; + export function connect(options: { path: string }, connectListener?: () => void): Socket; + export function createConnection(path: string, connectListener?: () => void): Socket; + export function createConnection(options: { path: string }, connectListener?: () => void): Socket; /* The happy-eyeballs attempt budget (Node's process-wide knob; the * autoSelectFamily dial consults it): one runtime int, default 250ms * like Node's. */ diff --git a/packages/compiler/src/backend/llvm/lib-network.ts b/packages/compiler/src/backend/llvm/lib-network.ts index 63661ab4c..5136d4aa0 100644 --- a/packages/compiler/src/backend/llvm/lib-network.ts +++ b/packages/compiler/src/backend/llvm/lib-network.ts @@ -401,6 +401,22 @@ export function emitNetworkHttpLibCall(host: LlvmEmitterContext, e: LibCallExpr) host.emitPendingCheck(); return out; } + if (e.fn === "net.connectPath" || e.fn === "net.connectPathCb") { + const args = e.args.map((a) => host.emitExpr(a)); + let cb = "null"; + if (e.fn === "net.connectPathCb") { + host.toHeap(args[2]!); + cb = args[2]!.name; + } + host.declare(`declare ptr @scr_net_connect_path_checked(ptr, i1 zeroext, ptr)`); + const t = B.tmp(); + B.line( + `${t} = call ptr @scr_net_connect_path_checked(ptr ${args[0]!.name}, i1 ${args[1]!.name}, ptr ${cb})`, + ); + const out = host.own({ name: t, type: e.type }); + host.emitPendingCheck(); + return out; + } if (e.fn === "net.sockOnData" || e.fn === "http.reqOnData") { const cbT = e.args[1]!.type; if (cbT.kind !== "func") diff --git a/packages/compiler/src/backend/llvm/lib-shared.ts b/packages/compiler/src/backend/llvm/lib-shared.ts index 182ac49ab..1051fb36e 100644 --- a/packages/compiler/src/backend/llvm/lib-shared.ts +++ b/packages/compiler/src/backend/llvm/lib-shared.ts @@ -1967,6 +1967,8 @@ export const USES_TIMERS_LIB_FNS = new Set([ "net.listenOptsReusePortCb", "net.connect", "net.connectCb", + "net.connectPath", + "net.connectPathCb", "net.connectLookup", "net.connectLookupDyn", "net.connectAttempt", diff --git a/packages/compiler/src/frontend/lowering/completion-callbacks.ts b/packages/compiler/src/frontend/lowering/completion-callbacks.ts index 1f4013d5e..198b35abd 100644 --- a/packages/compiler/src/frontend/lowering/completion-callbacks.ts +++ b/packages/compiler/src/frontend/lowering/completion-callbacks.ts @@ -45,6 +45,7 @@ const COMPLETION_CALLBACKS: ReadonlyMap = new Map< ["http.requestDynCb", "last"], ["http.requestUrlCb", "last"], ["net.connectCb", 2], + ["net.connectPathCb", 2], ["net.listenCb", 2], ["net.listenOptsCb", 4], ["net.listenOptsReusePortCb", 5], diff --git a/packages/compiler/src/frontend/lowering/expressions/identifiers.ts b/packages/compiler/src/frontend/lowering/expressions/identifiers.ts index a6957401a..7e6762925 100644 --- a/packages/compiler/src/frontend/lowering/expressions/identifiers.ts +++ b/packages/compiler/src/frontend/lowering/expressions/identifiers.ts @@ -55,6 +55,7 @@ import { varRef } from "../../../ir/build.js"; import { ambientUndefReadType, ambientUndefinedFnSymbolOf, + ambientUndefinedVarSymbolOf, contextualUndefReadType, fenceEarlyAliasUse, isTemporalGlobalRoot, @@ -862,14 +863,7 @@ export function lowerIdentifierExpr(lowerer: Lowerer, expr: ts.Identifier, loc: // Stdlib/@types/node ambients never reach here (their chokepoint // is above); only user-file declares do. { - const sym = lowerer.resolveValueSymbol(expr); - const decl = sym ? sym.declarations[0] : undefined; - if ( - decl !== undefined && - ts.isVariableDeclaration(decl) && - decl.initializer === undefined && - ts.getCombinedModifierFlags(decl) & ts.ModifierFlags.Ambient - ) { + if (ambientUndefinedVarSymbolOf(lowerer, expr)) { const declared = lowerer.mapTypeOf(lowerer.typeOf(expr)); if (declared && declared.kind !== "void") { return { diff --git a/packages/compiler/src/frontend/lowering/expressions/typeof.ts b/packages/compiler/src/frontend/lowering/expressions/typeof.ts index f6743768a..5f1e4a830 100644 --- a/packages/compiler/src/frontend/lowering/expressions/typeof.ts +++ b/packages/compiler/src/frontend/lowering/expressions/typeof.ts @@ -34,7 +34,11 @@ import { varRef } from "../../../ir/build.js"; import { isExecExceptionType, refuseExecExceptionCode } from "../builtins/errors.js"; import { InternalCompilerError } from "../../../errors.js"; import { TEMPORAL_GLOBAL } from "../../builtin-modules.js"; -import { isTemporalGlobalRoot } from "../lower-namespaces.js"; +import { + ambientUndefinedFnSymbolOf, + ambientUndefinedVarSymbolOf, + isTemporalGlobalRoot, +} from "../lower-namespaces.js"; import { isF64HandleTyped } from "../f64-handles.js"; export function lowerTypeofExpr(lowerer: Lowerer, expr: ts.TypeOfExpression, loc: SrcLoc): IrExpr { @@ -220,6 +224,16 @@ export function lowerTypeofExpr(lowerer: Lowerer, expr: ts.TypeOfExpression, loc if (isTemporalGlobalRoot(lowerer, tested, sym)) { return { kind: "strLit", value: "undefined", type: STRING, loc }; } + // A name the program declares ambiently but nothing defines + // (`declare const Bun`, `declare function f(): void`): Node erases the + // declaration, so typeof answers "undefined" without reading it, where + // a read throws ReferenceError. + if ( + ambientUndefinedVarSymbolOf(lowerer, tested) || + ambientUndefinedFnSymbolOf(lowerer, tested) + ) { + return { kind: "strLit", value: "undefined", type: STRING, loc }; + } // `const { inspect } = require("util")` / `import { inspect } from // "node:util"`: a binding of a builtin module's exported function, // class or object. JavaScript sources represent such a value without diff --git a/packages/compiler/src/frontend/lowering/lower-namespaces.ts b/packages/compiler/src/frontend/lowering/lower-namespaces.ts index 13d1c29fc..4ea102769 100644 --- a/packages/compiler/src/frontend/lowering/lower-namespaces.ts +++ b/packages/compiler/src/frontend/lowering/lower-namespaces.ts @@ -676,6 +676,30 @@ export function ambientUndefReadType(lowerer: Lowerer, node: ts.Node): IrType | return null; } +/** The symbol of a variable a program file declares ambiently without + * an initializer (`declare const Bun: ...`), which nothing defines: Node + * erases the declaration, so a read throws the catchable ReferenceError + * " is not defined" and typeof answers "undefined". Null for other + * identifiers, including the standard library's declarations, which + * describe real runtime values. */ +export function ambientUndefinedVarSymbolOf( + lowerer: Lowerer, + ident: ts.Identifier, +): ts.Symbol | null { + const sym = lowerer.resolveValueSymbol(ident); + const decl = sym ? sym.declarations[0] : undefined; + if ( + sym === null || + decl === undefined || + lowerer.isStdlibSymbol(sym) || + !ts.isVariableDeclaration(decl) || + decl.initializer !== undefined || + !(ts.getCombinedModifierFlags(decl) & ts.ModifierFlags.Ambient) + ) + return null; + return sym; +} + /** The symbol behind `ident` when it names an ambient `declare function` * NOTHING defines: every function declaration on the symbol is body-less * and ambient in a USER program file (merge partners — the fundule's diff --git a/packages/compiler/src/frontend/lowering/lower-server.ts b/packages/compiler/src/frontend/lowering/lower-server.ts index 614d6a206..d3132d02f 100644 --- a/packages/compiler/src/frontend/lowering/lower-server.ts +++ b/packages/compiler/src/frontend/lowering/lower-server.ts @@ -804,6 +804,19 @@ export function lowerNetModuleCall( `the supported form is ${bi.member}(port, host?, connectListener?)`, ); } + // connect(path[, connectListener]): a string first argument names a + // Unix socket, or a localhost port when it reads as a non-negative + // number (Node's isPipeName, decided at run time). + if (lowerer.mapTypeOf(lowerer.typeOf(args[0]!))?.kind === "string") { + return lowerNetConnectPath( + lowerer, + expr, + bi.member, + lowerer.lowerExpr(args[0]!), + true, + args[1], + ); + } const port = lowerer.lowerExprExpecting(args[0]!, F64); // The optional middle host: a 2-arg call's second argument is the // host when string-typed, the connect listener when func-typed. @@ -841,6 +854,47 @@ export function lowerNetModuleCall( ); } +/** net.connect / net.createConnection of a Unix socket path (a string + * argument or the `path` option) with an optional connect listener. Node + * ignores any other argument between them for a path, so only a listener + * may follow. */ +function lowerNetConnectPath( + lowerer: Lowerer, + expr: ts.CallExpression, + member: string, + path: IrExpr, + pipeNameRule: boolean, + listener: ts.Expression | undefined, +): IrExpr { + const loc = locOf(expr); + const rule: IrExpr = { kind: "boolLit", value: pipeNameRule, type: BOOL, loc }; + if (expr.arguments.nodes.length > 2) { + lowerer.noLowering( + `${member} of a path with ${expr.arguments.nodes.length} arguments`, + expr, + `the supported form is ${member}(path[, connectListener])`, + ); + } + if (listener === undefined) + return { kind: "libCall", fn: "net.connectPath", args: [path, rule], type: NETSOCKET_T, loc }; + const { cb } = lowerCallbackArg( + lowerer, + listener, + "connect listeners", + 0, + () => false, + "use ()", + [], + ); + return { + kind: "libCall", + fn: "net.connectPathCb", + args: [path, rule, cb], + type: NETSOCKET_T, + loc, + }; +} + /** True when `t` is the lookup-function shape the connect lowering * accepts: (hostname: string, options: unknown, callback) => void where * callback is (err: , addresses: { address: @@ -929,6 +983,29 @@ function lowerNetConnectOptions( } } } + // connect({ path }[, connectListener]): the Unix-socket form. + { + const keys = optsNode.properties.nodes.map((p) => + (ts.isPropertyAssignment(p) || ts.isShorthandPropertyAssignment(p)) && + (ts.isIdentifier(p.name) || ts.isStringLiteral(p.name)) + ? p.name.text + : null, + ); + if (keys.length === 1 && keys[0] === "path") { + const prop = optsNode.properties.nodes[0]!; + const path = ts.isPropertyAssignment(prop) + ? lowerer.lowerExpr(prop.initializer) + : lowerer.lowerShorthandValue(prop as ts.ShorthandPropertyAssignment); + if (path.type.kind !== "string") { + lowerer.noLowering( + `a ${member} 'path' option of '${lowerer.fmt(path.type)}' values`, + prop, + "the path is a string here", + ); + } + return lowerNetConnectPath(lowerer, expr, member, path, false, args[1]); + } + } let port: IrExpr | null = null; let host: IrExpr | null = null; let lookup: IrExpr | null = null; diff --git a/packages/compiler/src/frontend/lowering/static-typeof.ts b/packages/compiler/src/frontend/lowering/static-typeof.ts index 852cc3117..cef2d21aa 100644 --- a/packages/compiler/src/frontend/lowering/static-typeof.ts +++ b/packages/compiler/src/frontend/lowering/static-typeof.ts @@ -1,12 +1,14 @@ import * as ts from "../ts7/adapter.js"; import type { Lowerer } from "./lowerer.js"; +import { ambientUndefinedFnSymbolOf, ambientUndefinedVarSymbolOf } from "./lower-namespaces.js"; /** The answer `typeof ` gives without evaluating anything, for * the global-detection idioms (`typeof window < "u"`, `typeof global !== - * "undefined"`): a name nothing declares answers "undefined" (Node's - * answer for an undeclared global such as window or self, and typeof never - * throws for one), and the Node global object and process answer - * "object". Null for every other operand. */ + * "undefined"`): a name nothing declares, or one the program declares + * ambiently and nothing defines, answers "undefined" (Node's answer for an + * undeclared global such as window or self, and typeof never throws for + * one), and the Node global object and process answer "object". Null for + * every other operand. */ export function staticTypeofAnswer(lowerer: Lowerer, operand: ts.Expression): string | null { let e = operand; while (ts.isParenthesizedExpression(e)) e = e.expression; @@ -14,6 +16,10 @@ export function staticTypeofAnswer(lowerer: Lowerer, operand: ts.Expression): st if (lowerer.checker.getSymbolAtLocation(e) === undefined) return "undefined"; for (const name of ["global", "globalThis", "process"]) if (lowerer.isStdlibGlobal(e, name)) return "object"; + // A name the program declares ambiently but nothing defines (`declare + // const Bun`): undeclared at run time. + if (ambientUndefinedVarSymbolOf(lowerer, e) || ambientUndefinedFnSymbolOf(lowerer, e)) + return "undefined"; return null; } diff --git a/packages/compiler/src/ir/builtin-effects.ts b/packages/compiler/src/ir/builtin-effects.ts index 709f72d45..a0d75c864 100644 --- a/packages/compiler/src/ir/builtin-effects.ts +++ b/packages/compiler/src/ir/builtin-effects.ts @@ -944,6 +944,8 @@ export const MAY_THROW_LIB_FNS: ReadonlySet = new Set([ "dns.setDefaultResultOrder", "net.connect", "net.connectCb", + "net.connectPath", + "net.connectPathCb", // resetAndDestroy on a TLS socket throws ERR_INVALID_HANDLE_TYPE. "net.sockResetAndDestroy", // SocketAddress's option checks and parse()'s input check. diff --git a/packages/compiler/src/ir/builtin-signatures.ts b/packages/compiler/src/ir/builtin-signatures.ts index 0be10a37a..843c65ed5 100644 --- a/packages/compiler/src/ir/builtin-signatures.ts +++ b/packages/compiler/src/ir/builtin-signatures.ts @@ -1554,6 +1554,15 @@ function buildLibFnSignatures() { argTypes: [F64, STRING, { kind: "func", params: [], ret: VOID }], result: NETSOCKET_T, }, + /** net.connect/createConnection(path[, connectListener]) and the + * `path` option: a client of the Unix socket at `path`. With the flag + * (a positional string), Node's isPipeName applies first: a string that + * reads as a non-negative number is a localhost port. */ + "net.connectPath": { argTypes: [STRING, BOOL], result: NETSOCKET_T }, + "net.connectPathCb": { + argTypes: [STRING, BOOL, { kind: "func", params: [], ret: VOID }], + result: NETSOCKET_T, + }, // The lookup's exact func shape is program data (its answer callback's // union/record types) — checked specially in the libCall case. /** connect({ port, host, autoSelectFamily: true, lookup }) — the diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/array-prototype-call-native-arrays.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/array-prototype-call-native-arrays.ts.json new file mode 100644 index 000000000..2904322c5 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/array-prototype-call-native-arrays.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/array-prototype-call-native-arrays.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/net-unix-socket-connect-default-import.mjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/net-unix-socket-connect-default-import.mjs.json new file mode 100644 index 000000000..a1bb1df19 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/net-unix-socket-connect-default-import.mjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/net-unix-socket-connect-default-import.mjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/net-unix-socket-connect.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/net-unix-socket-connect.ts.json new file mode 100644 index 000000000..7bf0a928d --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/net-unix-socket-connect.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/net-unix-socket-connect.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/typeof-ambient-undefined-names.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/typeof-ambient-undefined-names.ts.json new file mode 100644 index 000000000..a2bd8dc1c --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/typeof-ambient-undefined-names.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/typeof-ambient-undefined-names.ts" + ], + "diags": [] +} diff --git a/packages/runtime/src/scr_dyn_invoke.c b/packages/runtime/src/scr_dyn_invoke.c index e31359efd..8ea75df0b 100644 --- a/packages/runtime/src/scr_dyn_invoke.c +++ b/packages/runtime/src/scr_dyn_invoke.c @@ -1801,7 +1801,10 @@ static ScrDyn *dyn_array_method_value_call(ScrClosure *closure, ScrDyn *const *a ScrStr *method = scr_box_get_ref(closure->caps[0]); ScrDyn *receiver = scr_dyn_this_get(); ScrDyn *result; - if (receiver->kind == SCR_DYN_ARR) { + /* An array, or a live view of a native array, runs the method its own + * dispatch has; other receivers take the generic array-like algorithms. */ + if (receiver->kind == SCR_DYN_ARR || + (receiver->kind == SCR_DYN_TYPED_REF && scr_dyn_typed_ref_array_ops(receiver))) { result = scr_dyn_invoke(receiver, method->data, args, argc, method->data); } else { ScrDyn *pack = scr_dyn_new_arr(); diff --git a/packages/runtime/src/scr_net.c b/packages/runtime/src/scr_net.c index a940fb7f1..776829e8c 100644 --- a/packages/runtime/src/scr_net.c +++ b/packages/runtime/src/scr_net.c @@ -2741,6 +2741,24 @@ ScrNetSocket *scr_net_connect_checked(double port, ScrStr *host /*borrowed, null return scr_net_connect(port, host, cb); } +/* net.connect/createConnection(path[, connectListener]) and the `path` + * option from compiled code: a client of the Unix socket at `path` + * (borrowed). For a positional string (`pipe_name_rule`), Node's isPipeName + * applies first: a string that reads as a non-negative number is a port on + * localhost. `cb` (moves, nullable) is the connect listener. */ +ScrNetSocket *scr_net_connect_path_checked(ScrStr *path, bool pipe_name_rule, ScrClosure *cb) { + double port = pipe_name_rule ? scr_string_to_number(path) : -1; + if (port >= 0) { + ScrStr *host = scr_str_new("localhost", 9); + ScrNetSocket *s = scr_net_connect_checked(port, host, cb); + scr_str_release(host); + return s; + } + ScrNetSocket *s = scr_net_connect_path(path, false); + if (cb) scr_net_ls_add(&s->conn_ls, cb, NULL, true); + return s; +} + static bool scr_net_dyn_truthy(const ScrDyn *v) { switch (v->kind) { case SCR_DYN_UNDEF: diff --git a/packages/runtime/src/scr_net.h b/packages/runtime/src/scr_net.h index 6a41e26d8..af4b8e1b8 100644 --- a/packages/runtime/src/scr_net.h +++ b/packages/runtime/src/scr_net.h @@ -265,6 +265,10 @@ ScrNetSocket *scr_net_connect_lookup_dyn(double port, ScrStr *host /*borrowed*/, /* A client socket for the Unix socket at `path`, dialed now or (deferred) * by scr_net_sock_dial_start. A failure is the deferred 'error'. */ ScrNetSocket *scr_net_connect_path(ScrStr *path /*borrowed*/, bool deferred); /* +1 */ +/* net.connect(path[, cb]) and connect({ path }): with `pipe_name_rule`, a + * numeric string is a localhost port, as in Node. */ +ScrNetSocket *scr_net_connect_path_checked(ScrStr *path /*borrowed*/, bool pipe_name_rule, + ScrClosure *cb /*moves, nullable*/); /* +1 */ /* server.listen(path[, cb]): a Unix-socket server, its file removed at close. */ void scr_net_listen_path(ScrNetServer *s, ScrStr *path /*borrowed*/, ScrClosure *cb /*moves, nullable*/); void scr_net_sock_dial_start(ScrNetSocket *s); diff --git a/tests/corpus/array-prototype-call-native-arrays.ts b/tests/corpus/array-prototype-call-native-arrays.ts new file mode 100644 index 000000000..ce198ff73 --- /dev/null +++ b/tests/corpus/array-prototype-call-native-arrays.ts @@ -0,0 +1,22 @@ +// Array.prototype methods called with .call on native arrays run the arrays' own +// methods, which mutate the array in place: copyWithin, fill, reverse, +// sort, push, pop, splice, and the read-only join, indexOf and includes. +const optional: (number | undefined)[] = [1, 2, 3, 4]; +Array.prototype.copyWithin.call(optional, 0, 2, 4); +console.log(optional.join(",")); +const nums: number[] = [5, 1, 4, 2, 3]; +Array.prototype.fill.call(nums, 0, 0, 1); +console.log(nums.join(",")); +console.log(Array.prototype.reverse.call(nums) === nums, nums.join(",")); +Array.prototype.sort.call(nums, (a: number, b: number) => a - b); +console.log(nums.join(",")); +console.log(Array.prototype.push.call(nums, 9, 10), nums.join(",")); +console.log(Array.prototype.pop.call(nums), nums.length); +console.log(Array.prototype.splice.call(nums, 1, 2).join(","), nums.join(",")); +console.log(Array.prototype.join.call(nums, "-"), Array.prototype.indexOf.call(nums, 4)); +const words: string[] = ["c", "a", "b"]; +Array.prototype.sort.call(words); +console.log(words.join(","), Array.prototype.includes.call(words, "a")); +const records: { id: number }[] = [{ id: 1 }, { id: 2 }, { id: 3 }]; +Array.prototype.copyWithin.call(records, 1, 0, 1); +console.log(records.map((r) => r.id).join(","), records[0] === records[1]); diff --git a/tests/corpus/net-unix-socket-connect-default-import.mjs b/tests/corpus/net-unix-socket-connect-default-import.mjs new file mode 100644 index 000000000..7e58c18e2 --- /dev/null +++ b/tests/corpus/net-unix-socket-connect-default-import.mjs @@ -0,0 +1,22 @@ +// net.createConnection(path) through the default import of node:net, as a +// JavaScript module reads it: a client of a Unix domain socket server. +import net from "node:net"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +const dir = fs.mkdtempSync(path.join(os.tmpdir(), "scriptc-unix-default-")); +const socketPath = path.join(dir, "server.sock"); +const server = net.createServer((socket) => socket.end("hello")); +server.listen(socketPath, () => { + const sock = net.createConnection(socketPath); + sock.once("connect", () => console.log("connected")); + sock.on("data", (chunk) => console.log("data", String(chunk))); + sock.once("error", (e) => console.log("error", String(e))); + sock.on("close", () => { + server.close(() => { + fs.rmSync(dir, { recursive: true, force: true }); + console.log("closed"); + }); + }); +}); diff --git a/tests/corpus/net-unix-socket-connect.ts b/tests/corpus/net-unix-socket-connect.ts new file mode 100644 index 000000000..77832d60d --- /dev/null +++ b/tests/corpus/net-unix-socket-connect.ts @@ -0,0 +1,69 @@ +// net.createConnection and net.connect to a Unix domain socket: a path +// argument, the `path` option, a connect listener, data both ways, the +// error for a path nothing listens on, and a numeric string, which Node +// treats as a localhost port. +import * as net from "node:net"; +import * as fs from "node:fs"; +import * as os from "node:os"; +import * as path from "node:path"; + +const dir = fs.mkdtempSync(path.join(os.tmpdir(), "scriptc-unix-connect-")); +const socketPath = path.join(dir, "server.sock"); +const missingPath = path.join(dir, "missing.sock"); + +const server = net.createServer((socket) => { + socket.on("data", (chunk) => { + socket.write(`echo:${chunk.toString()}`); + }); +}); + +function exchange(label: string, socket: net.Socket, message: string): Promise { + return new Promise((resolve) => { + socket.on("data", (chunk) => { + console.log(label, chunk.toString()); + socket.end(); + }); + socket.on("close", () => resolve()); + socket.write(message); + }); +} + +server.listen(socketPath, async () => { + console.log("listening"); + const first = net.createConnection(socketPath); + first.once("connect", () => console.log("first connected")); + await exchange("first", first, "one"); + + let listenerCalls = 0; + const second = net.connect(socketPath, () => { + listenerCalls++; + }); + await exchange("second", second, "two"); + console.log("second listener calls", listenerCalls); + + const third = net.createConnection({ path: socketPath }); + await exchange("third", third, "three"); + + await new Promise((resolve) => { + const missing = net.createConnection(missingPath); + missing.once("connect", () => console.log("missing connected")); + missing.once("error", (error: NodeJS.ErrnoException) => { + console.log("missing", error.code, error.message === `connect ENOENT ${missingPath}`); + resolve(); + }); + }); + + server.close(() => { + const tcp = net.createServer((socket) => socket.end("tcp")); + tcp.listen(0, "localhost", () => { + const { port } = tcp.address() as net.AddressInfo; + const client = net.createConnection(String(port)); + client.on("data", (chunk) => console.log("numeric string", chunk.toString())); + client.on("close", () => { + tcp.close(); + fs.rmSync(dir, { recursive: true, force: true }); + console.log("done"); + }); + }); + }); +}); diff --git a/tests/corpus/typeof-ambient-undefined-names.ts b/tests/corpus/typeof-ambient-undefined-names.ts new file mode 100644 index 000000000..94713319e --- /dev/null +++ b/tests/corpus/typeof-ambient-undefined-names.ts @@ -0,0 +1,21 @@ +// typeof on names the program declares (`declare const`, `declare function`) +// but nothing defines at run time answers "undefined"; reading them throws +// ReferenceError. +declare const Bun: { randomUUIDv7: () => string } | undefined; +declare const Deno: { version: { deno: string } }; +declare function webkitRequestAnimationFrame(callback: () => void): number; +console.log("start"); +console.log(typeof Bun, typeof Deno, typeof webkitRequestAnimationFrame); +console.log(typeof Bun !== "undefined" ? Bun?.randomUUIDv7() : "no Bun"); +const runtime = typeof Deno === "object" ? "deno" : typeof Bun === "object" ? "bun" : "node"; +console.log(runtime); +try { + console.log(Bun); +} catch (error) { + console.log((error as Error).name, (error as Error).message); +} +try { + console.log(typeof Deno.version); +} catch (error) { + console.log((error as Error).name, (error as Error).message); +} diff --git a/tests/harness/corpus-costs/array-prototype-call-native-arrays.ts.json b/tests/harness/corpus-costs/array-prototype-call-native-arrays.ts.json new file mode 100644 index 000000000..d88973daa --- /dev/null +++ b/tests/harness/corpus-costs/array-prototype-call-native-arrays.ts.json @@ -0,0 +1 @@ +{ "plain": 5514, "san": 7205 } diff --git a/tests/harness/corpus-costs/net-unix-socket-connect-default-import.mjs.json b/tests/harness/corpus-costs/net-unix-socket-connect-default-import.mjs.json new file mode 100644 index 000000000..0c62f10ce --- /dev/null +++ b/tests/harness/corpus-costs/net-unix-socket-connect-default-import.mjs.json @@ -0,0 +1 @@ +{ "plain": 3242, "san": 5402 } diff --git a/tests/harness/corpus-costs/net-unix-socket-connect.ts.json b/tests/harness/corpus-costs/net-unix-socket-connect.ts.json new file mode 100644 index 000000000..d03fa6389 --- /dev/null +++ b/tests/harness/corpus-costs/net-unix-socket-connect.ts.json @@ -0,0 +1 @@ +{ "plain": 3555, "san": 6682 } diff --git a/tests/harness/corpus-costs/typeof-ambient-undefined-names.ts.json b/tests/harness/corpus-costs/typeof-ambient-undefined-names.ts.json new file mode 100644 index 000000000..91c82c655 --- /dev/null +++ b/tests/harness/corpus-costs/typeof-ambient-undefined-names.ts.json @@ -0,0 +1 @@ +{ "plain": 4212, "san": 4590 } From 2d2343e36fd19bd47f1247f143dd58227403b6f2 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Mon, 26 Oct 2026 05:03:24 -0500 Subject: [PATCH 07/23] Keep program partitions local to the code an edit changes --- native/llvm-codegen/README.md | 2 +- native/llvm-codegen/src/partition.cpp | 178 ++++++++++++------ native/llvm-codegen/src/partition.h | 14 +- .../compiler/test/native-partitions.test.ts | 109 ++++++++++- 4 files changed, 231 insertions(+), 72 deletions(-) diff --git a/native/llvm-codegen/README.md b/native/llvm-codegen/README.md index 02ac6e3ef..0af8a11d2 100644 --- a/native/llvm-codegen/README.md +++ b/native/llvm-codegen/README.md @@ -33,7 +33,7 @@ lowering, verifies before and after optimization, and publishes through a private sibling file so a failed or interrupted request cannot truncate the requested output. -Repeating `--output` divides the program into that many partitions, which are compiled concurrently and linked together. The helper splits the emitter's LLVM assembly text directly, so partitions also parse concurrently; input outside the emitter's narrow grammar is parsed whole and split with LLVM instead. Internal definitions become hidden so partitions can reference each other. At opt-level 0 each partition is compiled independently. Optimized partitions follow the ThinLTO model: each is simplified and summarized on its own thread, a thin link decides which small functions every partition imports for inlining and which hidden definitions become internal again, and each partition is then optimized and compiled on its own thread. Partition placement depends only on the module and the partition count, so output never depends on the host. +Repeating `--output` divides the program into that many partitions, which are compiled concurrently and linked together. The helper splits the emitter's LLVM assembly text directly, so partitions also parse concurrently; input outside the emitter's narrow grammar is parsed whole and split with LLVM instead. Internal definitions that no other definition reaches are left out of every partition, and the others become hidden so partitions can reference each other. At opt-level 0 each partition is compiled independently. Optimized partitions follow the ThinLTO model: each is simplified and summarized on its own thread, a thin link decides which small functions every partition imports for inlining and which hidden definitions become internal again, and each partition is then optimized and compiled on its own thread. Partition placement depends only on the module and the partition count, so output never depends on the host. WebAssembly code generation differs from LLVM's default pipeline in two ways that do not change its output. The pass that gives multiple-entry loops a single entry runs only on functions whose control flow is irreducible; its reachability analysis grows with the square of a function's block count even when it has nothing to change. Object emission also leaves out the module pass that collects external symbols for assembly declarations, so each function's machine code is freed once it is printed instead of when the whole module is compiled. diff --git a/native/llvm-codegen/src/partition.cpp b/native/llvm-codegen/src/partition.cpp index 93b9af4a3..ef5230129 100644 --- a/native/llvm-codegen/src/partition.cpp +++ b/native/llvm-codegen/src/partition.cpp @@ -98,7 +98,7 @@ void placeGroups(std::vector &Symbols, ArrayRef Roots, StringMap GroupBytes; for (unsigned I = 0; I < Symbols.size(); ++I) { const Symbol &S = Symbols[I]; - if (S.Kind != SymbolKind::Function) + if (S.Kind != SymbolKind::Function || S.Partition == NoPartition) continue; Total += Bytes(S); StringRef Group = Symbols[Roots[I]].Group; @@ -119,8 +119,10 @@ void placeGroups(std::vector &Symbols, ArrayRef Roots, StringMap KeyBytes; for (unsigned I = 0; I < Symbols.size(); ++I) { const Symbol &S = Symbols[I]; + if (S.Kind != SymbolKind::Function || S.Partition == NoPartition) + continue; StringRef Group = Symbols[Roots[I]].Group; - if (S.Kind != SymbolKind::Function || Group.empty()) + if (Group.empty()) continue; uint64_t Chunks = 1; while (Chunks * Chunk < GroupBytes[Group]) @@ -534,64 +536,6 @@ std::optional partitionProgramText(StringRef Text, } } - // An internal function referenced by exactly one other function, and by - // no global, is a whole-program inlining candidate; keep it with that - // function so its partition can inline it without importing. A function - // body larger than this is past LLVM's inline cost limit even for the only - // call to a local function, so it is placed by its own name. Otherwise a - // program's module initializers, which only the entry point calls, would - // all share the entry point's partition. - constexpr size_t MaxInlinedBodyBytes = 256 * 1024; - constexpr unsigned Shared = ~1u; - std::vector Referrer(Symbols.size(), NoPartition); - for (unsigned I = 0; I < Symbols.size(); ++I) { - const Symbol &S = Symbols[I]; - if (S.Kind == SymbolKind::Declaration) - continue; - for (unsigned Ref : S.Refs) { - if (Ref == I) - continue; - if (S.Kind == SymbolKind::Variable || Referrer[Ref] != NoPartition) - Referrer[Ref] = Shared; - else - Referrer[Ref] = I; - } - } - std::vector Roots(Symbols.size()); - for (unsigned I = 0; I < Symbols.size(); ++I) { - if (Symbols[I].Kind != SymbolKind::Function) - continue; - unsigned Root = I; - for (unsigned Steps = 0; Steps < 1024; ++Steps) { - const Symbol &S = Symbols[Root]; - unsigned Parent = Referrer[Root]; - if (!S.Local || S.Body.size() > MaxInlinedBodyBytes || - Parent == NoPartition || Parent == Shared || - Symbols[Parent].Kind != SymbolKind::Function || Parent == I) - break; - Root = Parent; - } - Roots[I] = Root; - Symbols[I].Partition = stableHash(Symbols[Root].Name) % Count; - } - placeGroups(Symbols, Roots, Count); - for (const Symbol &S : Symbols) { - if (S.Kind != SymbolKind::Function) - continue; - for (unsigned Ref : S.Refs) - if (Symbols[Ref].Kind == SymbolKind::Variable && - Symbols[Ref].Partition == NoPartition) - Symbols[Ref].Partition = S.Partition; - } - for (Symbol &S : Symbols) - if (S.Kind == SymbolKind::Variable && S.Partition == NoPartition) - S.Partition = stableHash(S.Name) % Count; - - ProgramPartitions Result; - for (const Symbol &S : Symbols) - if (S.Kind != SymbolKind::Declaration && !S.Local) - Result.PublicSymbols.insert(S.Name); - // Metadata nodes are numbered in the emitter's global order, so one added // node would renumber the references in every partition. Each partition // instead numbers the nodes it uses in order of first use, which keeps an @@ -606,10 +550,122 @@ std::optional partitionProgramText(StringRef Text, for (StringRef Line : NamedMetadata) scanReferences(Line, Index, NamedSymbols, NamedRoots); + // An internal definition that no other definition reaches is left out of + // every partition: the optimizer would delete it, and the emitter writes + // variants of helpers that the program never calls. Leaving them out + // shortens every partition's parse, summary and optimization, and since + // placement ignores them too, an edit that adds one changes no partition. + std::vector Live(Symbols.size(), 0); + { + std::vector Work; + auto Reach = [&](unsigned I) { + if (!Live[I]) { + Live[I] = 1; + Work.push_back(I); + } + }; + for (unsigned I = 0; I < Symbols.size(); ++I) + if (Symbols[I].Kind != SymbolKind::Declaration && !Symbols[I].Local) + Reach(I); + for (unsigned I : NamedSymbols) + Reach(I); + for (auto &Entry : Metadata) + for (unsigned I : Entry.second.Symbols) + Reach(I); + while (!Work.empty()) { + unsigned I = Work.back(); + Work.pop_back(); + for (unsigned Ref : Symbols[I].Refs) + Reach(Ref); + } + } + + // An internal definition reached from exactly one other definition is an + // inlining candidate of it, or data only it uses; keep it with that + // definition so its partition can inline it without importing, and so a + // helper an edit adds lands in the partition of the code that uses it. A + // chain through a global continues to the function that uses the global. + // A function body larger than this is past LLVM's inline cost limit even + // for the only call to a local function, so it is placed by its own name. + // A chain stops where it would leave a function's source-module group: + // module initializers, which only the entry point calls, keep their own + // partitions instead of all sharing the entry point's. + constexpr size_t MaxInlinedBodyBytes = 256 * 1024; + constexpr unsigned Shared = ~1u; + std::vector Referrer(Symbols.size(), NoPartition); + for (unsigned I = 0; I < Symbols.size(); ++I) { + const Symbol &S = Symbols[I]; + if (S.Kind == SymbolKind::Declaration || !Live[I]) + continue; + for (unsigned Ref : S.Refs) { + if (Ref == I) + continue; + Referrer[Ref] = Referrer[Ref] == NoPartition ? I : Shared; + } + } + auto Follows = [&](unsigned Child, unsigned Parent) { + const Symbol &S = Symbols[Child]; + if (!S.Local || Parent == NoPartition || Parent == Shared || + Symbols[Parent].Kind == SymbolKind::Declaration) + return false; + if (S.Kind == SymbolKind::Function && + (S.Body.size() > MaxInlinedBodyBytes || + (Symbols[Parent].Kind == SymbolKind::Function && !S.Group.empty() && + S.Group != Symbols[Parent].Group))) + return false; + return true; + }; + std::vector Roots(Symbols.size()); + for (unsigned I = 0; I < Symbols.size(); ++I) { + if (Symbols[I].Kind == SymbolKind::Declaration || !Live[I]) + continue; + unsigned Root = I; + for (unsigned Steps = 0; Steps < 1024; ++Steps) { + unsigned Parent = Referrer[Root]; + if (!Follows(Root, Parent) || Parent == I) + break; + Root = Parent; + } + // A function whose chain ends at a global that is not itself placed + // with a function (a table many functions use) is placed by its own + // name, so the functions of one table spread over the partitions. + if (Symbols[I].Kind == SymbolKind::Function && + Symbols[Root].Kind != SymbolKind::Function) + Root = I; + Roots[I] = Root; + if (Symbols[I].Kind == SymbolKind::Function) + Symbols[I].Partition = stableHash(Symbols[Root].Name) % Count; + } + placeGroups(Symbols, Roots, Count); + // A global follows the function its chain ends at, then the first + // function that references it, then its own name. + for (unsigned I = 0; I < Symbols.size(); ++I) + if (Symbols[I].Kind == SymbolKind::Variable && Live[I] && + Symbols[Roots[I]].Kind == SymbolKind::Function) + Symbols[I].Partition = Symbols[Roots[I]].Partition; + for (unsigned I = 0; I < Symbols.size(); ++I) { + const Symbol &S = Symbols[I]; + if (S.Kind != SymbolKind::Function || !Live[I]) + continue; + for (unsigned Ref : S.Refs) + if (Symbols[Ref].Kind == SymbolKind::Variable && + Symbols[Ref].Partition == NoPartition) + Symbols[Ref].Partition = S.Partition; + } + for (unsigned I = 0; I < Symbols.size(); ++I) + if (Symbols[I].Kind == SymbolKind::Variable && Live[I] && + Symbols[I].Partition == NoPartition) + Symbols[I].Partition = stableHash(Symbols[I].Name) % Count; + + ProgramPartitions Result; + for (const Symbol &S : Symbols) + if (S.Kind != SymbolKind::Declaration && !S.Local) + Result.PublicSymbols.insert(S.Name); + Result.Sources.resize(Count); std::vector> Owned(Count); for (unsigned I = 0; I < Symbols.size(); ++I) - if (Symbols[I].Kind != SymbolKind::Declaration) + if (Symbols[I].Kind != SymbolKind::Declaration && Live[I]) Owned[Symbols[I].Partition].push_back(I); auto Assemble = [&](unsigned P) { diff --git a/native/llvm-codegen/src/partition.h b/native/llvm-codegen/src/partition.h index 35ef4c543..dc000d751 100644 --- a/native/llvm-codegen/src/partition.h +++ b/native/llvm-codegen/src/partition.h @@ -24,11 +24,15 @@ struct ProgramPartitions { // more partitions, functions under a "; scriptc-group " line are placed // with the other functions of that source-module group, which is divided // into chunks when it is larger than the mean partition or 1 MiB; other -// functions are placed by a stable hash of their name. An internal function referenced by exactly one other function stays -// with that function unless its body is too large to inline, and a global -// stays with the first function that references it. Returns nullopt for any -// construct outside the emitter's narrow top-level grammar; callers then -// split a parsed module. +// functions are placed by a stable hash of their name. An internal +// definition referenced by exactly one other definition stays with it, also +// through a global, unless it is a function whose body is too large to +// inline or that would join a function of another source-module group; a +// global otherwise stays with the first function that references it. +// Internal definitions that no non-internal definition, named metadata or +// metadata node reaches are left out of every partition. Returns nullopt +// for any construct outside the emitter's narrow top-level grammar; callers +// then split a parsed module. std::optional partitionProgramText(llvm::StringRef Text, unsigned Count); diff --git a/packages/compiler/test/native-partitions.test.ts b/packages/compiler/test/native-partitions.test.ts index fcd0132e8..113cd95ad 100644 --- a/packages/compiler/test/native-partitions.test.ts +++ b/packages/compiler/test/native-partitions.test.ts @@ -204,12 +204,111 @@ describe.runIf(helper !== null)("LLVM program partitions", () => { expect([...after].filter((entry) => !sources.has(entry))).toHaveLength(1); expect(await exitCode(dir, edited, "edited")).toBe(expectedExit(7, steps)); - // With fewer partitions, functions keep their placement by name. - const contents = (paths: string[]) => Promise.all(paths.map((path) => readFile(path))); + // With fewer partitions, functions keep their placement by name, and a + // group line only stops a single-caller chain: step0 joins main without + // groups, and keeps its own placement in group 0. const few = { outputs: 4, optLevel: "0" } as const; - expect( - await contents(await emit(dir, grouped(0, 24), { ...few, name: "few-grouped" })), - ).toEqual(await contents(await emit(dir, program(), { ...few, name: "few-plain" }))); + const placed = async (objects: string[]) => { + const where = new Map(); + for (const [index, object] of objects.entries()) { + const { stdout } = await execFileAsync("nm", [object]); + for (const line of stdout.split("\n")) { + const defined = /^[0-9a-fA-F]+ [A-Za-z] _?(step\d+|main)$/.exec(line.trim()); + if (defined !== null) where.set(defined[1]!, index); + } + } + return where; + }; + const fewGrouped = await placed( + await emit(dir, grouped(0, 24), { ...few, name: "few-grouped" }), + ); + const fewPlain = await placed(await emit(dir, program(), { ...few, name: "few-plain" })); + expect(fewPlain.get("step0")).toBe(fewPlain.get("main")); + for (let step = 1; step < 24; step++) + expect(fewGrouped.get(`step${step}`)).toBe(fewPlain.get(`step${step}`)); + expect(fewGrouped.get("main")).toBe(fewPlain.get("main")); + }); + + test("internal definitions nothing reaches are left out of every partition", async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-partitions-unreachable-")); + dirs.push(dir); + // An unreferenced internal function that uses live code and its own + // state, which only it references. + const unused = (name: string) => + [ + `@${name}State = internal global i64 7`, + `define internal void @${name}(i64 %value) {`, + ` %state = load i64, ptr @${name}State`, + " call void @bump(i64 %state)", + " ret void", + "}", + ].join("\n"); + const cache = join(dir, "cache"); + const options = { outputs: 4, optLevel: "0", cache } as const; + const objects = await emit(dir, program(0, unused("unusedOne")), { ...options, name: "first" }); + expect(await exitCode(dir, objects, "first")).toBe(expectedExit()); + for (const object of objects) { + const { stdout } = await execFileAsync("nm", [object]); + expect(stdout).not.toMatch(/unusedOne/); + } + // Adding another one changes no partition. + const before = await cacheEntries(cache, ".o"); + const more = await emit(dir, program(0, `${unused("unusedOne")}\n${unused("unusedTwo")}`), { + ...options, + name: "more", + }); + expect(await cacheEntries(cache, ".o")).toEqual(before); + expect(await exitCode(dir, more, "more")).toBe(expectedExit()); + }); + + test("a helper that one function uses goes to that function's partition", async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-partitions-helpers-")); + dirs.push(dir); + // step7 calls @helper directly, and step9 calls @viaTable through an + // internal global only step9 reads. + const helpers = (withHelpers: boolean) => + program() + .split("\n") + .flatMap((line) => { + if (!withHelpers) return [line]; + if (line === "define internal void @step7(i64 %value) noinline {") + return [ + "define internal void @helper(i64 %value) noinline {", + " call void @bump(i64 %value)", + " ret void", + "}", + "@table = internal constant ptr @viaTable", + "define internal void @viaTable(i64 %value) noinline {", + " call void @bump(i64 %value)", + " ret void", + "}", + line, + " call void @helper(i64 0)", + ]; + if (line === "define internal void @step9(i64 %value) noinline {") + return [line, " %fn = load ptr, ptr @table", " call void %fn(i64 0)"]; + return [line]; + }) + .join("\n"); + const cache = join(dir, "cache"); + const options = { outputs: 4, optLevel: "0", cache } as const; + await emit(dir, helpers(false), { ...options, name: "before" }); + const before = await cacheEntries(cache, ".o"); + const objects = await emit(dir, helpers(true), { ...options, name: "after" }); + expect(await exitCode(dir, objects, "after")).toBe(expectedExit()); + const where = new Map(); + for (const [index, object] of objects.entries()) { + const { stdout } = await execFileAsync("nm", [object]); + for (const line of stdout.split("\n")) { + const defined = /^[0-9a-fA-F]+ [A-Za-z] _?(step7|step9|helper|viaTable)$/.exec(line.trim()); + if (defined !== null) where.set(defined[1]!, index); + } + } + expect(where.get("helper")).toBe(where.get("step7")); + expect(where.get("viaTable")).toBe(where.get("step9")); + // Only the partitions of step7 and step9 change. + const changed = [...(await cacheEntries(cache, ".o"))].filter((entry) => !before.has(entry)); + expect(changed).toHaveLength(new Set([where.get("step7"), where.get("step9")]).size); }); test("a table written one element per line keeps the text partitioner and its reuse", async () => { From 7ae99f63bc31da77fb6aeb620d1a5ba87c9ea516 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Mon, 26 Oct 2026 05:52:57 -0500 Subject: [PATCH 08/23] Give stream option callbacks Node's callback identities Node passes a stream's one onwrite function to every _write and _writev call, and a new function to every _transform call. A write or writev option callback now receives the same closure on every call (scr_stream_write_closure), kept or not, so a Set, WeakSet or WeakMap finds it again. A reused transform, flush, final, construct or destroy callback is first forgotten by weak collections, as at its free, so a WeakSet or WeakMap that recorded the previous call's callback does not find it on the next call. --- .../src/backend/llvm/expr-stream-callbacks.ts | 12 ++-- packages/runtime/src/scr_stream.c | 72 ++++++++++++++----- packages/runtime/src/scr_stream.h | 5 +- tests/corpus/7517-stream-kept-callbacks.ts | 64 ++++++++++++++++- 4 files changed, 128 insertions(+), 25 deletions(-) diff --git a/packages/compiler/src/backend/llvm/expr-stream-callbacks.ts b/packages/compiler/src/backend/llvm/expr-stream-callbacks.ts index 7c5cba75b..3d9d8da76 100644 --- a/packages/compiler/src/backend/llvm/expr-stream-callbacks.ts +++ b/packages/compiler/src/backend/llvm/expr-stream-callbacks.ts @@ -684,11 +684,15 @@ export function streamCbThunkFor( passed.push(`ptr %sr`); } // The stream-owning completion closure (a 1-cap closure boxing the - // retained stream) — shared by the typed and dyn done shapes. The - // stream reuses it while it holds the only reference. + // retained stream) — shared by the typed and dyn done shapes. Every + // write and writev call receives the same one, as Node passes one + // onwrite function; other calls receive a new one, which the stream + // reuses while it holds the only reference. + const doneEntry = + kind === "w" || kind === "v" ? "scr_stream_write_closure" : "scr_stream_done_closure"; const doneClosure = (fnRef: string, tag: string): string => { - host.declare(`declare ptr @scr_stream_done_closure(ptr, ptr)`); - d.push(` %${tag}clo = call ptr @scr_stream_done_closure(ptr %s, ptr ${fnRef})`); + host.declare(`declare ptr @${doneEntry}(ptr, ptr)`); + d.push(` %${tag}clo = call ptr @${doneEntry}(ptr %s, ptr ${fnRef})`); return `%${tag}clo`; }; for (let i = off; i < declared.length; i++) { diff --git a/packages/runtime/src/scr_stream.c b/packages/runtime/src/scr_stream.c index b45d27528..2f4ad185e 100644 --- a/packages/runtime/src/scr_stream.c +++ b/packages/runtime/src/scr_stream.c @@ -246,10 +246,14 @@ struct ScrStreamState { ScrClosure *destroy_cb; ScrStreamErrInv destroy_inv; - /* The completion closure (`callback`) the emitted option-callback - * thunks last passed, owned: scr_stream_done_closure hands it out again - * while this is its only reference. It holds the stream, so it is - * dropped at 'close' and traced for a stream dropped before it closes. */ + /* The completion closures (`callback`) the emitted option-callback + * thunks pass, owned. write_done is the one every write and writev call + * receives (scr_stream_write_closure); done_cache is the last one a + * transform, flush, final, construct or destroy call received, handed + * out again while this is its only reference (scr_stream_done_closure). + * Both hold the stream, so they are dropped at 'close' and traced for a + * stream dropped before it closes. */ + ScrClosure *write_done; ScrClosure *done_cache; /* finished()/pipeline() watchers: fired once at the terminal point @@ -423,6 +427,7 @@ static void scr_stream_state_drop(ScrStreamState *st, bool gc) { if (st->transform_cb) scr_closure_release(st->transform_cb); if (st->flush_cb) scr_closure_release(st->flush_cb); if (st->destroy_cb) scr_closure_release(st->destroy_cb); + if (st->write_done) scr_closure_release(st->write_done); if (st->done_cache) scr_closure_release(st->done_cache); for (size_t i = 0; i < st->w.end_cbs_n; i++) scr_closure_release(st->w.end_cbs[i]); for (size_t i = 0; i < st->fin.n; i++) scr_closure_release(st->fin.cb[i]); @@ -472,6 +477,7 @@ void scr_stream_st_trace(ScrStreamState *st, ScrTraceVisit visit, void *ctx) { if (st->transform_cb) visit(st->transform_cb, ctx); if (st->flush_cb) visit(st->flush_cb, ctx); if (st->destroy_cb) visit(st->destroy_cb, ctx); + if (st->write_done) visit(st->write_done, ctx); if (st->done_cache) visit(st->done_cache, ctx); for (size_t i = 0; i < st->w.end_cbs_n; i++) visit(st->w.end_cbs[i], ctx); for (size_t i = 0; i < st->fin.n; i++) visit(st->fin.cb[i], ctx); @@ -527,24 +533,53 @@ void scr_stream_release(ScrStream *s) { void *scr_stream_retain_v(void *s) { return scr_stream_retain(s); } void scr_stream_release_v(void *s) { scr_stream_release(s); } -/* The completion closure an option callback receives as `callback`: a - * 1-cap closure calling fn over a box that holds the stream. The stream - * keeps the last one and hands it out again while the cache holds its - * only reference and it has no own properties; otherwise a new closure - * replaces it, and one the user kept keeps its own stream reference. - * Nothing is cached after 'close'. Returns +1. */ -ScrClosure *scr_stream_done_closure(ScrStream *s, void *fn) { - ScrStreamState *st = s->st; - ScrClosure *c = st->done_cache; - if (c != NULL && c->rc == 1 && c->fn == fn && c->props == NULL) return scr_closure_retain(c); - c = scr_closure_new(fn, 1); +/* A completion closure: a 1-cap closure calling fn over a box that holds + * the stream. Returns +1. */ +static ScrClosure *scr_stream_done_new(ScrStream *s, void *fn) { + ScrClosure *c = scr_closure_new(fn, 1); ScrBox *bx = scr_box_new_obj(scr_stream_retain_v, scr_stream_release_v, scr_stream_trace); c->caps[0] = bx; scr_box_set_ref(bx, scr_stream_retain(s)); - if (st->close_emitted) return c; - ScrClosure *old = st->done_cache; - st->done_cache = scr_closure_retain(c); + return c; +} + +/* Keeps c in *slot, replacing the closure there. Nothing is kept after + * 'close'. Borrows c. */ +static void scr_stream_done_keep(ScrStreamState *st, ScrClosure **slot, ScrClosure *c) { + if (st->close_emitted) return; + ScrClosure *old = *slot; + *slot = scr_closure_retain(c); if (old) scr_closure_release(old); +} + +/* The `callback` a write or writev option callback receives. Node passes + * the stream's one onwrite function to every _write and _writev call, so + * every call receives the same closure, kept or not. Returns +1. */ +ScrClosure *scr_stream_write_closure(ScrStream *s, void *fn) { + ScrStreamState *st = s->st; + ScrClosure *c = st->write_done; + if (c != NULL && c->fn == fn) return scr_closure_retain(c); + c = scr_stream_done_new(s, fn); + scr_stream_done_keep(st, &st->write_done, c); + return c; +} + +/* The `callback` a transform, flush, final, construct or destroy option + * callback receives. Node passes a new function to each call. The stream + * keeps the last closure and hands it out again while it holds the only + * reference and the closure has no own properties; weak collections then + * forget it first, as they would at its free, so it is a new key to them. + * Otherwise a new closure replaces it, and one the user kept keeps its own + * stream reference. Returns +1. */ +ScrClosure *scr_stream_done_closure(ScrStream *s, void *fn) { + ScrStreamState *st = s->st; + ScrClosure *c = st->done_cache; + if (c != NULL && c->rc == 1 && c->fn == fn && c->props == NULL) { + scr_weak_dispose_inline(c); + return scr_closure_retain(c); + } + c = scr_stream_done_new(s, fn); + scr_stream_done_keep(st, &st->done_cache, c); return c; } @@ -7989,6 +8024,7 @@ static void scr_stream_run_tick(ScrStreamTick *t) { if (st->transform_cb) { scr_closure_release(st->transform_cb); st->transform_cb = NULL; } if (st->flush_cb) { scr_closure_release(st->flush_cb); st->flush_cb = NULL; } if (st->destroy_cb) { scr_closure_release(st->destroy_cb); st->destroy_cb = NULL; } + if (st->write_done) { scr_closure_release(st->write_done); st->write_done = NULL; } if (st->done_cache) { scr_closure_release(st->done_cache); st->done_cache = NULL; } } break; diff --git a/packages/runtime/src/scr_stream.h b/packages/runtime/src/scr_stream.h index 9b950d445..578712df2 100644 --- a/packages/runtime/src/scr_stream.h +++ b/packages/runtime/src/scr_stream.h @@ -681,7 +681,10 @@ void scr_stream_release(ScrStream *s); void *scr_stream_retain_v(void *s); void scr_stream_release_v(void *s); void scr_stream_trace(void *obj, ScrTraceVisit visit, void *ctx); -/* The `callback` an option callback receives (+1), reused while the +/* The `callback` a write or writev option callback receives (+1): the + * same closure for every call, as Node passes one onwrite function. */ +ScrClosure *scr_stream_write_closure(ScrStream *s, void *fn); +/* The `callback` any other option callback receives (+1), reused while the * stream holds its only reference. */ ScrClosure *scr_stream_done_closure(ScrStream *s, void *fn); void scr_stream_install(void); diff --git a/tests/corpus/7517-stream-kept-callbacks.ts b/tests/corpus/7517-stream-kept-callbacks.ts index f721fab31..9b4067529 100644 --- a/tests/corpus/7517-stream-kept-callbacks.ts +++ b/tests/corpus/7517-stream-kept-callbacks.ts @@ -3,7 +3,9 @@ // calls its callback and keeps it (`cb(); keep(cb)`) and calls it again // after later writes (ERR_MULTIPLE_CALLBACK), and a Transform that calls // back synchronously, from setImmediate, and from a promise. Every write -// and transform completes in Node's order. +// and transform completes in Node's order. Every write and writev call +// receives the same callback, as in Node (a Set, a WeakSet and a WeakMap +// find it again); a transform call receives a new one each time. import { Transform, Writable } from "node:stream"; type Done = (error?: Error | null) => void; @@ -73,7 +75,65 @@ function transforms(): void { }); const parts: string[] = []; t.on("data", (chunk: Buffer) => parts.push(chunk.toString())); - t.on("end", () => console.log("transform", parts.join(""))); + t.on("end", () => { + console.log("transform", parts.join("")); + weakKeys(); + }); for (let i = 0; i < 9; i++) t.write(String(i)); t.end(); } + +function weakKeys(): void { + const seen = new WeakSet(); + const notes = new WeakMap(); + const held = new Set(); + const w = new Writable({ + write(chunk: Buffer, _encoding: string, callback: Done) { + const text = chunk.toString(); + console.log("weak write", text, seen.has(callback), notes.get(callback) ?? "none", held.has(callback)); + seen.add(callback); + notes.set(callback, text); + if (text === "q") held.add(callback); + callback(); + }, + }); + for (const text of ["p", "q", "r", "s"]) w.write(text); + w.end(batches); +} + +function batches(): void { + const seen = new WeakSet(); + let first: Done | null = null; + const w = new Writable({ + writev(chunks: Array<{ chunk: Buffer }>, callback: Done) { + const text = chunks.map((c) => c.chunk.toString()).join(""); + console.log("writev", text, seen.has(callback), first === callback); + seen.add(callback); + if (first === null) first = callback; + setImmediate(() => callback()); + }, + }); + w.write("a"); + w.write("b"); + w.write("c"); + w.end(weakTransform); +} + +function weakTransform(): void { + type TransformDone = (error?: Error | null, data?: Buffer) => void; + const seen = new WeakSet(); + const held = new Set(); + const t = new Transform({ + transform(chunk: Buffer, _encoding: string, callback: TransformDone) { + const text = chunk.toString(); + console.log("weak transform", text, seen.has(callback), held.has(callback)); + seen.add(callback); + if (text === "x") held.add(callback); + callback(null, chunk); + }, + }); + t.resume(); + t.on("end", () => console.log("weak transform end")); + for (const text of ["x", "y", "z"]) t.write(text); + t.end(); +} From a758ff1b4ed6211438ae12b249b9c628d73383d6 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Mon, 26 Oct 2026 06:02:01 -0500 Subject: [PATCH 09/23] Select sanitized runtime variants by unit, and skip stripping without llvm-strip The native sanitizer build chose the variants to compile by artifact path. Since the runtime pack stores identical objects once, two variants of a unit can share a path, so it compiled such a unit for both variants and linked it twice (duplicate symbols in the sanitized native bootstrap). selectedRuntimeUnits now returns each selected unit with its variant; selectRuntimePackArtifacts and the sanitizer both use it. Staging the native CLI's assets ran llvm-strip unconditionally and failed on hosts without it. Without llvm-strip on PATH, tsc-ts now keeps its symbols and the build prints a warning; the release hosts install LLVM. --- .../compiler/src/backend/runtime-pack-core.ts | 17 +- .../compiler/src/native/sanitizer.test.ts | 229 ++++++++++++------ packages/compiler/src/native/sanitizer.ts | 27 +-- scripts/native-cli-assets.mts | 14 +- 4 files changed, 197 insertions(+), 90 deletions(-) diff --git a/packages/compiler/src/backend/runtime-pack-core.ts b/packages/compiler/src/backend/runtime-pack-core.ts index c2e9dd07f..9c99bbdcf 100644 --- a/packages/compiler/src/backend/runtime-pack-core.ts +++ b/packages/compiler/src/backend/runtime-pack-core.ts @@ -399,6 +399,19 @@ export function runtimePackFlavorKey( return optimized; } +/** The units a flavor links for `features`, each with its selected + * variant. Variants of one unit can share an artifact file (the pack stores + * identical objects once), so a variant is identified by its unit and id, + * not by its path. */ +export function selectedRuntimeUnits( + flavor: RuntimePackFlavorManifest, + features: RuntimeFeatureSet, +): { source: string; variant: RuntimePackVariant }[] { + return flavor.runtime_units + .filter((unit) => evaluateRuntimePredicate(unit.predicate, features)) + .map((unit) => ({ source: unit.source, variant: selectVariant(unit, features) })); +} + /** Selection is deterministic and independent of installation paths. Both * hosts verify and privately stage exactly these artifacts before linking. */ export function selectRuntimePackArtifacts( @@ -420,9 +433,7 @@ export function selectRuntimePackArtifacts( ); return { features, - runtime: selectedFlavor.runtime_units - .filter((unit) => evaluateRuntimePredicate(unit.predicate, features)) - .map((unit) => selectVariant(unit, features)), + runtime: selectedRuntimeUnits(selectedFlavor, features).map(({ variant }) => variant), archives: manifest.archives.filter((archive) => evaluateRuntimePredicate(archive.predicate, features), ), diff --git a/packages/compiler/src/native/sanitizer.test.ts b/packages/compiler/src/native/sanitizer.test.ts index df2bdf69b..782a21962 100644 --- a/packages/compiler/src/native/sanitizer.test.ts +++ b/packages/compiler/src/native/sanitizer.test.ts @@ -8,6 +8,78 @@ import type { RuntimePackManifest } from "../backend/runtime-pack-core.js"; import { NATIVE_TARGETS } from "../backend/targets.js"; import { buildSanitizedRuntime } from "./sanitizer.js"; +/** Every runtime feature off except regex. */ +const features = { + workers: false, + youngNursery: false, + eventTarget: false, + wasm: false, + dynTier: false, + streamDyn: false, + handleProps: false, + declAdopt: false, + typeOrigins: false, + dynForAwait: false, + eventsOnceSignal: false, + mime: false, + utilTiming: false, + dynBuffer: false, + builtinValues: false, + computedRequire: false, + warningReplace: false, + stdioExtended: false, + brotli: false, + moduleNamespace: false, + regex: true, + copying: false, + textDecoderLegacy: false, + fileHandle: false, + fetch: false, + zlib: false, + assert: false, + inspect: false, + dynInvoke: false, + dc: false, + dynAsync: false, + events: false, + emitter: false, + symbol: false, + bigint: false, + searchParams: false, + qs: false, + utilPromisify: false, + dynBuiltinClass: false, + dynBuiltinStream: false, + parseArgs: false, + stream: false, + net: false, + http: false, + http2: false, + dgram: false, + dnsPromises: false, + dgramDyn: false, + textCodecDyn: false, + watch: false, + foreignFfi: false, + nodeTest: false, + tls: false, + tlsCa: false, + cipher: false, + ecdh: false, + dh: false, + pki: false, + ed25519: false, + x509: false, + webcrypto: false, + perfLoop: false, + nativeFetch: false, + netEffective: false, + httpEffective: false, + tlsEffective: false, + tlsCaEffective: false, + zlibEffective: false, +}; + test.skipIf(process.platform !== "linux" && process.platform !== "darwin")( "sanitized vendor archives compile with host libc declarations", () => { @@ -72,76 +144,7 @@ test.skipIf(process.platform !== "linux" && process.platform !== "darwin")( runtime: [], archives: [artifact], systemLibraries: [], - features: { - workers: false, - youngNursery: false, - eventTarget: false, - wasm: false, - dynTier: false, - streamDyn: false, - handleProps: false, - declAdopt: false, - typeOrigins: false, - dynForAwait: false, - eventsOnceSignal: false, - mime: false, - utilTiming: false, - dynBuffer: false, - builtinValues: false, - computedRequire: false, - warningReplace: false, - stdioExtended: false, - brotli: false, - moduleNamespace: false, - regex: true, - copying: false, - textDecoderLegacy: false, - fileHandle: false, - fetch: false, - zlib: false, - assert: false, - inspect: false, - dynInvoke: false, - dc: false, - dynAsync: false, - events: false, - emitter: false, - symbol: false, - bigint: false, - searchParams: false, - qs: false, - utilPromisify: false, - dynBuiltinClass: false, - dynBuiltinStream: false, - parseArgs: false, - stream: false, - net: false, - http: false, - http2: false, - dgram: false, - dnsPromises: false, - dgramDyn: false, - textCodecDyn: false, - watch: false, - foreignFfi: false, - nodeTest: false, - tls: false, - tlsCa: false, - cipher: false, - ecdh: false, - dh: false, - pki: false, - ed25519: false, - x509: false, - webcrypto: false, - perfLoop: false, - nativeFetch: false, - netEffective: false, - httpEffective: false, - tlsEffective: false, - tlsCaEffective: false, - zlibEffective: false, - }, + features, }, }, directory, @@ -160,3 +163,89 @@ test.skipIf(process.platform !== "linux" && process.platform !== "darwin")( } }, ); + +test.skipIf(process.platform !== "linux" && process.platform !== "darwin")( + "variants that share an artifact file compile once, as the selected variant", + () => { + const target = NATIVE_TARGETS.find( + (item) => + item.platform === process.platform && + item.architecture === process.arch && + !item.name.endsWith("-musl"), + ); + if (target === undefined) throw new Error("unsupported sanitizer test host"); + const runtimeSourceRoot = fileURLToPath(new URL("../../../runtime", import.meta.url)); + const { version } = JSON.parse( + readFileSync(join(runtimeSourceRoot, "package.json"), "utf8"), + ) as { version: string }; + const directory = mkdtempSync("/tmp/scriptc-sanitized-shared-"); + // The pack stores the identical objects of both variants once. + const shared = { + path: "artifacts/dev/runtime/default/scr_hash.o", + sha256: "0".repeat(64), + size: 0, + }; + const variants = [ + { ...shared, id: "default", when: { workers: false }, defines: [] }, + { ...shared, id: "default-workers", when: { workers: true }, defines: ["SCR_WORKERS"] }, + ]; + const manifest: RuntimePackManifest = { + schema: "scriptc.runtime-pack.v1", + format: 1, + package: target.runtimePackPackage, + version, + target: { + name: target.name, + llvm_triple: target.llvmTriple, + architecture: target.architecture, + object_format: target.objectFormat, + minimum_os: target.minimumOs, + }, + runtime_abi: { version: RUNTIME_ABI_VERSION, marker: RUNTIME_ABI_MARKER }, + compiler: { command: "clang", identity: "test", target: target.llvmTriple }, + macros: { executable: [], excluded: [], sanitizer: "external-toolchain-required" }, + flavors: { + dev: { + optimization: "-O0", + runtime_units: [{ source: "scr_hash.c", predicate: true, variants }], + }, + }, + archives: [], + system_libraries: [], + licenses: [], + }; + try { + const built = buildSanitizedRuntime( + { + compilerVersion: version, + target, + helper: target.helper, + runtimeSourceRoot, + ts7Executable: "unused", + helperExecutable: "unused", + helperPackageRoot: "unused", + runtimePackRoot: "unused", + linker: "unused", + linkerArgs: [], + dsymutil: "unused", + }, + { + root: directory, + manifest, + licenses: manifest.licenses, + packageText: "", + manifestText: "", + flavor: "dev", + selected: { runtime: [variants[0]!], archives: [], systemLibraries: [], features }, + }, + directory, + "executable", + null, + ); + expect(built.runtimeObjects).toEqual([join(directory, "scr_hash.o")]); + expect(statSync(join(directory, "scr_hash.o")).size).toBeGreaterThan(0); + } finally { + rmSync(directory, { recursive: true, force: true }); + } + }, +); diff --git a/packages/compiler/src/native/sanitizer.ts b/packages/compiler/src/native/sanitizer.ts index 5d228a615..abc15d043 100644 --- a/packages/compiler/src/native/sanitizer.ts +++ b/packages/compiler/src/native/sanitizer.ts @@ -9,7 +9,11 @@ import { type NativeRuntimePack, type NativeRuntimeSelection, } from "../backend/runtime-pack-native.js"; -import { runtimePackFlavorKey, type RuntimePackMode } from "../backend/runtime-pack-core.js"; +import { + runtimePackFlavorKey, + selectedRuntimeUnits, + type RuntimePackMode, +} from "../backend/runtime-pack-core.js"; import { LRE_SOURCES, ZLIB_SOURCES } from "../backend/vendor-inputs.js"; import { contentDigest, type NativeCache } from "./cache.js"; import type { NativeToolchain } from "./toolchain.js"; @@ -127,7 +131,6 @@ export function buildSanitizedRuntime( const key = runtimePackFlavorKey(manifest, selection.flavor, mode); const flavor = manifest.flavors[key]; if (flavor === undefined) throw new NativeCodegenError("SC3003", `runtime pack lacks ${key}`); - const wanted = new Set(selection.selected.runtime.map((artifact) => artifact.path)); const common = [ "-std=c11", "-pthread", @@ -147,18 +150,14 @@ export function buildSanitizedRuntime( join(brotli, "include"), ...(toolchain.target.platform === "linux" ? ["-ffunction-sections", "-fdata-sections"] : []), ]; - const runtimeObjects: string[] = []; - for (const unit of flavor.runtime_units) { - for (const variant of unit.variants) { - if (wanted.has(variant.path)) - runtimeObjects.push( - compile(join(src, unit.source), unit.source.replace(/\.c$/, ".o"), [ - ...common, - ...variant.defines.map((define) => "-D" + define), - ]), - ); - } - } + // The units and variants the pack selection chose, compiled from source. + const runtimeObjects = selectedRuntimeUnits(flavor, selection.selected.features).map( + ({ source, variant }) => + compile(join(src, source), source.replace(/\.c$/, ".o"), [ + ...common, + ...variant.defines.map((define) => "-D" + define), + ]), + ); const archives: string[] = []; for (const archive of selection.selected.archives) { const name = basename(archive.path) diff --git a/scripts/native-cli-assets.mts b/scripts/native-cli-assets.mts index 8162c9091..95cd493db 100644 --- a/scripts/native-cli-assets.mts +++ b/scripts/native-cli-assets.mts @@ -1,4 +1,4 @@ -import { execFileSync } from "node:child_process"; +import { spawnSync } from "node:child_process"; import { cpSync, existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from "node:fs"; import { basename, dirname, join, relative } from "node:path"; import type { NativeTargetSpec } from "../packages/compiler/src/backend/targets.js"; @@ -27,10 +27,18 @@ function copyPackage(source: string, destination: string, members: readonly stri /** Removes the symbol table and debug sections of a copied executable. * llvm-strip rewrites the linker-signed ad hoc code signature of a Mach-O * executable, which arm64 macOS requires to run it. Windows executables keep - * their debug information in separate .pdb files, which are not copied. */ + * their debug information in separate .pdb files, which are not copied. + * Without llvm-strip on PATH the executable keeps its symbols; the release + * hosts install it with LLVM. */ function stripExecutable(path: string, platform: string): void { if (platform === "win32") return; - execFileSync("llvm-strip", [path], { stdio: "inherit" }); + const result = spawnSync("llvm-strip", [path], { stdio: "inherit" }); + if ((result.error as NodeJS.ErrnoException | undefined)?.code === "ENOENT") { + console.warn(`llvm-strip is not on PATH; ${basename(path)} keeps its symbols`); + return; + } + if (result.error !== undefined) throw result.error; + if (result.status !== 0) throw new Error(`llvm-strip ${path} exited with ${result.status}`); } /** Rewrites CRLF line endings to LF in the text files under `directory`. A From 9820569fc07438256477d4aa89e02712d966d571 Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Sun, 25 Oct 2026 19:54:47 -0700 Subject: [PATCH 10/23] Compare canonical mapping with mapType in an opt-in shadow mode Stage N1 of the localized-failure design (canonical type normalization). mapCanonical (frontend/canon-map.ts) maps a canonical node to IR from its structural form: primitives, literals, units, the top types, arrays, flat unions and plain records. Nominal objects, callables, index-signature shapes, tuples, bigint and symbol keep their per-type recognitions through mapType on the node's checker type. Under SCRIPTC_CANON_SHADOW=1, every successful top-level mapType also runs canon and mapCanonical in throwaway registries and appends a JSON line per comparison (equal, diverge, canon-fail) to SCRIPTC_CANON_SHADOW_OUT. The census decides which rule families can later move to the normalizer. The hook is absent otherwise, so no output changes. Co-Authored-By: Claude Opus 5.5 --- packages/compiler/src/frontend/canon-map.ts | 153 ++++++++++++++++++ packages/compiler/src/frontend/canon.ts | 33 +++- .../compiler/src/frontend/lowering/lowerer.ts | 3 + packages/compiler/src/frontend/type-mapper.ts | 6 + .../compiler/test/canon-normalize.test.ts | 36 +++++ 5 files changed, 227 insertions(+), 4 deletions(-) create mode 100644 packages/compiler/src/frontend/canon-map.ts diff --git a/packages/compiler/src/frontend/canon-map.ts b/packages/compiler/src/frontend/canon-map.ts new file mode 100644 index 000000000..e7d55e450 --- /dev/null +++ b/packages/compiler/src/frontend/canon-map.ts @@ -0,0 +1,153 @@ +/* Canonical mapping (localized-failure design §4.3), stage N1: the IR type + * of a canonical node, built from about ten structural rules plus the + * nominal table. Nominal objects (classes, standard-library interfaces, + * handles), callables, index-signature shapes, tuples, bigint and symbol + * keep their per-type recognitions: they map through mapType on the + * checker type the node came from. Everything structural (primitives, + * literals, units, arrays, flat unions, plain records) maps here, from the + * canonical form alone. + * + * Shadow mode (SCRIPTC_CANON_SHADOW=1) compares this mapping with mapType + * for every successful top-level mapping and appends one JSON line per + * comparison to SCRIPTC_CANON_SHADOW_OUT. It maps into throwaway + * registries, so the program's own IR is untouched. */ +import { appendFileSync } from "node:fs"; +import { BOOL, DYN, F64, NULL_T, STRING, UNDEFINED_T, typeKey, type IrType } from "../ir/ir.js"; +import { canonKey, Normalizer, type Canon } from "./canon.js"; +import * as ts from "./ts7/adapter.js"; +import { + mapType, + ShapeRegistry, + UnionRegistry, + withUndefinedArm, + type TypeMapperCtx, +} from "./type-mapper.js"; + +const MAX_DEPTH = 24; + +/** The IR type of a canonical node, or null when no rule applies. */ +export function mapCanonical(c: Canon, ctx: TypeMapperCtx, depth = 0): IrType | null { + if (depth > MAX_DEPTH) return null; + const viaChecker = (): IrType | null => (c.src === undefined ? null : mapType(c.src, ctx)); + switch (c.k) { + case "prim": + return c.p === "number" + ? F64 + : c.p === "string" + ? STRING + : c.p === "boolean" + ? BOOL + : viaChecker(); + case "lit": + return typeof c.value === "number" ? F64 : typeof c.value === "string" ? STRING : BOOL; + case "unit": + return c.u === "undefined" ? UNDEFINED_T : c.u === "null" ? NULL_T : viaChecker(); + case "top": + return DYN; + case "param": + return null; + case "opaque": + case "tuple": + return viaChecker(); + case "array": { + const elem = mapCanonical(c.elem, ctx, depth + 1); + return elem === null ? null : { kind: "array", elem }; + } + case "union": { + const byKey = new Map(); + for (const arm of c.arms) { + const mapped = mapCanonical(arm, ctx, depth + 1); + if (mapped === null || mapped.kind === "void") return null; + if (mapped.kind === "union") { + const def = ctx.unions.get(mapped.unionId); + if (def === undefined) return null; + for (const a of def.arms) byKey.set(typeKey(a), a); + } else byKey.set(typeKey(mapped), mapped); + } + const arms = [...byKey.values()]; + if (arms.length === 1) return arms[0]!; + if (arms.some((a) => a.kind === "dyn")) return DYN; + arms.sort((a, b) => (typeKey(a) < typeKey(b) ? -1 : 1)); + return { kind: "union", unionId: ctx.unions.intern(arms) }; + } + case "obj": { + if (c.nominal !== undefined || c.call().length > 0 || c.construct().length > 0) + return viaChecker(); + if (c.index().length > 0) return viaChecker(); + const props = c.props(); + // `{}` holds any non-nullish value: the checked dynamic, as mapType. + if (props.length === 0) return DYN; + const fields: { name: string; type: IrType }[] = []; + for (const p of props) { + let t = mapCanonical(p.type(), ctx, depth + 1); + if (t === null) return null; + if (p.optional && t.kind !== "dyn") { + const armed = withUndefinedArm(t, ctx.unions); + if (armed === null) return null; + t = armed; + } + fields.push({ name: p.name, type: t }); + } + const declaredOrder = fields.map((f) => f.name); + fields.sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0)); + return { + kind: "record", + shapeId: ctx.shapes.intern(fields, false, undefined, declaredOrder), + }; + } + } +} + +export type ShadowStatus = "equal" | "diverge" | "canon-fail"; + +/** Installs nothing unless SCRIPTC_CANON_SHADOW=1: the hook mapType calls + * after each successful top-level mapping. */ +export function canonShadowHook( + checker: ts.TypeChecker, +): ((type: ts.Type, mapped: IrType, ctx: TypeMapperCtx) => void) | undefined { + if (process.env["SCRIPTC_CANON_SHADOW"] !== "1") return undefined; + const out = process.env["SCRIPTC_CANON_SHADOW_OUT"]; + let busy = false; + return (type, mapped, ctx) => { + if (busy) return; + busy = true; + try { + const resolve = ctx.resolveTypeParamTs; + const normalizer = new Normalizer(checker); + const c = normalizer.canon(type, resolve ? (t) => resolve(t) : new Map()); + const { typeMemo: _memo, canonShadow: _hook, ...rest } = ctx; + const shadowCtx: TypeMapperCtx = { + ...rest, + shapes: new ShapeRegistry(), + unions: new UnionRegistry(), + canMemoizeType: () => false, + }; + let canonical: IrType | null = null; + try { + canonical = mapCanonical(c, shadowCtx); + } catch { + canonical = null; + } + const status: ShadowStatus = + canonical === null + ? "canon-fail" + : typeKey(canonical) === typeKey(mapped) + ? "equal" + : "diverge"; + if (out !== undefined) + appendFileSync( + out, + `${JSON.stringify({ + status, + kind: c.k, + type: checker.typeToString(type).slice(0, 160), + canon: canonKey(c).slice(0, 160), + strict: typeKey(mapped), + ...(canonical !== null ? { canonical: typeKey(canonical) } : {}), + })}\n`, + ); + } finally { + busy = false; + } + }; +} diff --git a/packages/compiler/src/frontend/canon.ts b/packages/compiler/src/frontend/canon.ts index 5d75ae806..d40312388 100644 --- a/packages/compiler/src/frontend/canon.ts +++ b/packages/compiler/src/frontend/canon.ts @@ -44,7 +44,13 @@ export interface AliasRef { tsType: ts.Type; } -export type Canon = +export type Canon = CanonNode & { + /** The checker type the node was normalized from (a type parameter's + * binding for substituted parameters). */ + src?: ts.Type; +}; + +type CanonNode = | { k: "prim"; p: CanonPrim } | { k: "lit"; value: string | number | boolean } | { k: "unit"; u: "undefined" | "null" | "void" | "never" } @@ -67,8 +73,10 @@ export type Canon = | { k: "param"; name: string } | { k: "opaque"; ts: ts.Type; reason: string }; -/** Type-parameter bindings: the checker type a parameter is bound to. */ -export type CanonEnv = ReadonlyMap; +/** Type-parameter bindings: the checker type a parameter is bound to, by + * the parameter's symbol, or a resolver over the parameter type (the + * lowerer's resolveTypeParamTs). */ +export type CanonEnv = ReadonlyMap | ((param: ts.Type) => ts.Type | null); const PRIM_FLAGS: readonly [number, CanonPrim][] = [ [ts.TypeFlags.Number, "number"], @@ -259,11 +267,15 @@ export class Normalizer { if (known !== undefined) return known; } const node = this.build(type, env, key); + if (node.src === undefined && node.k !== "param") node.src = type; if (key !== null && !this.interned.has(key)) this.interned.set(key, node); return node; } private envKey(env: CanonEnv): string { + // A resolver's bindings are the caller's current instantiation: a + // normalizer is used for one instantiation at a time. + if (typeof env === "function") return "fn"; if (env.size === 0) return ""; const parts: string[] = []; for (const [param, bound] of env) { @@ -294,7 +306,12 @@ export class Normalizer { for (const [flag, p] of PRIM_FLAGS) if (flags & flag) return { k: "prim", p }; if (flags & ts.TypeFlags.TypeParameter) { const symbol = type.getSymbol(); - const bound = symbol === undefined ? undefined : env.get(symbol); + const bound = + typeof env === "function" + ? (env(type) ?? undefined) + : symbol === undefined + ? undefined + : env.get(symbol); if (bound !== undefined && bound !== type) return this.canon(bound, env); return { k: "param", name: symbol === undefined ? "?" : ts.symbolName(symbol) }; } @@ -406,6 +423,14 @@ export class Normalizer { }); return node; } + // A mapped type the checker could not resolve (over an unbound or + // symbolically bound parameter) shows no members: irreducible here. + if ( + ((type as ts.ObjectType).objectFlags & ts.ObjectFlags.Mapped) !== 0 && + checker.getPropertiesOfType(type).length === 0 && + checker.getIndexInfosOfType(type).length === 0 + ) + return { k: "opaque", ts: type, reason: "unresolved mapped type" }; const symbol = type.getSymbol(); const decl = symbol?.declarations[0]; const nominal = diff --git a/packages/compiler/src/frontend/lowering/lowerer.ts b/packages/compiler/src/frontend/lowering/lowerer.ts index 18547772f..3b78344ed 100644 --- a/packages/compiler/src/frontend/lowering/lowerer.ts +++ b/packages/compiler/src/frontend/lowering/lowerer.ts @@ -7,6 +7,7 @@ import { arrayRootOf, lowerWithArraySpecies, } from "./builtins/array-classes.js"; +import { canonShadowHook } from "../canon-map.js"; import { compactDeclOrigins, inPackageFile, @@ -4088,6 +4089,7 @@ export class Lowerer { this.externalTypeSpecifiersByFile = mode.externalTypeSpecifiersByFile ?? directExternalTypeSpecifiersByFile(this.externalTypes); this.checker = program.getTypeChecker(); + const shadow = canonShadowHook(this.checker); this.typeCtx = { checker: this.checker, positionOf: (node) => this.positions.of(node), @@ -4101,6 +4103,7 @@ export class Lowerer { const id = (type as { id?: number }).id; return id === undefined ? null : (this.degradedTypes.get(id)?.lenient ?? null); }, + ...(shadow !== undefined ? { canonShadow: shadow } : {}), isErasing: () => this.erasingDepth > 0, genericClassInstance: (decl, ref, mapArgument) => this.genericClassInstanceType(decl, ref, mapArgument), diff --git a/packages/compiler/src/frontend/type-mapper.ts b/packages/compiler/src/frontend/type-mapper.ts index 86ff00129..72b29629b 100644 --- a/packages/compiler/src/frontend/type-mapper.ts +++ b/packages/compiler/src/frontend/type-mapper.ts @@ -1287,6 +1287,10 @@ export interface TypeMapperCtx { * only after the strict mapping (and its bound-alias retry) returns null, * at every depth, so every occurrence of a degraded type maps alike. */ degradedType?: (type: ts.Type) => IrType | null; + /** Canonical normalization's shadow mode (canon-map.ts): called after + * each successful top-level mapping; installed only under + * SCRIPTC_CANON_SHADOW=1. It records, and never changes a result. */ + canonShadow?: (type: ts.Type, mapped: IrType, ctx: TypeMapperCtx) => void; /** GENERIC program classes (monomorphization by flow): the mapping of a * concrete instantiation reference (`Box`) — the Lowerer * registers/reuses the instantiation (`Box%0`) and answers its object @@ -1911,6 +1915,8 @@ export function mapType(type: ts.Type, ctx: TypeMapperCtx): IrType | null { ) { ctx.typeMemo?.set(memoKey, mapped); } + if (mapped !== null && topLevel && ctx.canonShadow !== undefined) + ctx.canonShadow(type, mapped, ctx); return mapped; } finally { mapTypeDepth--; diff --git a/packages/compiler/test/canon-normalize.test.ts b/packages/compiler/test/canon-normalize.test.ts index ac25924f3..c9140c6f3 100644 --- a/packages/compiler/test/canon-normalize.test.ts +++ b/packages/compiler/test/canon-normalize.test.ts @@ -3,6 +3,14 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { canonEqual, canonKey, Normalizer, type Canon } from "../src/frontend/canon.js"; +import { mapCanonical } from "../src/frontend/canon-map.js"; +import { + mapType, + ShapeRegistry, + UnionRegistry, + type TypeMapperCtx, +} from "../src/frontend/type-mapper.js"; +import { typeKey } from "../src/ir/ir.js"; import { loadProgram } from "../src/frontend/program-node.js"; import * as ts from "../src/frontend/ts7/adapter.js"; @@ -271,3 +279,31 @@ describe("canon", () => { expect(key(n.canon(aliases.get("Pair")!))).toBe("[string, number]"); }); }); + +describe("mapCanonical", () => { + const ctx = (): TypeMapperCtx => ({ + checker, + shapes: new ShapeRegistry(), + unions: new UnionRegistry(), + classNamer: (decl) => decl.name?.text ?? "anonymous", + isStdlibFile: (sf) => load.program.isSourceFileDefaultLibrary(sf), + isNpmFile: () => false, + isExternalTypeFile: () => false, + isProgramFile: (sf) => load.moduleOrder.includes(sf), + }); + test.for(["Normalized", "Same", "Flat", "Id", "Pair"])("%s maps like mapType", (name) => { + const strict = mapType(aliases.get(name)!, ctx()); + const canonical = mapCanonical(new Normalizer(checker).canon(aliases.get(name)!), ctx()); + expect(strict).not.toBeNull(); + expect(canonical && typeKey(canonical)).toBe(typeKey(strict!)); + }); + test("a generic alias maps under bindings, where mapType needs the instance table", () => { + const { type, params } = asTypes.get("Catalog")!; + const env = new Map([ + [params[0]!, valueTypes.get("sampleNumber")!], + [params[1]!, valueTypes.get("sampleText")!], + ]); + const canonical = mapCanonical(new Normalizer(checker).canon(type, env), ctx()); + expect(canonical?.kind).toBe("record"); + }); +}); From 5c1494161735778eae3727dc160b49b5c0704286 Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Sun, 25 Oct 2026 20:49:33 -0700 Subject: [PATCH 11/23] Bound canonical keys and keep the shadow from breaking a compile Co-Authored-By: Claude Opus 5.5 --- packages/compiler/src/frontend/canon-map.ts | 3 +++ packages/compiler/src/frontend/canon.ts | 20 +++++++++++++++++--- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/packages/compiler/src/frontend/canon-map.ts b/packages/compiler/src/frontend/canon-map.ts index e7d55e450..2f40af21b 100644 --- a/packages/compiler/src/frontend/canon-map.ts +++ b/packages/compiler/src/frontend/canon-map.ts @@ -111,6 +111,7 @@ export function canonShadowHook( return (type, mapped, ctx) => { if (busy) return; busy = true; + // The shadow records and must never break a compile. try { const resolve = ctx.resolveTypeParamTs; const normalizer = new Normalizer(checker); @@ -146,6 +147,8 @@ export function canonShadowHook( ...(canonical !== null ? { canonical: typeKey(canonical) } : {}), })}\n`, ); + } catch { + // A census line lost, nothing else. } finally { busy = false; } diff --git a/packages/compiler/src/frontend/canon.ts b/packages/compiler/src/frontend/canon.ts index d40312388..9066e2e88 100644 --- a/packages/compiler/src/frontend/canon.ts +++ b/packages/compiler/src/frontend/canon.ts @@ -86,9 +86,23 @@ const PRIM_FLAGS: readonly [number, CanonPrim][] = [ [ts.TypeFlags.ESSymbol | ts.TypeFlags.UniqueESSymbol, "symbol"], ]; -/** A stable key for a canonical node, for deduplication and tests. */ -export function canonKey(c: Canon, depth = 0): string { - if (depth > 6) return "…"; +/** A stable key for a canonical node, for deduplication and tests. Deep + * and wide types are cut off (depth 6, about 4,000 characters), so equal + * keys mean equal up to that horizon. */ +export function canonKey(c: Canon): string { + const budget = { left: 4000 }; + return canonKeyAt(c, 0, budget); +} + +function canonKeyAt(c: Canon, depth: number, budget: { left: number }): string { + const key = canonKeyInner(c, depth, budget); + budget.left -= key.length; + return key; +} + +function canonKeyInner(c: Canon, depth: number, budget: { left: number }): string { + if (depth > 6 || budget.left <= 0) return "…"; + const canonKey = (n: Canon, d: number) => canonKeyAt(n, d, budget); switch (c.k) { case "prim": return c.p; From f03741b03e7c5b106497f3129727ef388687b8fe Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Mon, 26 Oct 2026 05:49:39 -0700 Subject: [PATCH 12/23] Keep the branded-primitive rule compilable by the self-hosted frontend Once shadow mode imports the normalizer, the self-hosted frontend lowers it, and it cannot compare union-typed canonical nodes by identity. Co-Authored-By: Claude Opus 5.5 --- packages/compiler/src/frontend/canon.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/compiler/src/frontend/canon.ts b/packages/compiler/src/frontend/canon.ts index 9066e2e88..706ec1e89 100644 --- a/packages/compiler/src/frontend/canon.ts +++ b/packages/compiler/src/frontend/canon.ts @@ -392,7 +392,8 @@ export class Normalizer { return arms.length === 1 ? arms[0]! : { k: "union", arms }; } const prims = parts.filter((p) => p.k === "prim" || p.k === "lit"); - if (prims.length === 1 && parts.every((p) => p === prims[0] || p.k === "obj")) return prims[0]!; + if (prims.length === 1 && parts.every((p) => p.k === "prim" || p.k === "lit" || p.k === "obj")) + return prims[0]!; if (parts.every((p) => p.k === "obj" || (p.k === "top" && p.t === "object"))) return this.object(type, env, key); return { k: "opaque", ts: type, reason: "intersection" }; From 52a228d0f5024ed714002e33ba53eecf9dcd3e5b Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Mon, 26 Oct 2026 07:50:59 -0500 Subject: [PATCH 13/23] Keep declarations a removed branch shares with another --- .../src/frontend/shape-inference/overlay.ts | 23 +++++++++++-------- .../npm/cases/shape-overlay-facts/main.ts | 6 +++-- .../node_modules/shape-facts/index.d.ts | 1 + .../node_modules/shape-facts/index.js | 19 +++++++++++++-- tests/harness/shape-overlay.test.ts | 5 ++-- 5 files changed, 38 insertions(+), 16 deletions(-) diff --git a/packages/compiler/src/frontend/shape-inference/overlay.ts b/packages/compiler/src/frontend/shape-inference/overlay.ts index 7747fac20..edaad7b80 100644 --- a/packages/compiler/src/frontend/shape-inference/overlay.ts +++ b/packages/compiler/src/frontend/shape-inference/overlay.ts @@ -502,15 +502,18 @@ export function planShapeOverlay( // Dead branches: outermost first; edits inside a replaced range drop. const dead = (deadByFile.get(sf) ?? []).sort((a, b) => a.pos - b.pos); const ranges: [number, number][] = []; - const hoisted = new Map(); + const removed = new Set(); for (const branch of dead) { const start = ts.tokenStartOfNode(branch); if (ranges.some(([lo, hi]) => start >= lo && start < hi)) continue; - const names = hoistedNames(branch); - if (names === null) continue; - hoisted.set(start, names); + if (hoistedNames(branch, removed) === null) continue; + removed.add(branch); ranges.push([start, branch.end]); } + // A var that only removed branches declare stays declared in each. + const hoisted = new Map(); + for (const branch of removed) + hoisted.set(ts.tokenStartOfNode(branch), hoistedNames(branch, removed)!); if (ranges.length > 0) { const kept = edits.filter((e) => !ranges.some(([lo, hi]) => e.pos > lo && e.pos < hi)); edits.length = 0; @@ -664,11 +667,11 @@ function neutralizeComment(comment: string): string { } /** The `var` names a dead branch declares that its function (or file) - * declares nowhere else: replacing the branch must keep them bound, as - * undefined. Null when the branch declares a function in a block, whose - * binding outside the block depends on the file's strictness; such a - * branch is kept. */ -function hoistedNames(branch: ts.Node): readonly string[] | null { + * declares nowhere else outside the `removed` branches: replacing the + * branch must keep them bound, as undefined. Null when the branch declares + * a function in a block, whose binding outside the block depends on the + * file's strictness; such a branch is kept. */ +function hoistedNames(branch: ts.Node, removed: ReadonlySet): readonly string[] | null { let scope: ts.Node | undefined = branch.parent; while (scope !== undefined && !ts.isSourceFile(scope) && !functionScope(scope)) scope = scope.parent; @@ -702,7 +705,7 @@ function hoistedNames(branch: ts.Node): readonly string[] | null { const body = ts.isSourceFile(scope) ? scope : (scope as { body?: ts.Node }).body; if (body !== undefined) ts.walkPreorder(body, (n) => { - if (n === branch) return "skip"; + if (n === branch || removed.has(n)) return "skip"; if (n !== body && ts.isFunctionDeclaration(n) && n.name !== undefined) elsewhere.add(n.name.text); if (n !== body && (functionScope(n) || ts.isClassDeclaration(n) || ts.isClassExpression(n))) diff --git a/tests/fixtures/npm/cases/shape-overlay-facts/main.ts b/tests/fixtures/npm/cases/shape-overlay-facts/main.ts index 3ed1cf5de..2c83bcb85 100644 --- a/tests/fixtures/npm/cases/shape-overlay-facts/main.ts +++ b/tests/fixtures/npm/cases/shape-overlay-facts/main.ts @@ -1,6 +1,7 @@ // The shape overlay's facts: results that include undefined, literals that -// hold comment-like text, and a var declared in a branch no call takes. -import { banner, hoisted, lookup, lookupOrReturn } from "shape-facts"; +// hold comment-like text, a var declared in a branch no call takes, and +// a var declared in two such branches. +import { banner, hoisted, lookup, lookupOrReturn, shared } from "shape-facts"; console.log(lookup("b")); console.log(lookup("z")); @@ -8,3 +9,4 @@ console.log(lookupOrReturn("a")); console.log(lookupOrReturn("y")); console.log(banner(7)); console.log(hoisted("note:")); +console.log(shared("level:")); diff --git a/tests/fixtures/npm/cases/shape-overlay-facts/node_modules/shape-facts/index.d.ts b/tests/fixtures/npm/cases/shape-overlay-facts/node_modules/shape-facts/index.d.ts index b1865415d..12ff3f2d8 100644 --- a/tests/fixtures/npm/cases/shape-overlay-facts/node_modules/shape-facts/index.d.ts +++ b/tests/fixtures/npm/cases/shape-overlay-facts/node_modules/shape-facts/index.d.ts @@ -2,3 +2,4 @@ export declare function lookup(key: string): string; export declare function lookupOrReturn(key: string): string; export declare function banner(n: number): string; export declare function hoisted(label: string): string; +export declare function shared(label: string): string; diff --git a/tests/fixtures/npm/cases/shape-overlay-facts/node_modules/shape-facts/index.js b/tests/fixtures/npm/cases/shape-overlay-facts/node_modules/shape-facts/index.js index 505af75a0..8cfa73620 100644 --- a/tests/fixtures/npm/cases/shape-overlay-facts/node_modules/shape-facts/index.js +++ b/tests/fixtures/npm/cases/shape-overlay-facts/node_modules/shape-facts/index.js @@ -1,6 +1,7 @@ // Functions whose results include the undefined of a fall-through or a -// bare return, comment-like text in literals, and a branch no call takes -// that declares a var read outside it. +// bare return, comment-like text in literals, a branch no call takes +// that declares a var read outside it, and two such branches that declare +// the same var. const entries = [ { key: "a", value: 1 }, @@ -53,3 +54,17 @@ function noteOf(extra) { export function hoisted(label) { return label + " " + noteOf(undefined); } + +function levelOf(warn, fail) { + if (warn) { + var level = "warn"; + } + if (fail) { + var level = "fail"; + } + return typeof level + " " + String(level); +} + +export function shared(label) { + return label + " " + levelOf(undefined, undefined); +} diff --git a/tests/harness/shape-overlay.test.ts b/tests/harness/shape-overlay.test.ts index 54bcfc1ce..6c843c7cb 100644 --- a/tests/harness/shape-overlay.test.ts +++ b/tests/harness/shape-overlay.test.ts @@ -63,7 +63,8 @@ test("shapeOverlay: false compiles without the overlay", { timeout: 900_000 }, a // shape-facts: results that include the undefined of a fall-through or a bare // return keep the caller's falsy branch, comment-like text in literals keeps -// its value, and a replaced branch keeps its var declared. +// its value, and a replaced branch keeps its var declared, also when two +// replaced branches declare the same var. const factsEntry = join(repoRoot, "tests/fixtures/npm/cases/shape-overlay-facts/main.ts"); test( @@ -78,7 +79,7 @@ test( throw new Error(result.diagnostics.map((d) => `${d.code}: ${d.message}`).join("\n")); const report = (result.warnings ?? []).find((w) => w.code === "SC6103"); expect(report?.message).toMatch(/of 1 package \(shape-facts\)/); - expect(report?.message).toMatch(/replaces 1 branch it proved unreachable/); + expect(report?.message).toMatch(/replaces 3 branches it proved unreachable/); const native = await execFileAsync(result.binaryPath, [], { cwd: dirname(factsEntry), encoding: "utf8", From f1298287d60878ac10f12aac3577bd58b6e9993b Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Mon, 26 Oct 2026 07:58:58 -0500 Subject: [PATCH 14/23] Resolve each shipped declaration path once isFallbackDtsPath runs for every type lowering maps (isStdlibFile), and since the fallback declarations moved into per-module files (#2256) it asked declarationPath for the fallback module directory each time. Without an installed declaration root, each answer resolved @scriptc/compiler/scriptc-node-fallback.d.ts through Node's module resolver. Lowering three.js's buffers-animation fixture went from 6.3 s to 43 s. declarationPath now keeps its answers by name until the declaration root changes. --- packages/compiler/src/frontend/dts-paths.ts | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/packages/compiler/src/frontend/dts-paths.ts b/packages/compiler/src/frontend/dts-paths.ts index 638e58641..7e53a2a9b 100644 --- a/packages/compiler/src/frontend/dts-paths.ts +++ b/packages/compiler/src/frontend/dts-paths.ts @@ -7,13 +7,27 @@ export { tsgoPath } from "./ts7/session-path.js"; const require = createRequire(import.meta.url); let installedDeclarationRoot: string | null = null; +/** The paths `declarationPath` answered, by name, for the current root. + * Lowering asks for them once per mapped type (isFallbackDtsPath), and each + * answer without an installed root costs a package resolution. */ +const declarationPaths = new Map(); /** A native distribution supplies its relocatable declaration directory. */ export function setDeclarationRoot(root: string): void { installedDeclarationRoot = root; + declarationPaths.clear(); } function declarationPath(name: string): string { + let path = declarationPaths.get(name); + if (path === undefined) { + path = resolveDeclarationPath(name); + declarationPaths.set(name, path); + } + return path; +} + +function resolveDeclarationPath(name: string): string { return tsgoPath( installedDeclarationRoot === null ? name === "scriptc.d.ts" From f9b5533ee646d2b720a8e43bdfe199b51d038da5 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Mon, 26 Oct 2026 08:13:15 -0500 Subject: [PATCH 15/23] Read IR nodes' fields by name in three llvm-emit passes --- .github/workflows/ci.yml | 2 + package.json | 1 + .../compiler/src/backend/llvm/unwind-abi.ts | 20 +- packages/compiler/src/ir/char-unit-locals.ts | 60 +- packages/compiler/src/ir/fields.generated.ts | 2067 +++++++++++++++++ packages/compiler/src/ir/fields.test.ts | 111 + .../compiler/src/ir/paired-while-loops.ts | 33 +- scripts/generate-ir-fields.mjs | 215 ++ 8 files changed, 2453 insertions(+), 56 deletions(-) create mode 100644 packages/compiler/src/ir/fields.generated.ts create mode 100644 packages/compiler/src/ir/fields.test.ts create mode 100644 scripts/generate-ir-fields.mjs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2b690b100..f9b725472 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -122,6 +122,8 @@ jobs: SCRIPTC_TEST_WORKERS: "1" - name: Verify mixed-source generated AST getters run: pnpm generate:ts7-ast --check + - name: Verify the generated IR field readers + run: pnpm generate:ir-fields --check native_macos: needs: [validation_scope, lanes] diff --git a/package.json b/package.json index a3cc42fc7..9b0213b44 100644 --- a/package.json +++ b/package.json @@ -16,6 +16,7 @@ "test:ts7": "node scripts/test-ts7.mjs", "test:checker-identity": "node scripts/checker-identity.mjs", "generate:ts7-ast": "node scripts/generate-ts7-ast-schema.mjs", + "generate:ir-fields": "node scripts/generate-ir-fields.mjs", "test:fetch-conformance": "vitest run tests/harness/fetch-conformance.test.ts", "test:test262": "tsx tests/test262/run.ts", "test:test262:fetch": "node tests/test262/fetch.mjs", diff --git a/packages/compiler/src/backend/llvm/unwind-abi.ts b/packages/compiler/src/backend/llvm/unwind-abi.ts index 91722fb1d..821c5351b 100644 --- a/packages/compiler/src/backend/llvm/unwind-abi.ts +++ b/packages/compiler/src/backend/llvm/unwind-abi.ts @@ -2,6 +2,7 @@ * pending flag (see unwindAbiFunctions). */ import type { IrExpr, IrModule } from "../../ir/ir.js"; import { everyStmtList } from "../../ir/traverse.js"; +import { eachStringField } from "../../ir/fields.generated.js"; /** How many calls above a `throw` functions keep the pending flag. */ const THROW_LEVELS = 3; @@ -129,13 +130,14 @@ export function unwindAbiFunctions( } // Any function a node names other than as a call's callee may be reached // through a pointer. + // Every string field counts (eachStringField), as a key walk of the node + // would find them; a call's callee does not. const referenced = new Set(); - const noteNames = (node: object, skip: string | undefined): void => { - for (const key in node) { - if (key === skip) continue; - const value = (node as Record)[key]; - if (typeof value === "string" && names.has(value)) referenced.add(value); - } + const noteName = (value: string): void => { + if (names.has(value)) referenced.add(value); + }; + const noteCallName = (value: string, field: string): void => { + if (field !== "callee") noteName(value); }; // The call graph, for the functions near a throw and the recursive ones. // Calls through values (closures, class values, dynamic calls, callbacks @@ -171,10 +173,10 @@ export function unwindAbiFunctions( // flag they keep landing pads out of the top-level code inlined // into the entry. if (coroutine || fn.name === mod.entry) blocked.add(e.callee); - noteNames(e, "callee"); + eachStringField(e, noteCallName); return true; } - noteNames(e, undefined); + eachStringField(e, noteName); if (e.kind === "virtualCall") for (const target of e.targets ?? methodsNamed.get(e.method) ?? []) edge(fn.name, target); else if (e.kind === "new") { @@ -193,7 +195,7 @@ export function unwindAbiFunctions( stmt: (s) => { const kind = s.kind as string; if (kind === "throw" || kind === "rethrow") throws.add(fn.name); - noteNames(s, undefined); + eachStringField(s, noteName); return true; }, }); diff --git a/packages/compiler/src/ir/char-unit-locals.ts b/packages/compiler/src/ir/char-unit-locals.ts index fc1d5873b..be1eae2c3 100644 --- a/packages/compiler/src/ir/char-unit-locals.ts +++ b/packages/compiler/src/ir/char-unit-locals.ts @@ -8,6 +8,7 @@ import { type IrStmt, } from "./ir.js"; import { mapExprChildren, mapStmtChildren } from "./traverse.js"; +import { type IrFieldWalker, walkStmtFields } from "./fields.generated.js"; type CharAt = IrExpr & { kind: "strIntrinsic" }; type UnitUse = IrExpr & { kind: "strEq" | "strCmp" }; @@ -302,26 +303,24 @@ function rewriteFunction(fn: IrFunction): IrFunction { const copyOf = new Map(); // Whether a candidate string is declared from a charAt. let charAtDeclared = false; - const collect = (value: unknown): void => { - if (Array.isArray(value)) for (const item of value) collect(item); - else if (value !== null && typeof value === "object") { - const node = value as Record; - if (node.kind === "varDecl" && candidates.has(node.localId as string) && node.init) { - const id = node.localId as string; - const init = node.init as IrExpr; - if (locals.get(id)?.type.kind === "string" && isCharAt(init)) charAtDeclared = true; - const read = optionalCharRead(init); - if (read) reads.set(id, read); - else if (init.kind === "varRef" && reads.has(init.localId)) { - reads.set(id, reads.get(init.localId)!); - copyOf.set(id, init.localId); - } + // The declarations, in the order a walk of every field finds them. + const collector: IrFieldWalker = { + node: (node) => { + if (node.kind !== "varDecl" || !candidates.has(node.localId) || !node.init) return; + const id = node.localId; + const init = node.init; + if (locals.get(id)?.type.kind === "string" && isCharAt(init)) charAtDeclared = true; + const read = optionalCharRead(init); + if (read) reads.set(id, read); + else if (init.kind === "varRef" && reads.has(init.localId)) { + reads.set(id, reads.get(init.localId)!); + copyOf.set(id, init.localId); } - for (const key in node) if (key !== "loc") collect(node[key]); - } + }, + string: () => {}, }; // Both walks only record candidates: a function without one holds none. - if (candidates.size > 0) collect(fn.body); + if (candidates.size > 0) for (const stmt of fn.body) walkStmtFields(stmt, collector, false); const localTags = (operand: IrExpr): OptionalCharRead | null => operand.kind === "varRef" ? (reads.get(operand.localId) ?? null) : null; @@ -333,24 +332,22 @@ function rewriteFunction(fn: IrFunction): IrFunction { const note = (map: Map, id: string): void => { map.set(id, (map.get(id) ?? 0) + 1); }; - const count = (value: unknown): void => { - if (typeof value === "string") { - if (candidates.has(value)) note(mentions, value); - } else if (Array.isArray(value)) { - for (const item of value) count(item); - } else if (value !== null && typeof value === "object") { - const node = value as Record; + const counter: IrFieldWalker = { + node: (node) => { const id = understoodMention(node, locals, reads, localTags); if (id !== null && candidates.has(id)) { note(understood, id); if (node.kind === "varDecl") note(declarations, id); } - for (const key in node) if (key !== "loc") count(node[key]); - } + }, + string: (value) => { + if (candidates.has(value)) note(mentions, value); + }, }; // Only candidates declared from a charAt or an optional read can be held - // (understoodMention), and count() only decides which are. - if (reads.size > 0 || charAtDeclared) count(fn.body); + // (understoodMention), and the count only decides which are. + if (reads.size > 0 || charAtDeclared) + for (const stmt of fn.body) walkStmtFields(stmt, counter, false); // A copy is held only with its source, and the source's mention in the // copy's initializer is understood only while the copy is held. const held = new Set(); @@ -492,14 +489,12 @@ function unitLocal(local: IrLocal): IrLocal { * over one-unit cases; a union local's declaration from an optional read, * or its test against a one-unit literal or for undefined. */ function understoodMention( - node: Record, + n: IrExpr | IrStmt, locals: ReadonlyMap, reads: ReadonlyMap, localTags: (operand: IrExpr) => OptionalCharRead | null, ): string | null { - // count() visits every object in the body (types, locations, cases), so - // only the node kinds handled here are read as IR nodes - const kind = node.kind; + const kind = n.kind; if ( kind !== "varDecl" && kind !== "strEq" && @@ -510,7 +505,6 @@ function understoodMention( kind !== "unionIsTag" ) return null; - const n = node as unknown as IrExpr | IrStmt; if (n.kind === "varDecl") { if (!n.init) return null; const local = locals.get(n.localId); diff --git a/packages/compiler/src/ir/fields.generated.ts b/packages/compiler/src/ir/fields.generated.ts new file mode 100644 index 000000000..a46259b6e --- /dev/null +++ b/packages/compiler/src/ir/fields.generated.ts @@ -0,0 +1,2067 @@ +// Generated from packages/compiler/src/ir/ir.ts by scripts/generate-ir-fields.mjs. +// Run `pnpm generate:ir-fields` after changing the fields of an IR node. +import type { IrExpr, IrStmt, IrType } from "./ir.js"; + +/** Calls `visit` with each string field of `node` (including `kind`) and its + * name: the string values `for (const key in node)` finds on a node, read + * by name in the order ir.ts declares them. */ +export function eachStringField( + node: IrExpr | IrStmt, + visit: (value: string, field: string) => void, +): void { + switch (node.kind) { + case "arrIntrinsic": { + visit(node.kind, "kind"); + visit(node.method, "method"); + return; + } + case "arrayDelete": { + visit(node.kind, "kind"); + return; + } + case "arrayGet": { + visit(node.kind, "kind"); + return; + } + case "arrayHas": { + visit(node.kind, "kind"); + return; + } + case "arrayLit": { + visit(node.kind, "kind"); + return; + } + case "arrayNewLen": { + visit(node.kind, "kind"); + return; + } + case "arraySet": { + visit(node.kind, "kind"); + return; + } + case "arraySetLength": { + visit(node.kind, "kind"); + return; + } + case "arraySetUndefined": { + visit(node.kind, "kind"); + return; + } + case "arrayState": { + visit(node.kind, "kind"); + return; + } + case "assign": { + visit(node.kind, "kind"); + visit(node.localId, "localId"); + return; + } + case "assignExpr": { + visit(node.kind, "kind"); + visit(node.localId, "localId"); + return; + } + case "awaitExpr": { + visit(node.kind, "kind"); + return; + } + case "awaitUnionExpr": { + visit(node.kind, "kind"); + return; + } + case "bin": { + visit(node.kind, "kind"); + visit(node.op, "op"); + return; + } + case "block": { + visit(node.kind, "kind"); + return; + } + case "boolLit": { + visit(node.kind, "kind"); + return; + } + case "break": { + visit(node.kind, "kind"); + if (typeof node.label === "string") visit(node.label, "label"); + return; + } + case "bytesIntrinsic": { + visit(node.kind, "kind"); + visit(node.method, "method"); + return; + } + case "bytesNew": { + visit(node.kind, "kind"); + return; + } + case "bytesSet": { + visit(node.kind, "kind"); + return; + } + case "call": { + visit(node.kind, "kind"); + visit(node.callee, "callee"); + return; + } + case "callValue": { + visit(node.kind, "kind"); + return; + } + case "caughtCheck": { + visit(node.kind, "kind"); + visit(node.className, "className"); + return; + } + case "caughtNarrow": { + visit(node.kind, "kind"); + return; + } + case "caughtTest": { + visit(node.kind, "kind"); + visit(node.test, "test"); + if (typeof node.className === "string") visit(node.className, "className"); + return; + } + case "caughtToDyn": { + visit(node.kind, "kind"); + return; + } + case "chainRecv": { + visit(node.kind, "kind"); + visit(node.id, "id"); + return; + } + case "classRef": { + visit(node.kind, "kind"); + visit(node.className, "className"); + return; + } + case "closure": { + visit(node.kind, "kind"); + visit(node.fnName, "fnName"); + return; + } + case "continue": { + visit(node.kind, "kind"); + if (typeof node.label === "string") visit(node.label, "label"); + return; + } + case "doWhile": { + visit(node.kind, "kind"); + return; + } + case "downcast": { + visit(node.kind, "kind"); + return; + } + case "dynArrLit": { + visit(node.kind, "kind"); + return; + } + case "dynCall": { + visit(node.kind, "kind"); + visit(node.calleeName, "calleeName"); + return; + } + case "dynCheck": { + visit(node.kind, "kind"); + return; + } + case "dynDestrCheck": { + visit(node.kind, "kind"); + visit(node.spelling, "spelling"); + if (typeof node.firstProp === "string") visit(node.firstProp, "firstProp"); + return; + } + case "dynFrom": { + visit(node.kind, "kind"); + if (typeof node.fnName === "string") visit(node.fnName, "fnName"); + return; + } + case "dynHasKey": { + visit(node.kind, "kind"); + visit(node.key, "key"); + return; + } + case "dynInvoke": { + visit(node.kind, "kind"); + visit(node.method, "method"); + visit(node.calleeName, "calleeName"); + return; + } + case "dynIterN": { + visit(node.kind, "kind"); + if (typeof node.notIterableMessage === "string") visit(node.notIterableMessage, "notIterableMessage"); + return; + } + case "dynKeyGet": { + visit(node.kind, "kind"); + return; + } + case "dynObjLit": { + visit(node.kind, "kind"); + return; + } + case "dynScalarEq": { + visit(node.kind, "kind"); + return; + } + case "dynTest": { + visit(node.kind, "kind"); + visit(node.test, "test"); + if (typeof node.bytesElem === "string") visit(node.bytesElem, "bytesElem"); + return; + } + case "exprStmt": { + visit(node.kind, "kind"); + return; + } + case "ffiCall": { + visit(node.kind, "kind"); + visit(node.import, "import"); + return; + } + case "fieldAbsent": { + visit(node.kind, "kind"); + visit(node.unionId, "unionId"); + return; + } + case "fieldGet": { + visit(node.kind, "kind"); + visit(node.className, "className"); + visit(node.field, "field"); + return; + } + case "fieldIncDec": { + visit(node.kind, "kind"); + visit(node.op, "op"); + visit(node.className, "className"); + visit(node.field, "field"); + return; + } + case "fieldSet": { + visit(node.kind, "kind"); + visit(node.className, "className"); + visit(node.field, "field"); + return; + } + case "for": { + visit(node.kind, "kind"); + return; + } + case "forOf": { + visit(node.kind, "kind"); + visit(node.localId, "localId"); + return; + } + case "genResume": { + visit(node.kind, "kind"); + visit(node.mode, "mode"); + return; + } + case "if": { + visit(node.kind, "kind"); + return; + } + case "incDec": { + visit(node.kind, "kind"); + visit(node.op, "op"); + visit(node.localId, "localId"); + return; + } + case "instanceOf": { + visit(node.kind, "kind"); + visit(node.className, "className"); + return; + } + case "instanceOfValue": { + visit(node.kind, "kind"); + return; + } + case "intrinsic": { + visit(node.kind, "kind"); + visit(node.name, "name"); + return; + } + case "jsonStringify": { + visit(node.kind, "kind"); + if (typeof node.indent === "string") visit(node.indent, "indent"); + return; + } + case "libCall": { + visit(node.kind, "kind"); + visit(node.fn, "fn"); + return; + } + case "logical": { + visit(node.kind, "kind"); + visit(node.op, "op"); + return; + } + case "mapIntrinsic": { + visit(node.kind, "kind"); + visit(node.method, "method"); + return; + } + case "mapNew": { + visit(node.kind, "kind"); + return; + } + case "moduleNsRef": { + visit(node.kind, "kind"); + visit(node.moduleId, "moduleId"); + return; + } + case "new": { + visit(node.kind, "kind"); + visit(node.className, "className"); + return; + } + case "newPromise": { + visit(node.kind, "kind"); + return; + } + case "newValue": { + visit(node.kind, "kind"); + return; + } + case "nullish": { + visit(node.kind, "kind"); + return; + } + case "numLit": { + visit(node.kind, "kind"); + if (typeof node.spelling === "string") visit(node.spelling, "spelling"); + return; + } + case "optChain": { + visit(node.kind, "kind"); + visit(node.id, "id"); + return; + } + case "orDefault": { + visit(node.kind, "kind"); + if (typeof node.retag === "string") visit(node.retag, "retag"); + return; + } + case "promiseVoidWiden": { + visit(node.kind, "kind"); + return; + } + case "promiseWithResolvers": { + visit(node.kind, "kind"); + return; + } + case "recordClone": { + visit(node.kind, "kind"); + return; + } + case "recordGet": { + visit(node.kind, "kind"); + visit(node.shapeId, "shapeId"); + visit(node.field, "field"); + return; + } + case "recordHas": { + visit(node.kind, "kind"); + visit(node.shapeId, "shapeId"); + visit(node.field, "field"); + return; + } + case "recordKeyDelete": { + visit(node.kind, "kind"); + visit(node.shapeId, "shapeId"); + return; + } + case "recordKeyGet": { + visit(node.kind, "kind"); + visit(node.shapeId, "shapeId"); + return; + } + case "recordKeySet": { + visit(node.kind, "kind"); + visit(node.shapeId, "shapeId"); + return; + } + case "recordLit": { + visit(node.kind, "kind"); + return; + } + case "recordNullProto": { + visit(node.kind, "kind"); + visit(node.shapeId, "shapeId"); + return; + } + case "recordOvfHas": { + visit(node.kind, "kind"); + visit(node.shapeId, "shapeId"); + return; + } + case "recordOvfKeys": { + visit(node.kind, "kind"); + visit(node.shapeId, "shapeId"); + return; + } + case "recordSet": { + visit(node.kind, "kind"); + visit(node.shapeId, "shapeId"); + visit(node.field, "field"); + return; + } + case "regexIntrinsic": { + visit(node.kind, "kind"); + visit(node.method, "method"); + return; + } + case "regexLit": { + visit(node.kind, "kind"); + visit(node.pattern, "pattern"); + visit(node.flags, "flags"); + return; + } + case "rethrow": { + visit(node.kind, "kind"); + visit(node.localId, "localId"); + return; + } + case "return": { + visit(node.kind, "kind"); + return; + } + case "runtimeFence": { + visit(node.kind, "kind"); + visit(node.code, "code"); + visit(node.message, "message"); + return; + } + case "selfRef": { + visit(node.kind, "kind"); + return; + } + case "seqExpr": { + visit(node.kind, "kind"); + return; + } + case "setIntrinsic": { + visit(node.kind, "kind"); + visit(node.method, "method"); + return; + } + case "setNew": { + visit(node.kind, "kind"); + return; + } + case "strCmp": { + visit(node.kind, "kind"); + visit(node.op, "op"); + return; + } + case "strConcat": { + visit(node.kind, "kind"); + return; + } + case "strEq": { + visit(node.kind, "kind"); + return; + } + case "strIntrinsic": { + visit(node.kind, "kind"); + visit(node.method, "method"); + return; + } + case "strLit": { + visit(node.kind, "kind"); + visit(node.value, "value"); + return; + } + case "switch": { + visit(node.kind, "kind"); + return; + } + case "templateStrings": { + visit(node.kind, "kind"); + visit(node.key, "key"); + return; + } + case "ternary": { + visit(node.kind, "kind"); + return; + } + case "throw": { + visit(node.kind, "kind"); + return; + } + case "toBool": { + visit(node.kind, "kind"); + return; + } + case "toString": { + visit(node.kind, "kind"); + return; + } + case "tryCatch": { + visit(node.kind, "kind"); + if (typeof node.catchLocalId === "string") visit(node.catchLocalId, "catchLocalId"); + return; + } + case "unary": { + visit(node.kind, "kind"); + visit(node.op, "op"); + return; + } + case "unionDisc": { + visit(node.kind, "kind"); + visit(node.unionId, "unionId"); + visit(node.field, "field"); + return; + } + case "unionEq": { + visit(node.kind, "kind"); + visit(node.unionId, "unionId"); + return; + } + case "unionFuncEq": { + visit(node.kind, "kind"); + visit(node.unionId, "unionId"); + return; + } + case "unionIsTag": { + visit(node.kind, "kind"); + visit(node.unionId, "unionId"); + return; + } + case "unionKeyGet": { + visit(node.kind, "kind"); + visit(node.unionId, "unionId"); + return; + } + case "unionNarrow": { + visit(node.kind, "kind"); + visit(node.unionId, "unionId"); + return; + } + case "unionRetype": { + visit(node.kind, "kind"); + visit(node.fromId, "fromId"); + visit(node.unionId, "unionId"); + return; + } + case "unionWrap": { + visit(node.kind, "kind"); + visit(node.unionId, "unionId"); + return; + } + case "unitLit": { + visit(node.kind, "kind"); + visit(node.unit, "unit"); + return; + } + case "upcast": { + visit(node.kind, "kind"); + return; + } + case "varDecl": { + visit(node.kind, "kind"); + visit(node.localId, "localId"); + return; + } + case "varRef": { + visit(node.kind, "kind"); + visit(node.localId, "localId"); + return; + } + case "virtualCall": { + visit(node.kind, "kind"); + visit(node.className, "className"); + visit(node.method, "method"); + return; + } + case "while": { + visit(node.kind, "kind"); + return; + } + case "yieldExpr": { + visit(node.kind, "kind"); + return; + } + } +} + +/** What a field walk reports (walkExprFields): each expression and + * statement node before its fields, and each string a field holds. */ +export interface IrFieldWalker { + node(node: IrExpr | IrStmt): void; + string(value: string): void; +} + +/** Walks every field of `node` but `loc` (and `type` with `skipType`), and + * every field of each object under it: the strings and nodes a recursive + * `for (const key in value)` walk with the same skips finds, read by name in + * the order ir.ts declares the fields. */ +export function walkExprFields(node: IrExpr, walker: IrFieldWalker, skipType: boolean): void { + walker.node(node); + switch (node.kind) { + case "arrIntrinsic": { + walker.string(node.kind); + walker.string(node.method); + walkExprFields(node.receiver, walker, skipType); + for (const item0 of node.args) { + walkExprFields(item0, walker, skipType); + } + if (!skipType) { + walkTypeFields(node.type, walker, skipType); + } + return; + } + case "arrayGet": { + walker.string(node.kind); + walkExprFields(node.arr, walker, skipType); + walkExprFields(node.index, walker, skipType); + if (!skipType) { + walkTypeFields(node.type, walker, skipType); + } + return; + } + case "arrayHas": { + walker.string(node.kind); + walkExprFields(node.arr, walker, skipType); + walkExprFields(node.index, walker, skipType); + if (!skipType) { + walkTypeFields(node.type, walker, skipType); + } + return; + } + case "arrayLit": { + walker.string(node.kind); + for (const item1 of node.elems) { + walkExprFields(item1, walker, skipType); + } + if (node.spreads !== undefined && node.spreads !== null) { + for (const item2 of node.spreads) { + } + } + if (!skipType) { + walkTypeFields(node.type, walker, skipType); + } + return; + } + case "arrayNewLen": { + walker.string(node.kind); + walkExprFields(node.length, walker, skipType); + if (!skipType) { + walkTypeFields(node.type, walker, skipType); + } + return; + } + case "arrayState": { + walker.string(node.kind); + walkExprFields(node.arr, walker, skipType); + walkExprFields(node.index, walker, skipType); + if (!skipType) { + walkTypeFields(node.type, walker, skipType); + } + return; + } + case "assignExpr": { + walker.string(node.kind); + walker.string(node.localId); + walkExprFields(node.value, walker, skipType); + if (!skipType) { + walkTypeFields(node.type, walker, skipType); + } + return; + } + case "awaitExpr": { + walker.string(node.kind); + walkExprFields(node.value, walker, skipType); + if (!skipType) { + walkTypeFields(node.type, walker, skipType); + } + return; + } + case "awaitUnionExpr": { + walker.string(node.kind); + walkExprFields(node.value, walker, skipType); + if (!skipType) { + walkTypeFields(node.type, walker, skipType); + } + return; + } + case "bin": { + walker.string(node.kind); + walker.string(node.op); + walkExprFields(node.left, walker, skipType); + walkExprFields(node.right, walker, skipType); + if (!skipType) { + walkTypeFields(node.type, walker, skipType); + } + return; + } + case "boolLit": { + walker.string(node.kind); + if (!skipType) { + walkTypeFields(node.type, walker, skipType); + } + return; + } + case "bytesIntrinsic": { + walker.string(node.kind); + walker.string(node.method); + walkExprFields(node.receiver, walker, skipType); + for (const item3 of node.args) { + walkExprFields(item3, walker, skipType); + } + if (!skipType) { + walkTypeFields(node.type, walker, skipType); + } + return; + } + case "bytesNew": { + walker.string(node.kind); + if (node.source !== undefined && node.source !== null) { + walkExprFields(node.source, walker, skipType); + } + if (!skipType) { + walkTypeFields(node.type, walker, skipType); + } + return; + } + case "call": { + walker.string(node.kind); + walker.string(node.callee); + for (const item4 of node.args) { + walkExprFields(item4, walker, skipType); + } + if (!skipType) { + walkTypeFields(node.type, walker, skipType); + } + return; + } + case "callValue": { + walker.string(node.kind); + walkExprFields(node.callee, walker, skipType); + if (node.receiver !== undefined && node.receiver !== null) { + walkExprFields(node.receiver, walker, skipType); + } + for (const item5 of node.args) { + walkExprFields(item5, walker, skipType); + } + if (!skipType) { + walkTypeFields(node.type, walker, skipType); + } + return; + } + case "caughtCheck": { + walker.string(node.kind); + walkExprFields(node.value, walker, skipType); + walker.string(node.className); + if (!skipType) { + walkTypeFields(node.type, walker, skipType); + } + return; + } + case "caughtNarrow": { + walker.string(node.kind); + walkExprFields(node.value, walker, skipType); + if (!skipType) { + walkTypeFields(node.type, walker, skipType); + } + return; + } + case "caughtTest": { + walker.string(node.kind); + walkExprFields(node.value, walker, skipType); + walker.string(node.test); + if (node.className !== undefined && node.className !== null) { + walker.string(node.className); + } + if (!skipType) { + walkTypeFields(node.type, walker, skipType); + } + return; + } + case "caughtToDyn": { + walker.string(node.kind); + walkExprFields(node.value, walker, skipType); + if (!skipType) { + walkTypeFields(node.type, walker, skipType); + } + return; + } + case "chainRecv": { + walker.string(node.kind); + walker.string(node.id); + if (!skipType) { + walkTypeFields(node.type, walker, skipType); + } + return; + } + case "classRef": { + walker.string(node.kind); + walker.string(node.className); + if (node.captures !== undefined && node.captures !== null) { + for (const item6 of node.captures) { + walker.string(item6); + } + } + if (!skipType) { + walkTypeFields(node.type, walker, skipType); + } + return; + } + case "closure": { + walker.string(node.kind); + walker.string(node.fnName); + for (const item7 of node.captures) { + walker.string(item7); + } + if (!skipType) { + walkTypeFields(node.type, walker, skipType); + } + return; + } + case "downcast": { + walker.string(node.kind); + walkExprFields(node.value, walker, skipType); + if (!skipType) { + walkTypeFields(node.type, walker, skipType); + } + return; + } + case "dynArrLit": { + walker.string(node.kind); + for (const item8 of node.elems) { + walkExprFields(item8, walker, skipType); + } + if (!skipType) { + walkTypeFields(node.type, walker, skipType); + } + return; + } + case "dynCall": { + walker.string(node.kind); + walkExprFields(node.callee, walker, skipType); + if (node.receiver !== undefined && node.receiver !== null) { + walkExprFields(node.receiver, walker, skipType); + } + walker.string(node.calleeName); + if (node.calleeNameValue !== undefined && node.calleeNameValue !== null) { + walkExprFields(node.calleeNameValue, walker, skipType); + } + for (const item9 of node.args) { + walkExprFields(item9, walker, skipType); + } + if (node.spreads !== undefined && node.spreads !== null) { + for (const item10 of node.spreads) { + walker.string(item10.what); + } + } + if (!skipType) { + walkTypeFields(node.type, walker, skipType); + } + return; + } + case "dynCheck": { + walker.string(node.kind); + walkExprFields(node.value, walker, skipType); + if (!skipType) { + walkTypeFields(node.type, walker, skipType); + } + return; + } + case "dynDestrCheck": { + walker.string(node.kind); + walkExprFields(node.value, walker, skipType); + walker.string(node.spelling); + if (node.firstProp !== undefined && node.firstProp !== null) { + walker.string(node.firstProp); + } + if (!skipType) { + walkTypeFields(node.type, walker, skipType); + } + return; + } + case "dynFrom": { + walker.string(node.kind); + walkExprFields(node.value, walker, skipType); + if (node.fnName !== undefined && node.fnName !== null) { + walker.string(node.fnName); + } + if (!skipType) { + walkTypeFields(node.type, walker, skipType); + } + return; + } + case "dynHasKey": { + walker.string(node.kind); + walker.string(node.key); + walkExprFields(node.value, walker, skipType); + if (!skipType) { + walkTypeFields(node.type, walker, skipType); + } + return; + } + case "dynInvoke": { + walker.string(node.kind); + walkExprFields(node.recv, walker, skipType); + walker.string(node.method); + walker.string(node.calleeName); + if (node.calleeNameValue !== undefined && node.calleeNameValue !== null) { + walkExprFields(node.calleeNameValue, walker, skipType); + } + for (const item11 of node.args) { + walkExprFields(item11, walker, skipType); + } + if (!skipType) { + walkTypeFields(node.type, walker, skipType); + } + return; + } + case "dynIterN": { + walker.string(node.kind); + walkExprFields(node.value, walker, skipType); + if (node.notIterableMessage !== undefined && node.notIterableMessage !== null) { + walker.string(node.notIterableMessage); + } + if (!skipType) { + walkTypeFields(node.type, walker, skipType); + } + return; + } + case "dynKeyGet": { + walker.string(node.kind); + walkExprFields(node.key, walker, skipType); + walkExprFields(node.value, walker, skipType); + if (!skipType) { + walkTypeFields(node.type, walker, skipType); + } + return; + } + case "dynObjLit": { + walker.string(node.kind); + if (node.fields !== undefined && node.fields !== null) { + for (const item12 of node.fields) { + walkExprFields(item12.key, walker, skipType); + walkExprFields(item12.value, walker, skipType); + } + } + if (!skipType) { + walkTypeFields(node.type, walker, skipType); + } + return; + } + case "dynScalarEq": { + walker.string(node.kind); + walkExprFields(node.left, walker, skipType); + walkExprFields(node.right, walker, skipType); + if (!skipType) { + walkTypeFields(node.type, walker, skipType); + } + return; + } + case "dynTest": { + walker.string(node.kind); + walker.string(node.test); + if (node.bytesElem !== undefined && node.bytesElem !== null) { + walker.string(node.bytesElem); + } + walkExprFields(node.value, walker, skipType); + if (!skipType) { + walkTypeFields(node.type, walker, skipType); + } + return; + } + case "ffiCall": { + walker.string(node.kind); + walker.string(node.import); + for (const item13 of node.args) { + walkExprFields(item13, walker, skipType); + } + if (!skipType) { + walkTypeFields(node.type, walker, skipType); + } + return; + } + case "fieldAbsent": { + walker.string(node.kind); + walker.string(node.unionId); + if (!skipType) { + walkTypeFields(node.type, walker, skipType); + } + return; + } + case "fieldGet": { + walker.string(node.kind); + walkExprFields(node.obj, walker, skipType); + walker.string(node.className); + walker.string(node.field); + if (!skipType) { + walkTypeFields(node.type, walker, skipType); + } + return; + } + case "fieldIncDec": { + walker.string(node.kind); + walker.string(node.op); + walkExprFields(node.obj, walker, skipType); + walker.string(node.className); + walker.string(node.field); + if (!skipType) { + walkTypeFields(node.type, walker, skipType); + } + return; + } + case "genResume": { + walker.string(node.kind); + walker.string(node.mode); + walkExprFields(node.gen, walker, skipType); + if (node.arg !== undefined && node.arg !== null) { + walkExprFields(node.arg, walker, skipType); + } + if (!skipType) { + walkTypeFields(node.type, walker, skipType); + } + return; + } + case "incDec": { + walker.string(node.kind); + walker.string(node.op); + walker.string(node.localId); + if (!skipType) { + walkTypeFields(node.type, walker, skipType); + } + return; + } + case "instanceOf": { + walker.string(node.kind); + walkExprFields(node.value, walker, skipType); + walker.string(node.className); + if (!skipType) { + walkTypeFields(node.type, walker, skipType); + } + return; + } + case "instanceOfValue": { + walker.string(node.kind); + walkExprFields(node.value, walker, skipType); + walkExprFields(node.classValue, walker, skipType); + if (!skipType) { + walkTypeFields(node.type, walker, skipType); + } + return; + } + case "intrinsic": { + walker.string(node.kind); + walker.string(node.name); + for (const item14 of node.args) { + walkExprFields(item14, walker, skipType); + } + if (!skipType) { + walkTypeFields(node.type, walker, skipType); + } + return; + } + case "jsonStringify": { + walker.string(node.kind); + walkExprFields(node.value, walker, skipType); + if (node.indent !== undefined && node.indent !== null) { + walker.string(node.indent); + } + if (!skipType) { + walkTypeFields(node.type, walker, skipType); + } + return; + } + case "libCall": { + walker.string(node.kind); + walker.string(node.fn); + for (const item15 of node.args) { + walkExprFields(item15, walker, skipType); + } + if (!skipType) { + walkTypeFields(node.type, walker, skipType); + } + if (node.assertOptions !== undefined && node.assertOptions !== null) { + if (node.assertOptions.diff !== undefined && node.assertOptions.diff !== null) { + walker.string(node.assertOptions.diff); + } + } + return; + } + case "logical": { + walker.string(node.kind); + walker.string(node.op); + walkExprFields(node.left, walker, skipType); + walkExprFields(node.right, walker, skipType); + if (!skipType) { + walkTypeFields(node.type, walker, skipType); + } + return; + } + case "mapIntrinsic": { + walker.string(node.kind); + walker.string(node.method); + walkExprFields(node.receiver, walker, skipType); + for (const item16 of node.args) { + walkExprFields(item16, walker, skipType); + } + if (!skipType) { + walkTypeFields(node.type, walker, skipType); + } + return; + } + case "mapNew": { + walker.string(node.kind); + if (node.seed !== undefined && node.seed !== null) { + for (const item17 of node.seed) { + walkExprFields(item17.key, walker, skipType); + walkExprFields(item17.value, walker, skipType); + } + } + if (!skipType) { + walkTypeFields(node.type, walker, skipType); + } + return; + } + case "moduleNsRef": { + walker.string(node.kind); + walker.string(node.moduleId); + if (!skipType) { + walkTypeFields(node.type, walker, skipType); + } + return; + } + case "new": { + walker.string(node.kind); + walker.string(node.className); + for (const item18 of node.args) { + walkExprFields(item18, walker, skipType); + } + if (!skipType) { + walkTypeFields(node.type, walker, skipType); + } + return; + } + case "newPromise": { + walker.string(node.kind); + walkExprFields(node.executor, walker, skipType); + if (!skipType) { + walkTypeFields(node.type, walker, skipType); + } + return; + } + case "newValue": { + walker.string(node.kind); + walkExprFields(node.callee, walker, skipType); + for (const item19 of node.args) { + walkExprFields(item19, walker, skipType); + } + if (!skipType) { + walkTypeFields(node.type, walker, skipType); + } + return; + } + case "nullish": { + walker.string(node.kind); + walkExprFields(node.left, walker, skipType); + walkExprFields(node.right, walker, skipType); + if (!skipType) { + walkTypeFields(node.type, walker, skipType); + } + return; + } + case "numLit": { + walker.string(node.kind); + if (node.spelling !== undefined && node.spelling !== null) { + walker.string(node.spelling); + } + if (!skipType) { + walkTypeFields(node.type, walker, skipType); + } + return; + } + case "optChain": { + walker.string(node.kind); + walker.string(node.id); + walkExprFields(node.receiver, walker, skipType); + walkExprFields(node.body, walker, skipType); + if (!skipType) { + walkTypeFields(node.type, walker, skipType); + } + return; + } + case "orDefault": { + walker.string(node.kind); + walkExprFields(node.left, walker, skipType); + walkExprFields(node.right, walker, skipType); + if (node.retag !== undefined && node.retag !== null) { + walker.string(node.retag); + } + if (!skipType) { + walkTypeFields(node.type, walker, skipType); + } + return; + } + case "promiseVoidWiden": { + walker.string(node.kind); + walkExprFields(node.value, walker, skipType); + if (!skipType) { + walkTypeFields(node.type, walker, skipType); + } + return; + } + case "promiseWithResolvers": { + walker.string(node.kind); + if (!skipType) { + walkTypeFields(node.type, walker, skipType); + } + return; + } + case "recordClone": { + walker.string(node.kind); + walkExprFields(node.source, walker, skipType); + for (const item20 of node.overrides) { + walker.string(item20.name); + walkExprFields(item20.value, walker, skipType); + } + if (!skipType) { + walkTypeFields(node.type, walker, skipType); + } + return; + } + case "recordGet": { + walker.string(node.kind); + walkExprFields(node.obj, walker, skipType); + walker.string(node.shapeId); + walker.string(node.field); + if (!skipType) { + walkTypeFields(node.type, walker, skipType); + } + return; + } + case "recordHas": { + walker.string(node.kind); + walkExprFields(node.obj, walker, skipType); + walker.string(node.shapeId); + walker.string(node.field); + if (!skipType) { + walkTypeFields(node.type, walker, skipType); + } + return; + } + case "recordKeyGet": { + walker.string(node.kind); + walkExprFields(node.obj, walker, skipType); + walker.string(node.shapeId); + walkExprFields(node.key, walker, skipType); + if (!skipType) { + walkTypeFields(node.type, walker, skipType); + } + return; + } + case "recordLit": { + walker.string(node.kind); + for (const item21 of node.fields) { + walker.string(item21.name); + walkExprFields(item21.value, walker, skipType); + } + if (!skipType) { + walkTypeFields(node.type, walker, skipType); + } + return; + } + case "recordNullProto": { + walker.string(node.kind); + walkExprFields(node.obj, walker, skipType); + walker.string(node.shapeId); + if (!skipType) { + walkTypeFields(node.type, walker, skipType); + } + return; + } + case "recordOvfHas": { + walker.string(node.kind); + walkExprFields(node.obj, walker, skipType); + walker.string(node.shapeId); + walkExprFields(node.key, walker, skipType); + if (!skipType) { + walkTypeFields(node.type, walker, skipType); + } + return; + } + case "recordOvfKeys": { + walker.string(node.kind); + walkExprFields(node.obj, walker, skipType); + walker.string(node.shapeId); + if (!skipType) { + walkTypeFields(node.type, walker, skipType); + } + return; + } + case "regexIntrinsic": { + walker.string(node.kind); + walker.string(node.method); + walkExprFields(node.receiver, walker, skipType); + for (const item22 of node.args) { + walkExprFields(item22, walker, skipType); + } + if (!skipType) { + walkTypeFields(node.type, walker, skipType); + } + return; + } + case "regexLit": { + walker.string(node.kind); + walker.string(node.pattern); + walker.string(node.flags); + if (!skipType) { + walkTypeFields(node.type, walker, skipType); + } + return; + } + case "selfRef": { + walker.string(node.kind); + if (!skipType) { + walkTypeFields(node.type, walker, skipType); + } + return; + } + case "seqExpr": { + walker.string(node.kind); + for (const item23 of node.stmts) { + walkStmtFields(item23, walker, skipType); + } + walkExprFields(node.result, walker, skipType); + if (!skipType) { + walkTypeFields(node.type, walker, skipType); + } + return; + } + case "setIntrinsic": { + walker.string(node.kind); + walker.string(node.method); + walkExprFields(node.receiver, walker, skipType); + for (const item24 of node.args) { + walkExprFields(item24, walker, skipType); + } + if (!skipType) { + walkTypeFields(node.type, walker, skipType); + } + return; + } + case "setNew": { + walker.string(node.kind); + if (node.seed !== undefined && node.seed !== null) { + walkExprFields(node.seed, walker, skipType); + } + if (!skipType) { + walkTypeFields(node.type, walker, skipType); + } + return; + } + case "strCmp": { + walker.string(node.kind); + walker.string(node.op); + walkExprFields(node.left, walker, skipType); + walkExprFields(node.right, walker, skipType); + if (!skipType) { + walkTypeFields(node.type, walker, skipType); + } + return; + } + case "strConcat": { + walker.string(node.kind); + walkExprFields(node.left, walker, skipType); + walkExprFields(node.right, walker, skipType); + if (!skipType) { + walkTypeFields(node.type, walker, skipType); + } + return; + } + case "strEq": { + walker.string(node.kind); + walkExprFields(node.left, walker, skipType); + walkExprFields(node.right, walker, skipType); + if (!skipType) { + walkTypeFields(node.type, walker, skipType); + } + return; + } + case "strIntrinsic": { + walker.string(node.kind); + walker.string(node.method); + walkExprFields(node.receiver, walker, skipType); + for (const item25 of node.args) { + walkExprFields(item25, walker, skipType); + } + if (!skipType) { + walkTypeFields(node.type, walker, skipType); + } + return; + } + case "strLit": { + walker.string(node.kind); + walker.string(node.value); + if (!skipType) { + walkTypeFields(node.type, walker, skipType); + } + return; + } + case "templateStrings": { + walker.string(node.kind); + walker.string(node.key); + for (const item26 of node.cooked) { + walker.string(item26); + } + if (!skipType) { + walkTypeFields(node.type, walker, skipType); + } + return; + } + case "ternary": { + walker.string(node.kind); + walkExprFields(node.cond, walker, skipType); + walkExprFields(node.then, walker, skipType); + walkExprFields(node.else_, walker, skipType); + if (!skipType) { + walkTypeFields(node.type, walker, skipType); + } + return; + } + case "toBool": { + walker.string(node.kind); + walkExprFields(node.operand, walker, skipType); + if (!skipType) { + walkTypeFields(node.type, walker, skipType); + } + return; + } + case "toString": { + walker.string(node.kind); + walkExprFields(node.operand, walker, skipType); + if (!skipType) { + walkTypeFields(node.type, walker, skipType); + } + return; + } + case "unary": { + walker.string(node.kind); + walker.string(node.op); + walkExprFields(node.operand, walker, skipType); + if (!skipType) { + walkTypeFields(node.type, walker, skipType); + } + return; + } + case "unionDisc": { + walker.string(node.kind); + walker.string(node.unionId); + walker.string(node.field); + walkExprFields(node.value, walker, skipType); + if (!skipType) { + walkTypeFields(node.type, walker, skipType); + } + return; + } + case "unionEq": { + walker.string(node.kind); + walker.string(node.unionId); + walkExprFields(node.left, walker, skipType); + walkExprFields(node.right, walker, skipType); + if (!skipType) { + walkTypeFields(node.type, walker, skipType); + } + return; + } + case "unionFuncEq": { + walker.string(node.kind); + walker.string(node.unionId); + walkExprFields(node.union, walker, skipType); + walkExprFields(node.func, walker, skipType); + if (!skipType) { + walkTypeFields(node.type, walker, skipType); + } + return; + } + case "unionIsTag": { + walker.string(node.kind); + walker.string(node.unionId); + walkExprFields(node.value, walker, skipType); + if (!skipType) { + walkTypeFields(node.type, walker, skipType); + } + return; + } + case "unionKeyGet": { + walker.string(node.kind); + walker.string(node.unionId); + walkExprFields(node.key, walker, skipType); + walkExprFields(node.value, walker, skipType); + if (!skipType) { + walkTypeFields(node.type, walker, skipType); + } + return; + } + case "unionNarrow": { + walker.string(node.kind); + walker.string(node.unionId); + walkExprFields(node.value, walker, skipType); + if (!skipType) { + walkTypeFields(node.type, walker, skipType); + } + return; + } + case "unionRetype": { + walker.string(node.kind); + walker.string(node.fromId); + walker.string(node.unionId); + for (const item27 of node.tags) { + } + walkExprFields(node.value, walker, skipType); + if (!skipType) { + walkTypeFields(node.type, walker, skipType); + } + return; + } + case "unionWrap": { + walker.string(node.kind); + walker.string(node.unionId); + walkExprFields(node.value, walker, skipType); + if (!skipType) { + walkTypeFields(node.type, walker, skipType); + } + return; + } + case "unitLit": { + walker.string(node.kind); + walker.string(node.unit); + if (!skipType) { + walkTypeFields(node.type, walker, skipType); + } + return; + } + case "upcast": { + walker.string(node.kind); + walkExprFields(node.value, walker, skipType); + if (!skipType) { + walkTypeFields(node.type, walker, skipType); + } + return; + } + case "varRef": { + walker.string(node.kind); + walker.string(node.localId); + if (!skipType) { + walkTypeFields(node.type, walker, skipType); + } + return; + } + case "virtualCall": { + walker.string(node.kind); + walker.string(node.className); + walker.string(node.method); + for (const item28 of node.args) { + walkExprFields(item28, walker, skipType); + } + if (!skipType) { + walkTypeFields(node.type, walker, skipType); + } + if (node.targets !== undefined && node.targets !== null) { + for (const item29 of node.targets) { + walker.string(item29); + } + } + return; + } + case "yieldExpr": { + walker.string(node.kind); + if (node.value !== undefined && node.value !== null) { + walkExprFields(node.value, walker, skipType); + } + if (node.captureCompletion !== undefined && node.captureCompletion !== null) { + walkTypeFields(node.captureCompletion.returnType, walker, skipType); + } + if (!skipType) { + walkTypeFields(node.type, walker, skipType); + } + return; + } + } +} + +/** Walks every field of `node` but `loc` (and `type` with `skipType`), and + * every field of each object under it: the strings and nodes a recursive + * `for (const key in value)` walk with the same skips finds, read by name in + * the order ir.ts declares the fields. */ +export function walkStmtFields(node: IrStmt, walker: IrFieldWalker, skipType: boolean): void { + walker.node(node); + switch (node.kind) { + case "arrayDelete": { + walker.string(node.kind); + walkExprFields(node.arr, walker, skipType); + walkExprFields(node.index, walker, skipType); + return; + } + case "arraySet": { + walker.string(node.kind); + walkExprFields(node.arr, walker, skipType); + walkExprFields(node.index, walker, skipType); + walkExprFields(node.value, walker, skipType); + return; + } + case "arraySetLength": { + walker.string(node.kind); + walkExprFields(node.arr, walker, skipType); + walkExprFields(node.length, walker, skipType); + return; + } + case "arraySetUndefined": { + walker.string(node.kind); + walkExprFields(node.arr, walker, skipType); + walkExprFields(node.index, walker, skipType); + return; + } + case "assign": { + walker.string(node.kind); + walker.string(node.localId); + walkExprFields(node.value, walker, skipType); + return; + } + case "block": { + walker.string(node.kind); + for (const item30 of node.body) { + walkStmtFields(item30, walker, skipType); + } + if (node.labels !== undefined && node.labels !== null) { + for (const item31 of node.labels) { + walker.string(item31); + } + } + return; + } + case "break": { + walker.string(node.kind); + if (node.label !== undefined && node.label !== null) { + walker.string(node.label); + } + return; + } + case "bytesSet": { + walker.string(node.kind); + walkExprFields(node.arr, walker, skipType); + walkExprFields(node.index, walker, skipType); + walkExprFields(node.value, walker, skipType); + return; + } + case "continue": { + walker.string(node.kind); + if (node.label !== undefined && node.label !== null) { + walker.string(node.label); + } + return; + } + case "doWhile": { + walker.string(node.kind); + for (const item32 of node.body) { + walkStmtFields(item32, walker, skipType); + } + walkExprFields(node.cond, walker, skipType); + if (node.labels !== undefined && node.labels !== null) { + for (const item33 of node.labels) { + walker.string(item33); + } + } + return; + } + case "exprStmt": { + walker.string(node.kind); + walkExprFields(node.expr, walker, skipType); + return; + } + case "fieldSet": { + walker.string(node.kind); + walkExprFields(node.obj, walker, skipType); + walker.string(node.className); + walker.string(node.field); + walkExprFields(node.value, walker, skipType); + return; + } + case "for": { + walker.string(node.kind); + if (node.init !== undefined && node.init !== null) { + walkStmtFields(node.init, walker, skipType); + } + if (node.cond !== undefined && node.cond !== null) { + walkExprFields(node.cond, walker, skipType); + } + if (node.update !== undefined && node.update !== null) { + walkStmtFields(node.update, walker, skipType); + } + for (const item34 of node.body) { + walkStmtFields(item34, walker, skipType); + } + if (node.labels !== undefined && node.labels !== null) { + for (const item35 of node.labels) { + walker.string(item35); + } + } + return; + } + case "forOf": { + walker.string(node.kind); + walker.string(node.localId); + walkExprFields(node.iterable, walker, skipType); + for (const item36 of node.body) { + walkStmtFields(item36, walker, skipType); + } + if (node.labels !== undefined && node.labels !== null) { + for (const item37 of node.labels) { + walker.string(item37); + } + } + return; + } + case "if": { + walker.string(node.kind); + walkExprFields(node.cond, walker, skipType); + for (const item38 of node.then) { + walkStmtFields(item38, walker, skipType); + } + if (node.else_ !== undefined && node.else_ !== null) { + for (const item39 of node.else_) { + walkStmtFields(item39, walker, skipType); + } + } + return; + } + case "recordKeyDelete": { + walker.string(node.kind); + walkExprFields(node.obj, walker, skipType); + walker.string(node.shapeId); + walkExprFields(node.key, walker, skipType); + return; + } + case "recordKeySet": { + walker.string(node.kind); + walkExprFields(node.obj, walker, skipType); + walker.string(node.shapeId); + walkExprFields(node.key, walker, skipType); + walkExprFields(node.value, walker, skipType); + return; + } + case "recordSet": { + walker.string(node.kind); + walkExprFields(node.obj, walker, skipType); + walker.string(node.shapeId); + walker.string(node.field); + walkExprFields(node.value, walker, skipType); + return; + } + case "rethrow": { + walker.string(node.kind); + walker.string(node.localId); + return; + } + case "return": { + walker.string(node.kind); + if (node.value !== undefined && node.value !== null) { + walkExprFields(node.value, walker, skipType); + } + return; + } + case "runtimeFence": { + walker.string(node.kind); + walker.string(node.code); + walker.string(node.message); + return; + } + case "switch": { + walker.string(node.kind); + walkExprFields(node.disc, walker, skipType); + for (const item40 of node.cases) { + if (item40.test !== undefined && item40.test !== null) { + walkExprFields(item40.test, walker, skipType); + } + for (const item41 of item40.body) { + walkStmtFields(item41, walker, skipType); + } + } + if (node.labels !== undefined && node.labels !== null) { + for (const item42 of node.labels) { + walker.string(item42); + } + } + return; + } + case "throw": { + walker.string(node.kind); + walkExprFields(node.value, walker, skipType); + return; + } + case "tryCatch": { + walker.string(node.kind); + for (const item43 of node.tryBody) { + walkStmtFields(item43, walker, skipType); + } + if (node.catchBody !== undefined && node.catchBody !== null) { + for (const item44 of node.catchBody) { + walkStmtFields(item44, walker, skipType); + } + } + if (node.catchLocalId !== undefined && node.catchLocalId !== null) { + walker.string(node.catchLocalId); + } + if (node.finallyBody !== undefined && node.finallyBody !== null) { + for (const item45 of node.finallyBody) { + walkStmtFields(item45, walker, skipType); + } + } + return; + } + case "varDecl": { + walker.string(node.kind); + walker.string(node.localId); + if (node.init !== undefined && node.init !== null) { + walkExprFields(node.init, walker, skipType); + } + return; + } + case "while": { + walker.string(node.kind); + walkExprFields(node.cond, walker, skipType); + for (const item46 of node.body) { + walkStmtFields(item46, walker, skipType); + } + if (node.labels !== undefined && node.labels !== null) { + for (const item47 of node.labels) { + walker.string(item47); + } + } + return; + } + } +} + +/** Walks every field of `node` but `loc` (and `type` with `skipType`), and + * every field of each object under it: the strings and nodes a recursive + * `for (const key in value)` walk with the same skips finds, read by name in + * the order ir.ts declares the fields. */ +export function walkTypeFields(node: IrType, walker: IrFieldWalker, skipType: boolean): void { + switch (node.kind) { + case "array": { + walker.string(node.kind); + walkTypeFields(node.elem, walker, skipType); + return; + } + case "bigint": { + walker.string(node.kind); + return; + } + case "bool": { + walker.string(node.kind); + return; + } + case "bytes": { + walker.string(node.kind); + walker.string(node.elem); + return; + } + case "caught": { + walker.string(node.kind); + return; + } + case "child": { + walker.string(node.kind); + return; + } + case "childStream": { + walker.string(node.kind); + return; + } + case "childWriter": { + walker.string(node.kind); + return; + } + case "classval": { + walker.string(node.kind); + walker.string(node.className); + return; + } + case "cryptoHash": { + walker.string(node.kind); + return; + } + case "cryptoHmac": { + walker.string(node.kind); + return; + } + case "date": { + walker.string(node.kind); + return; + } + case "dgramSocket": { + walker.string(node.kind); + return; + } + case "dir": { + walker.string(node.kind); + return; + } + case "dyn": { + walker.string(node.kind); + return; + } + case "f64": { + walker.string(node.kind); + return; + } + case "fileHandle": { + walker.string(node.kind); + return; + } + case "fsWatcher": { + walker.string(node.kind); + return; + } + case "func": { + walker.string(node.kind); + for (const item48 of node.params) { + walkTypeFields(item48, walker, skipType); + } + walkTypeFields(node.ret, walker, skipType); + if (node.restAbi !== undefined && node.restAbi !== null) { + walker.string(node.restAbi); + } + return; + } + case "generator": { + walker.string(node.kind); + walkTypeFields(node.yieldT, walker, skipType); + walkTypeFields(node.retT, walker, skipType); + walkTypeFields(node.nextT, walker, skipType); + return; + } + case "http2Session": { + walker.string(node.kind); + return; + } + case "http2Stream": { + walker.string(node.kind); + return; + } + case "httpClientReq": { + walker.string(node.kind); + return; + } + case "httpReq": { + walker.string(node.kind); + return; + } + case "httpRes": { + walker.string(node.kind); + return; + } + case "jsval": { + walker.string(node.kind); + return; + } + case "map": { + walker.string(node.kind); + walkTypeFields(node.key, walker, skipType); + walkTypeFields(node.value, walker, skipType); + return; + } + case "moduleNs": { + walker.string(node.kind); + walker.string(node.moduleId); + return; + } + case "netServer": { + walker.string(node.kind); + return; + } + case "netSocket": { + walker.string(node.kind); + return; + } + case "nullT": { + walker.string(node.kind); + return; + } + case "object": { + walker.string(node.kind); + walker.string(node.className); + return; + } + case "procStream": { + walker.string(node.kind); + return; + } + case "promise": { + walker.string(node.kind); + walkTypeFields(node.inner, walker, skipType); + return; + } + case "record": { + walker.string(node.kind); + walker.string(node.shapeId); + return; + } + case "regex": { + walker.string(node.kind); + return; + } + case "searchParams": { + walker.string(node.kind); + return; + } + case "secureCtx": { + walker.string(node.kind); + return; + } + case "set": { + walker.string(node.kind); + walkTypeFields(node.elem, walker, skipType); + return; + } + case "spawnRes": { + walker.string(node.kind); + return; + } + case "stats": { + walker.string(node.kind); + return; + } + case "string": { + walker.string(node.kind); + return; + } + case "symbol": { + walker.string(node.kind); + return; + } + case "testCtx": { + walker.string(node.kind); + return; + } + case "undefinedT": { + walker.string(node.kind); + return; + } + case "union": { + walker.string(node.kind); + walker.string(node.unionId); + return; + } + case "url": { + walker.string(node.kind); + return; + } + case "void": { + walker.string(node.kind); + return; + } + } +} diff --git a/packages/compiler/src/ir/fields.test.ts b/packages/compiler/src/ir/fields.test.ts new file mode 100644 index 000000000..8d1aaa433 --- /dev/null +++ b/packages/compiler/src/ir/fields.test.ts @@ -0,0 +1,111 @@ +import { mkdtemp, readFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { expect, test } from "vitest"; +import { compile, deserializeModule } from "../index.js"; +import type { IrExpr, IrStmt } from "./ir.js"; +import { everyStmtList } from "./traverse.js"; +import { eachStringField, walkStmtFields } from "./fields.generated.js"; + +// Programs with many node kinds: classes, closures, async, generators, +// dynamic values, streams and records. +const PROGRAMS = [ + "1003-json-parse-unions.ts", + "114-live-record-delete.ts", + "1200-regex-test-basics.ts", + "1631-inspect-arrays.ts", + "4900-console-instances.ts", + "5612-readable-from-iterables.cjs", + "7110-als-completion-callbacks.ts", + "7321-speculative-loop-bail-sites.ts", +]; + +/** The string fields a key walk of `node` finds, as `field=value`. */ +function keyWalk(node: IrExpr | IrStmt): string[] { + const found: string[] = []; + for (const key in node) { + const value = (node as unknown as Record)[key]; + if (typeof value === "string") found.push(`${key}=${value}`); + } + return found.sort(); +} + +/** The events of a recursive key walk of `value` that skips `loc` (and + * `type` with `skipType`): `n:` for an IR node (an object with a kind + * and a location; `n:varDecl:` for a declaration), `s:` for + * each string. */ +function keyWalkDeep(value: unknown, events: string[], skipType: boolean): void { + if (typeof value === "string") events.push(`s:${value}`); + else if (Array.isArray(value)) for (const item of value) keyWalkDeep(item, events, skipType); + else if (value !== null && typeof value === "object") { + const record = value as Record; + if ("kind" in record && "loc" in record) events.push(nodeEvent(record as unknown as IrExpr)); + for (const key in record) + if (key !== "loc" && !(skipType && key === "type")) + keyWalkDeep(record[key], events, skipType); + } +} + +function nodeEvent(node: IrExpr | IrStmt): string { + return node.kind === "varDecl" ? `n:varDecl:${node.localId}` : `n:${node.kind}`; +} + +async function lowered(program: string, dir: string) { + const outPath = join(dir, "main.ir.json"); + const entry = join(import.meta.dirname, "../../../../tests/corpus", program); + const result = await compile(entry, { outDir: dir, outPath, outputKind: "ir" }); + if (!result.ok) + throw new Error(result.diagnostics.map((d) => `${d.code}: ${d.message}`).join("\n")); + return deserializeModule(await readFile(outPath, "utf8")); +} + +// A key walk visits a node's fields in the order the lowering created them, +// which can differ from ir.ts's order between sibling fields (a dynKeyGet is +// built key first). The walk's callers count what they find, except that +// declarations must come in the same order. +test("walkStmtFields reports what a recursive key walk finds", async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-ir-walk-")); + try { + for (const program of PROGRAMS) { + const mod = await lowered(program, dir); + for (const skipType of [false, true]) + for (const fn of mod.functions) { + const expected: string[] = []; + keyWalkDeep(fn.body, expected, skipType); + const events: string[] = []; + const walker = { + node: (node: IrExpr | IrStmt) => events.push(nodeEvent(node)), + string: (value: string) => events.push(`s:${value}`), + }; + for (const stmt of fn.body) walkStmtFields(stmt, walker, skipType); + const where = `${program}: ${fn.name}, skipType ${skipType}`; + expect([...events].sort(), where).toEqual([...expected].sort()); + const declarations = (list: string[]) => list.filter((e) => e.startsWith("n:varDecl:")); + expect(declarations(events), where).toEqual(declarations(expected)); + } + } + } finally { + await rm(dir, { recursive: true, force: true }); + } +}, 300_000); + +test("eachStringField reads every string field a key walk of a node finds", async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-ir-fields-")); + try { + for (const program of PROGRAMS) { + const mod = await lowered(program, dir); + let nodes = 0; + const check = (node: IrExpr | IrStmt): boolean => { + const read: string[] = []; + eachStringField(node, (value, field) => read.push(`${field}=${value}`)); + expect(read.sort(), `${program}: ${node.kind}`).toEqual(keyWalk(node)); + nodes++; + return true; + }; + for (const fn of mod.functions) everyStmtList(fn.body, { expr: check, stmt: check }); + expect(nodes, program).toBeGreaterThan(0); + } + } finally { + await rm(dir, { recursive: true, force: true }); + } +}, 300_000); diff --git a/packages/compiler/src/ir/paired-while-loops.ts b/packages/compiler/src/ir/paired-while-loops.ts index c096c34fe..030b255f2 100644 --- a/packages/compiler/src/ir/paired-while-loops.ts +++ b/packages/compiler/src/ir/paired-while-loops.ts @@ -6,8 +6,10 @@ import { type IrFunction, type IrLocal, type IrModule, + type IrParam, type IrStmt, } from "./ir.js"; +import { type IrFieldWalker, walkStmtFields } from "./fields.generated.js"; import { everyExprChild, everyStmtChild, mapExprChildren, mapStmtChildren } from "./traverse.js"; type WhileLoop = IrStmt & { kind: "while" }; @@ -99,22 +101,20 @@ function continuesLoop(loop: WhileLoop): boolean { return found; } -/** Mentions of each id in `node`, in any field (a string literal equal to - * an id counts too, which only refuses more loops). */ +/** Mentions of each id in `stmts`, in any field but types (a string + * literal equal to an id counts too, which only refuses more loops). */ function countMentions( - node: unknown, + stmts: readonly IrStmt[], ids: ReadonlySet, mentions: Map, ): void { - if (typeof node === "string") { - if (ids.has(node)) mentions.set(node, (mentions.get(node) ?? 0) + 1); - } else if (Array.isArray(node)) { - for (const item of node) countMentions(item, ids, mentions); - } else if (node !== null && typeof node === "object") { - const record = node as Record; - for (const key in record) - if (key !== "loc" && key !== "type") countMentions(record[key], ids, mentions); - } + const walker: IrFieldWalker = { + node: () => {}, + string: (value) => { + if (ids.has(value)) mentions.set(value, (mentions.get(value) ?? 0) + 1); + }, + }; + for (const stmt of stmts) walkStmtFields(stmt, walker, true); } /** A loop with the qualifying shape, the statements before it in its @@ -184,7 +184,12 @@ function rewriteFunction(fn: IrFunction): IrFunction { const params = new Set(fn.params.map((p) => p.localId)); // A counter this function reaches through its environment. const outer = new Map(); - countMentions([fn.captures ?? [], fn.classCaptures ?? []], ids, outer); + const noteOuter = (param: IrParam): void => { + for (const value of [param.localId, param.name]) + if (ids.has(value)) outer.set(value, (outer.get(value) ?? 0) + 1); + }; + for (const param of fn.captures ?? []) noteOuter(param); + for (const param of fn.classCaptures ?? []) noteOuter(param); let ranges: ReadonlyMap | null = null; const used = new Set(fn.locals.map((l) => l.id)); @@ -192,7 +197,7 @@ function rewriteFunction(fn: IrFunction): IrFunction { const rewritten = new Map>(); for (const { loop, counters, before, nested } of candidates) { const inside = new Map(); - countMentions(loop, new Set(counters), inside); + countMentions([loop], new Set(counters), inside); // Each counter's start: its declaration's initializer, or null for a // parameter. const starts = new Map(); diff --git a/scripts/generate-ir-fields.mjs b/scripts/generate-ir-fields.mjs new file mode 100644 index 000000000..5b5ad011a --- /dev/null +++ b/scripts/generate-ir-fields.mjs @@ -0,0 +1,215 @@ +import { readFileSync, writeFileSync } from "node:fs"; +import { createRequire } from "node:module"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; + +// Generates packages/compiler/src/ir/fields.generated.ts from the node types +// in ir.ts: for each IrExpr and IrStmt kind, the fields that can hold a +// string, read by name. Code that would enumerate a node's keys reads them +// through this switch instead: the native compiler enumerates and reads an +// untyped object's keys through its dynamic representation, one conversion +// per node. `--check` fails when the file is stale. +const root = fileURLToPath(new URL("..", import.meta.url)); +const require = createRequire(join(root, "package.json")); +const ts = require("typescript"); +const irPath = join(root, "packages/compiler/src/ir/ir.ts"); +const target = join(root, "packages/compiler/src/ir/fields.generated.ts"); + +const program = ts.createProgram([irPath], { strict: true, target: ts.ScriptTarget.ES2022 }); +const checker = program.getTypeChecker(); +const source = program.getSourceFile(irPath); +const aliases = new Map(); +ts.forEachChild(source, (node) => { + if (ts.isTypeAliasDeclaration(node)) aliases.set(node.name.text, node); +}); + +/** Whether a value of `type` can be a string. */ +function holdsString(type) { + if (type.isUnion()) return type.types.some(holdsString); + return (type.flags & (ts.TypeFlags.StringLike | ts.TypeFlags.Any | ts.TypeFlags.Unknown)) !== 0; +} + +/** Whether every value of `type` is a string. */ +function onlyString(type) { + if (type.isUnion()) return type.types.every(onlyString); + return (type.flags & ts.TypeFlags.StringLike) !== 0; +} + +/** Each kind's string fields, in declaration order, merged over the unions: + * a kind both unions declare lists the fields of both. */ +const kinds = new Map(); +for (const name of ["IrExpr", "IrStmt"]) { + const alias = aliases.get(name); + const union = checker.getTypeAtLocation(alias.name); + for (const member of union.isUnion() ? union.types : [union]) { + const kindSymbol = member.getProperty("kind"); + const kindType = checker.getTypeOfSymbolAtLocation(kindSymbol, alias); + if (!kindType.isStringLiteral()) throw new Error(`${name} member without a literal kind`); + const fields = kinds.get(kindType.value) ?? []; + for (const property of member.getProperties()) { + const type = checker.getTypeOfSymbolAtLocation(property, alias); + if (!holdsString(type)) continue; + const known = fields.find((field) => field.name === property.name); + const always = onlyString(type) && (property.flags & ts.SymbolFlags.Optional) === 0; + if (known === undefined) fields.push({ name: property.name, always }); + else known.always &&= always; + } + kinds.set(kindType.value, fields); + } +} + +const lines = [ + "// Generated from packages/compiler/src/ir/ir.ts by scripts/generate-ir-fields.mjs.", + "// Run `pnpm generate:ir-fields` after changing the fields of an IR node.", + 'import type { IrExpr, IrStmt, IrType } from "./ir.js";', + "", + "/** Calls `visit` with each string field of `node` (including `kind`) and its", + " * name: the string values `for (const key in node)` finds on a node, read", + " * by name in the order ir.ts declares them. */", + "export function eachStringField(", + " node: IrExpr | IrStmt,", + " visit: (value: string, field: string) => void,", + "): void {", + " switch (node.kind) {", +]; +for (const [kind, fields] of [...kinds].sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))) { + lines.push(` case ${JSON.stringify(kind)}: {`); + for (const { name, always } of fields) { + const read = `node.${name}`; + if (always) lines.push(` visit(${read}, ${JSON.stringify(name)});`); + else + lines.push(` if (typeof ${read} === "string") visit(${read}, ${JSON.stringify(name)});`); + } + lines.push(" return;", " }"); +} +lines.push(" }", "}", ""); + +// The deep walkers: every field of every object under an IR node, as a +// recursive key walk finds them, read by name. +const members = new Map(); +for (const name of ["IrExpr", "IrStmt", "IrType"]) { + const union = checker.getTypeAtLocation(aliases.get(name).name); + members.set(name, new Set(union.isUnion() ? union.types : [union])); +} +const nullish = ts.TypeFlags.Null | ts.TypeFlags.Undefined | ts.TypeFlags.Void; +const scalar = + ts.TypeFlags.NumberLike | + ts.TypeFlags.BooleanLike | + ts.TypeFlags.BigIntLike | + ts.TypeFlags.ESSymbolLike; + +/** The walker an IR union member belongs to, or null. */ +function irKind(type) { + for (const [name, set] of members) if (set.has(type)) return name; + return null; +} + +let temp = 0; +/** Lines that walk `expr` of `type`: strings to `walker.string`, IR nodes to + * their walkers, arrays by element and other objects by field. */ +function walkLines(expr, type, indent, depth) { + if (depth > 12) throw new Error(`field type nests too deep at ${expr}`); + const parts = type.isUnion() ? type.types : [type]; + const present = parts.filter((part) => (part.flags & nullish) === 0); + const optional = present.length !== parts.length; + const strings = present.filter((part) => (part.flags & ts.TypeFlags.StringLike) !== 0); + const objects = present.filter((part) => (part.flags & (ts.TypeFlags.StringLike | scalar)) === 0); + for (const part of present) + if ((part.flags & (ts.TypeFlags.Any | ts.TypeFlags.Unknown)) !== 0) + throw new Error(`untyped field ${expr}`); + const body = []; + if (objects.length > 0) { + const kinds = new Set(objects.map(irKind)); + if (kinds.size === 1 && !kinds.has(null)) { + const [kind] = kinds; + body.push(`walk${kind.slice(2)}Fields(${expr}, walker, skipType);`); + } else if (objects.every((part) => checker.isArrayType(part))) { + if (objects.length !== 1) throw new Error(`several array types at ${expr}`); + const item = `item${temp++}`; + body.push(`for (const ${item} of ${expr}) {`); + body.push(...walkLines(item, checker.getTypeArguments(objects[0])[0], " ", depth + 1)); + body.push("}"); + } else if (kinds.has(null) && objects.length === 1) { + for (const property of checker.getPropertiesOfType(objects[0])) { + if (property.name === "loc") continue; + const fieldType = checker.getTypeOfSymbol(property); + const field = `${expr}.${property.name}`; + const lines = walkLines(field, fieldType, "", depth + 1); + if (lines.length === 0) continue; + if (property.name === "type") + body.push("if (!skipType) {", ...lines.map((l) => ` ${l}`), "}"); + else body.push(...lines); + } + } else throw new Error(`mixed object types at ${expr}: ${checker.typeToString(type)}`); + } + let lines = body; + if (strings.length > 0 && objects.length > 0) + lines = [ + `if (typeof ${expr} === "string") walker.string(${expr});`, + ...(body.length > 0 ? [`else {`, ...body.map((l) => ` ${l}`), "}"] : []), + ]; + else if (strings.length > 0) lines = [`walker.string(${expr});`]; + if (lines.length > 0 && optional && !(strings.length > 0 && objects.length > 0)) + lines = [ + `if (${expr} !== undefined && ${expr} !== null) {`, + ...lines.map((l) => ` ${l}`), + "}", + ]; + else if (lines.length > 0 && optional) + lines = [ + `if (${expr} !== undefined && ${expr} !== null) {`, + ...lines.map((l) => ` ${l}`), + "}", + ]; + return lines.map((line) => indent + line); +} + +lines.push( + "/** What a field walk reports (walkExprFields): each expression and", + " * statement node before its fields, and each string a field holds. */", + "export interface IrFieldWalker {", + " node(node: IrExpr | IrStmt): void;", + " string(value: string): void;", + "}", + "", +); +for (const name of ["IrExpr", "IrStmt", "IrType"]) { + const short = name.slice(2); + const nodeCall = name === "IrType" ? [] : [" walker.node(node);"]; + lines.push( + `/** Walks every field of \`node\` but \`loc\` (and \`type\` with \`skipType\`), and`, + " * every field of each object under it: the strings and nodes a recursive", + " * `for (const key in value)` walk with the same skips finds, read by name in", + " * the order ir.ts declares the fields. */", + `export function walk${short}Fields(node: ${name}, walker: IrFieldWalker, skipType: boolean): void {`, + ...nodeCall, + " switch (node.kind) {", + ); + const byKind = new Map(); + for (const member of members.get(name)) { + const kind = checker.getTypeOfSymbol(member.getProperty("kind")).value; + if (byKind.has(kind)) throw new Error(`${name} kind ${kind} declared twice`); + byKind.set(kind, member); + } + for (const [kind, member] of [...byKind].sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))) { + lines.push(` case ${JSON.stringify(kind)}: {`); + for (const property of member.getProperties()) { + if (property.name === "loc") continue; + const fieldType = checker.getTypeOfSymbol(property); + const field = `node.${property.name}`; + const walk = walkLines(field, fieldType, "", 0); + if (walk.length === 0) continue; + if (property.name === "type") + lines.push(" if (!skipType) {", ...walk.map((l) => ` ${l}`), " }"); + else lines.push(...walk.map((l) => ` ${l}`)); + } + lines.push(" return;", " }"); + } + lines.push(" }", "}", ""); +} +const text = lines.join("\n"); + +if (process.argv.includes("--check")) { + if (readFileSync(target, "utf8") !== text) + throw new Error("fields.generated.ts is stale; run node scripts/generate-ir-fields.mjs"); +} else writeFileSync(target, text); From b1614d6e75d0d7ea3f08ec54d1391cfb2d022f2c Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Mon, 26 Oct 2026 08:30:42 -0500 Subject: [PATCH 16/23] Release a substitution's operands when it returns The temps a regex replace or replaceAll consumed (the subject that a previous substitution in a chain produced, a computed replacement) were released at the end of the statement. A chain of substitutions over a large string therefore held every intermediate result until the last one returned. The operands now live in a frame of their own that is released right after the call, so a chain holds at most its current subject and result. --- .../compiler/src/backend/llvm/expr-records.ts | 22 +++++++ .../test/regex-replace-lifetimes.test.ts | 57 +++++++++++++++++++ 2 files changed, 79 insertions(+) create mode 100644 packages/compiler/test/regex-replace-lifetimes.test.ts diff --git a/packages/compiler/src/backend/llvm/expr-records.ts b/packages/compiler/src/backend/llvm/expr-records.ts index af4bb092f..58f1297b3 100644 --- a/packages/compiler/src/backend/llvm/expr-records.ts +++ b/packages/compiler/src/backend/llvm/expr-records.ts @@ -195,9 +195,31 @@ function emitReplaceWith( return host.own({ name: t, type: e.type }); } +/** Substitutions whose result owns its text: once the runtime call returns, + * no operand is read again. */ +function releasesInputsOnReturn(method: string): boolean { + return method === "replace" || method === "replaceAll"; +} + export function emitRegexIntrinsic( host: LlvmEmitterContext, e: IrExpr & { kind: "regexIntrinsic" }, +): LlValue { + if (!releasesInputsOnReturn(e.method)) return emitRegexOperation(host, e); + // The operands' temps (the subject of a chained substitution, a computed + // replacement) are released when the call returns rather than at the + // end of the statement, so a chain of substitutions holds at most its + // current subject and result. + host.frames.push([]); + const out = emitRegexOperation(host, e); + host.toFrame(out); + host.releaseTemps(host.frames.pop()!); + return host.own(out); +} + +function emitRegexOperation( + host: LlvmEmitterContext, + e: IrExpr & { kind: "regexIntrinsic" }, ): LlValue { const B = host.B; const method = e.method; diff --git a/packages/compiler/test/regex-replace-lifetimes.test.ts b/packages/compiler/test/regex-replace-lifetimes.test.ts new file mode 100644 index 000000000..0c60a9e5e --- /dev/null +++ b/packages/compiler/test/regex-replace-lifetimes.test.ts @@ -0,0 +1,57 @@ +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { expect, test } from "vitest"; +import { compile } from "../src/index.js"; + +async function emitLlvm(source: string): Promise { + const dir = await mkdtemp(join(tmpdir(), "scriptc-regex-replace-")); + try { + const entry = join(dir, "main.ts"); + const outPath = join(dir, "main.ll"); + await writeFile(entry, source); + const result = await compile(entry, { outDir: dir, outPath, outputKind: "llvm" }); + expect(result.ok, JSON.stringify(result.diagnostics)).toBe(true); + return await readFile(outPath, "utf8"); + } finally { + await rm(dir, { recursive: true, force: true }); + } +} + +function functionBody(llvm: string, name: string): string[] { + const start = llvm.split("\n").findIndex((l) => l.startsWith("define") && l.includes(name)); + expect(start).toBeGreaterThanOrEqual(0); + const lines = llvm.split("\n").slice(start); + return lines.slice(0, lines.indexOf("}")); +} + +test("a chain of substitutions releases each intermediate result when the next one returns", async () => { + const llvm = await emitLlvm(` +function chain(s: string): number { + return s + .replace(/a[bc]/g, "<1>") + .replace(/b[cd]/g, "<2>") + .replace(/c[de]/g, "<3>") + .replaceAll(/<[^>]*>/g, "|").length; +} +console.log(chain(process.argv[2] ?? "abcabc")); +`); + const body = functionBody(llvm, "chain"); + const calls = body + .map((line, index) => ({ line, index })) + .filter(({ line }) => /call ptr @scr_regex_replace(_all|_with)?\(/.test(line)); + expect(calls).toHaveLength(4); + const results = calls.map(({ line }) => /^\s*(%[\w.]+) = call/.exec(line)![1]!); + // Each intermediate result is released after the call that consumes it + // and before the next substitution starts. + for (let k = 0; k < 3; k++) { + const release = body.findIndex( + (line, index) => + index > calls[k + 1]!.index && + line.includes("release") && + line.includes(`(ptr ${results[k]})`), + ); + expect(release, `release of ${results[k]}`).toBeGreaterThan(calls[k + 1]!.index); + if (k + 2 < calls.length) expect(release).toBeLessThan(calls[k + 2]!.index); + } +}); From 5aa6699b3dc38ed2429bb0e95564f62322a10730 Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Mon, 26 Oct 2026 07:09:02 -0700 Subject: [PATCH 17/23] Build the candidate's own compiler source in the zero-cost gate The self program compiles the compiler source of the checkout the programs come from (the baseline by default), so a change adding compiler code the self-hosted frontend cannot lower passed the gate and failed CI's self-hosting tests. With the self set, the gate now also builds the candidate's packages/compiler/src/native/cli.ts with the candidate (LLVM only, not compared) and fails when that build fails. Co-Authored-By: Claude Opus 5.5 --- scripts/zero-cost-gate.mjs | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/scripts/zero-cost-gate.mjs b/scripts/zero-cost-gate.mjs index 99b1bc468..a4427f7e8 100644 --- a/scripts/zero-cost-gate.mjs +++ b/scripts/zero-cost-gate.mjs @@ -79,7 +79,8 @@ runtime pack). Both compile the same program files, taken from --programs --set picks the good-code set: bench (benchmarks/runtime workloads and the benchmarks/builds applications), self (packages/compiler/src/native/cli.ts, -LLVM only), corpus (tests/corpus TypeScript programs; --corpus=all adds +LLVM only; the candidate's own compiler source must also build with the +candidate), corpus (tests/corpus TypeScript programs; --corpus=all adds JavaScript entries). --quick is --set=bench --timing-runs=0, for CI. --shard=i/n keeps every n-th corpus program; --only filters program ids. @@ -629,6 +630,30 @@ if (timingRuns > 0) { } } +// The self program is the baseline's compiler source (the programs come +// from one checkout), so a change that adds compiler code the self-hosted +// frontend cannot lower would pass unnoticed. The candidate's own compiler +// source must still build with the candidate (LLVM only, not compared). +if (sets.has("self") && programsRoot !== trees.candidate) { + const entry = join(trees.candidate, "packages/compiler/src/native/cli.ts"); + const result = await buildProgram( + { id: "self/candidate-source", entry, kind: "source" }, + "candidate", + ); + report.candidateSelf = result.ok + ? { ok: true, buildMs: result.ms } + : { ok: false, error: result.error }; + console.log( + result.ok + ? `self candidate compiler source builds with the candidate (${fmtMs(result.ms)})` + : `self candidate compiler source FAILS to build with the candidate`, + ); + if (!result.ok) { + failed = true; + report.failures.push(`self/candidate-source: ${result.error.split("\n")[0]}`); + } +} + const counts = {}; for (const p of report.programs) counts[p.verdict] = (counts[p.verdict] ?? 0) + 1; if (!report.programs.some((p) => ["identical", "allowed", "differs"].includes(p.verdict))) { From be293e3dfbbee4ce16d1c029249f48cae173d247 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Mon, 26 Oct 2026 10:19:25 -0500 Subject: [PATCH 18/23] Unroll counted loops by the period of a guard on the counter's low bits A counted loop whose body branches on `(i & M) === C`, with M one of 1, 3 and 7 (a check every second, fourth or eighth iteration), now asks LLVM to unroll it by M + 1 (llvm.loop.unroll.count on its back edge). In each unrolled copy the counter's low bits are known, so the guard folds to a constant and only one copy per period keeps the check. Before, LLVM tested the guard on every iteration and, when the guarded condition was cheap, evaluated that condition on every iteration too. The loop must step by 1 from a multiple of the period, have no nested loop, and stay under 4096 IR nodes once unrolled. Loops with the same period share one loop ID node, which the module emits only when a back edge refers to it. --- packages/compiler/src/backend/llvm/blocks.ts | 4 +- .../compiler/src/backend/llvm/emit-module.ts | 3 + .../src/backend/llvm/emit-statements.ts | 12 ++- .../src/backend/llvm/periodic-guards.test.ts | 80 +++++++++++++++ .../src/backend/llvm/periodic-guards.ts | 97 +++++++++++++++++++ .../test/periodic-guard-emission.test.ts | 53 ++++++++++ 6 files changed, 245 insertions(+), 4 deletions(-) create mode 100644 packages/compiler/src/backend/llvm/periodic-guards.test.ts create mode 100644 packages/compiler/src/backend/llvm/periodic-guards.ts create mode 100644 packages/compiler/test/periodic-guard-emission.test.ts diff --git a/packages/compiler/src/backend/llvm/blocks.ts b/packages/compiler/src/backend/llvm/blocks.ts index eee4d00b9..3ae65cb0f 100644 --- a/packages/compiler/src/backend/llvm/blocks.ts +++ b/packages/compiler/src/backend/llvm/blocks.ts @@ -103,8 +103,8 @@ export class BlockBuilder { } } - br(label: string): void { - this.terminate(`br label %${label}`); + br(label: string, attachment = ""): void { + this.terminate(`br label %${label}${attachment}`); } condBr(v: string, t: string, f: string, attachment = ""): void { diff --git a/packages/compiler/src/backend/llvm/emit-module.ts b/packages/compiler/src/backend/llvm/emit-module.ts index d25427e4f..15efa88c7 100644 --- a/packages/compiler/src/backend/llvm/emit-module.ts +++ b/packages/compiler/src/backend/llvm/emit-module.ts @@ -1,4 +1,5 @@ /* LlEmitter's module-level output: module parts, helper definitions, library declarations and FFI callback definitions. */ +import { loopUnrollMetadata } from "./periodic-guards.js"; import { relieveHeapPressure } from "../../heap-pressure.js"; import { emitWasmModules } from "./wasm-modules.js"; import { type LlEmitter, ffiCallbackDummyLl, llStrBytes } from "./emitter.js"; @@ -1099,6 +1100,7 @@ export function emitModuleParts(emitter: LlEmitter): string[] { out.push(...functionAttributes(emitter, hasNoInlineRecordClone)); out.push(``); out.push(...emitter.fieldAliasMetadata()); + out.push(...loopUnrollMetadata(out)); if (emitter.debug !== null) out.push(emitter.debug.render()); return out; } @@ -1274,6 +1276,7 @@ export function emitModuleParts(emitter: LlEmitter): string[] { ); } out.push(...emitter.fieldAliasMetadata()); + out.push(...loopUnrollMetadata(out)); if (emitter.debug !== null) out.push(emitter.debug.render()); return out; } diff --git a/packages/compiler/src/backend/llvm/emit-statements.ts b/packages/compiler/src/backend/llvm/emit-statements.ts index c1f3cf18f..45d341853 100644 --- a/packages/compiler/src/backend/llvm/emit-statements.ts +++ b/packages/compiler/src/backend/llvm/emit-statements.ts @@ -95,6 +95,7 @@ import { } from "./string-construction.js"; import { emitStringInputs } from "./string-lifetimes.js"; import { LOOPS } from "./loop-state.js"; +import { loopUnrollAttachment, periodicGuardPeriod } from "./periodic-guards.js"; import { loopAccessesTypedArrays } from "../loop-polls.js"; import { emitConstantChoice, matchConstantChoice } from "./constant-choices.js"; @@ -1315,6 +1316,13 @@ export function emitStmtBody(emitter: LlEmitter, s: IrStmt): void { const lb = B.newLabel("loop.b"); const lu = s.update || capturedLets.length > 0 || workerBudget !== null ? B.newLabel("loop.u") : lc; + // A periodic guard on the counter (periodic-guards.ts): the back + // edge asks LLVM to unroll the loop by the guard's period. + const period = + countedLoop && integerSlot + ? periodicGuardPeriod(s, countedLoop.localId, countedLoop.start, countedLoop.step) + : null; + const backEdge = period === null ? "" : loopUnrollAttachment(period); B.br(lc); B.startBlock(lc); enterLoopHeader(emitter, lc); @@ -1375,7 +1383,7 @@ export function emitStmtBody(emitter: LlEmitter, s: IrStmt): void { }); emitter.emitBlock(s.body); emitter.jumpTargets.pop(); - B.br(lu); + B.br(lu, lu === lc ? backEdge : ""); if (lu !== lc) { B.startBlock(lu); emitter.emitWorkerLoopCheck(workerBudget); @@ -1408,7 +1416,7 @@ export function emitStmtBody(emitter: LlEmitter, s: IrStmt): void { B.line(`store i64 ${next}, ptr ${pairedSlots[i]}`); }); } else if (s.update) emitter.emitStmt(s.update); - B.br(lc); + B.br(lc, backEdge); } }); B.startBlock(le); diff --git a/packages/compiler/src/backend/llvm/periodic-guards.test.ts b/packages/compiler/src/backend/llvm/periodic-guards.test.ts new file mode 100644 index 000000000..d1df4e7ab --- /dev/null +++ b/packages/compiler/src/backend/llvm/periodic-guards.test.ts @@ -0,0 +1,80 @@ +import { expect, test } from "vitest"; +import { countedFor } from "../../ir/build.js"; +import { BOOL, F64, type IrExpr, type IrStmt } from "../../ir/ir.js"; +import { + loopUnrollAttachment, + loopUnrollMetadata, + periodicGuardPeriod, +} from "./periodic-guards.js"; + +const loc = { file: "guards.ts", start: 0, end: 0 }; +const ref = (localId: string): IrExpr => ({ kind: "varRef", localId, type: F64, loc }); +const num = (value: number): IrExpr => ({ kind: "numLit", value, type: F64, loc }); +const bin = (op: string, left: IrExpr, right: IrExpr, type = F64): IrExpr => + ({ kind: "bin", op, left, right, type, loc }) as IrExpr; +const and = (left: IrExpr, right: IrExpr): IrExpr => ({ + kind: "logical", + op: "&&", + left, + right, + type: BOOL, + loc, +}); +const guard = (cond: IrExpr): IrStmt => ({ + kind: "if", + cond, + then: [{ kind: "break", loc } as IrStmt], + else_: null, + loc, +}); +/** `for (i = 0; i < 50; i++) { ...body }` */ +const loop = (body: IrStmt[]) => countedFor(loc, num(50), () => body) as IrStmt & { kind: "for" }; +const period = (body: IrStmt[], start = 0, step = 1) => + periodicGuardPeriod(loop(body), "i.0", num(start), step); + +test("a guard on the counter's low bits gives the loop its period", () => { + // if ((i & 3) === 3 && x > 4) break; + const masked = bin("===", bin("&", ref("i.0"), num(3)), num(3), BOOL); + expect(period([guard(and(masked, bin(">", ref("x"), num(4), BOOL)))])).toBe(4); + // if (0 == (7 & i)) ... + expect(period([guard(bin("==", num(0), bin("&", num(7), ref("i.0")), BOOL))])).toBe(8); + expect(period([guard(bin("!==", bin("&", ref("i.0"), num(1)), num(0), BOOL))])).toBe(2); +}); + +test("other guards and loops get no period", () => { + const masked = (mask: number, value: number, counter = "i.0") => + guard(bin("===", bin("&", ref(counter), num(mask)), num(value), BOOL)); + // Masks that are not 1, 3 or 7, values outside the mask, another variable. + expect(period([masked(5, 1)])).toBeNull(); + expect(period([masked(15, 15)])).toBeNull(); + expect(period([masked(3, 4)])).toBeNull(); + expect(period([masked(3, 3, "j")])).toBeNull(); + // The masked test is not the condition's first conjunct. + const later = and( + bin(">", ref("x"), num(4), BOOL), + bin("===", bin("&", ref("i.0"), num(3)), num(3), BOOL), + ); + expect(period([guard(later)])).toBeNull(); + // A start that is not a multiple of the period, a step other than 1. + expect(period([masked(3, 3)], 1)).toBeNull(); + expect(period([masked(3, 3)], 0, 2)).toBeNull(); + // A nested loop. + expect(period([masked(3, 3), loop([])])).toBeNull(); +}); + +test("a large body is not unrolled", () => { + const work: IrStmt[] = []; + for (let k = 0; k < 400; k++) + work.push({ kind: "assign", localId: "x", value: bin("+", ref("x"), num(k)), loc } as IrStmt); + expect( + period([guard(bin("===", bin("&", ref("i.0"), num(3)), num(3), BOOL)), ...work]), + ).toBeNull(); +}); + +test("the module carries the loop ID nodes its back edges refer to", () => { + const lines = [` br label %loop.c1${loopUnrollAttachment(4)}`, " ret void"]; + expect(loopUnrollMetadata(lines)).toEqual([ + `!1000004 = distinct !{!1000004, !{!"llvm.loop.unroll.count", i32 4}}`, + ]); + expect(loopUnrollMetadata([" ret void"])).toEqual([]); +}); diff --git a/packages/compiler/src/backend/llvm/periodic-guards.ts b/packages/compiler/src/backend/llvm/periodic-guards.ts new file mode 100644 index 000000000..3a07126db --- /dev/null +++ b/packages/compiler/src/backend/llvm/periodic-guards.ts @@ -0,0 +1,97 @@ +/* Counted loops with a periodic guard: a branch on the counter's low bits, + * such as `if ((j & 3) === 3 && ...)` (a check every fourth iteration). + * LLVM checks such a guard on every iteration, and when the guarded + * condition is cheap it speculates the condition into the loop. The loop's + * back edge asks LLVM to unroll it by the guard's period + * (`llvm.loop.unroll.count`): in each unrolled copy the counter's low bits + * are known, so the guard folds to a constant, and only one copy in each + * period keeps the check. The metadata changes how LLVM compiles the loop, + * not what it computes. */ +import type { IrExpr, IrStmt } from "../../ir/ir.js"; +import { everyExprChild, everyStmtChild } from "../../ir/traverse.js"; + +/** The most IR nodes a loop body may have, times the period, for the loop + * to be unrolled: a larger body gains less than its unrolled copies cost. */ +const UNROLLED_BODY_NODES = 4096; + +/** The loop IDs, one per period (2, 4 and 8). A loop ID is a distinct + * node whose first operand is itself; loops with the same period share + * one, as the copies of an inlined loop do. */ +const loopIds = new Map([ + [2, 1_000_002], + [4, 1_000_004], + [8, 1_000_008], +]); + +/** The guard period of `loop`, a counted loop on `counter` that starts at + * `start` and steps by `step`, or null. The period is M + 1 where the body + * has an `if` whose condition is, or starts with an `&&` of, + * `(counter & M) === C` (or `==`, `!==`, `!=`, either operand order) with + * M one of 1, 3 and 7 and C an integer from 0 to M. The loop steps by 1 + * from a multiple of the period, so that the counter's low bits are known + * in each unrolled copy, and its body has no nested loop. */ +export function periodicGuardPeriod( + loop: IrStmt & { kind: "for" }, + counter: string, + start: IrExpr, + step: number, +): number | null { + if (step !== 1 || start.kind !== "numLit") return null; + let period: number | null = null; + let nodes = 0; + let plain = true; + const expr = (e: IrExpr): boolean => { + nodes++; + return everyExprChild(e, expr, stmt); + }; + const stmt = (s: IrStmt): boolean => { + nodes++; + if (s.kind === "for" || s.kind === "while" || s.kind === "doWhile" || s.kind === "forOf") { + plain = false; + return false; + } + if (s.kind === "if" && period === null) period = guardPeriod(s.cond, counter); + return everyStmtChild(s, expr, stmt); + }; + for (const s of loop.body) if (!stmt(s)) break; + if (!plain || period === null) return null; + if (start.value % period !== 0 || nodes * period > UNROLLED_BODY_NODES) return null; + return period; +} + +function guardPeriod(cond: IrExpr, counter: string): number | null { + let test = cond; + while (test.kind === "logical" && test.op === "&&") test = test.left; + if (test.kind !== "bin" || !["===", "==", "!==", "!="].includes(test.op)) return null; + const masked = (e: IrExpr, other: IrExpr): number | null => { + if (e.kind !== "bin" || e.op !== "&" || other.kind !== "numLit") return null; + const [ref, mask] = e.left.kind === "varRef" ? [e.left, e.right] : [e.right, e.left]; + if (ref.kind !== "varRef" || ref.localId !== counter || mask.kind !== "numLit") return null; + const m = mask.value; + if (m !== 1 && m !== 3 && m !== 7) return null; + const c = other.value; + return Number.isInteger(c) && c >= 0 && c <= m ? m + 1 : null; + }; + return masked(test.left, test.right) ?? masked(test.right, test.left); +} + +/** The attachment that asks LLVM to unroll a loop by `period`, for its + * back edge. */ +export function loopUnrollAttachment(period: number): string { + return `, !llvm.loop !${loopIds.get(period)!}`; +} + +/** The loop ID nodes that the module's text `lines` refer to. */ +export function loopUnrollMetadata(lines: readonly string[]): string[] { + const used = new Set(); + for (const line of lines) { + if (!line.includes("!llvm.loop !")) continue; + for (const [period, id] of loopIds) if (line.includes(`!llvm.loop !${id}`)) used.add(period); + } + return [...used] + .sort((a, b) => a - b) + .map((period) => { + const id = loopIds.get(period)!; + return `!${id} = distinct !{!${id}, !{!"llvm.loop.unroll.count", i32 ${period}}}`; + }); +} diff --git a/packages/compiler/test/periodic-guard-emission.test.ts b/packages/compiler/test/periodic-guard-emission.test.ts new file mode 100644 index 000000000..dfb8c60ae --- /dev/null +++ b/packages/compiler/test/periodic-guard-emission.test.ts @@ -0,0 +1,53 @@ +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { expect, test } from "vitest"; +import { compile } from "../src/index.js"; + +async function emitLlvm(source: string): Promise { + const dir = await mkdtemp(join(tmpdir(), "scriptc-periodic-guards-")); + try { + const entry = join(dir, "main.ts"); + const outPath = join(dir, "main.ll"); + await writeFile(entry, source); + const result = await compile(entry, { outDir: dir, outPath, outputKind: "llvm" }); + if (!result.ok) + throw new Error(result.diagnostics.map((d) => `${d.code}: ${d.message}`).join("\n")); + return await readFile(outPath, "utf8"); + } finally { + await rm(dir, { recursive: true, force: true }); + } +} + +test("a counted loop with a check every fourth iteration asks LLVM to unroll it by 4", async () => { + const llvm = await emitLlvm(` +function escape(c: number): number { + let z = 0; + let m = 0; + for (let j = 0; j < 50; j++) { + z = z * z + c; + m = z * z; + if ((j & 3) === 3 && !(m <= 4)) break; + } + return m; +} +console.log(escape(Number(process.argv[2] ?? "0.25"))); +`); + expect(llvm).toMatch(/br label %loop\.c\d+, !llvm\.loop !1000004/); + expect(llvm).toContain(`!1000004 = distinct !{!1000004, !{!"llvm.loop.unroll.count", i32 4}}`); +}); + +test("a loop without a periodic guard has no loop metadata", async () => { + const llvm = await emitLlvm(` +function sum(n: number): number { + let s = 0; + for (let j = 0; j < n; j++) { + s += j; + if (s > 1000) break; + } + return s; +} +console.log(sum(Number(process.argv[2] ?? "10"))); +`); + expect(llvm).not.toContain("!llvm.loop"); +}); From 928c30e908fc2faf0caece6967dcf44500974d34 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Mon, 26 Oct 2026 10:19:57 -0500 Subject: [PATCH 19/23] Bind thisArg in the array callback methods of dynamic arrays forEach, map, filter, some, every, find, findIndex, findLast, findLastIndex and flatMap on a dynamic array, and through Array.prototype..call on a native array, called the callback without its thisArg, so `this` in the callback was not the second argument. These methods now bind `this` to the thisArg, or undefined when there is none, once per call, around all of the method's callbacks. --- packages/runtime/src/scr_dyn_invoke.c | 222 ++++++++++-------- .../array-prototype-call-native-arrays.ts | 43 ++++ 2 files changed, 164 insertions(+), 101 deletions(-) diff --git a/packages/runtime/src/scr_dyn_invoke.c b/packages/runtime/src/scr_dyn_invoke.c index 6deca008c..7a9b1bc35 100644 --- a/packages/runtime/src/scr_dyn_invoke.c +++ b/packages/runtime/src/scr_dyn_invoke.c @@ -916,6 +916,117 @@ void scr_dyn_invoke_discard_id(ScrDyn *recv, ScrDyn *callee, const char *method, scr_dyn_release(scr_dyn_invoke_prepared_id(recv, callee, method, mid_slot, args, argc, what)); } +/* The callback methods of a dense dyn array that take a thisArg: + * forEach, map, filter, some, every, find, findIndex, findLast, + * findLastIndex and flatMap. The caller binds `this` for the callbacks. + * `len` is the length before the first callback. */ +static ScrDyn *dyn_arr_callback_method(ScrDyn *recv, int mid, ScrDyn *const *args, size_t argc, + size_t len, ScrDyn *callback_recv) { + if ((mid == DYN_M_findLast) || (mid == DYN_M_findLastIndex)) { + if (!dyn_cb_check(args, argc)) return NULL; + ScrDyn *visible_recv = callback_recv ? callback_recv : recv; + for (size_t i = len; i > 0; i--) { + size_t index = i - 1; + ScrDyn *item = index < recv->v.arr.len + ? scr_dyn_retain(recv->v.arr.items[index]) + : scr_dyn_retain(scr_dyn_undefined()); + ScrDyn *result = dyn_call_cb(args[0], item, index, visible_recv); + if (!result) { scr_dyn_release(item); return NULL; } + bool matched = scr_dyn_truthy(result); + scr_dyn_release(result); + if (matched) { + if ((mid == DYN_M_findLast)) return item; + scr_dyn_release(item); + return scr_dyn_new_num((double)index); + } + scr_dyn_release(item); + } + return (mid == DYN_M_findLast) + ? scr_dyn_retain(scr_dyn_undefined()) : scr_dyn_new_num(-1); + } + if ((mid == DYN_M_forEach) || (mid == DYN_M_map) || + (mid == DYN_M_filter) || (mid == DYN_M_some) || + (mid == DYN_M_every) || (mid == DYN_M_find) || + (mid == DYN_M_findIndex)) { + if (!dyn_cb_check(args, argc)) return NULL; + ScrDyn *cb = args[0]; + ScrDyn *out = ((mid == DYN_M_map) || (mid == DYN_M_filter)) ? scr_dyn_new_arr() : NULL; + /* Array iteration methods snapshot length before the first callback. + * A shrinking callback can remove a later dense entry; an expanding + * callback must not make the new tail observable to this pass. Live + * Web-boundary capsules are the externally visible receiver, so pass + * that capsule as callback arg 3: mutations through the callback's + * array argument then commit directly to the original static array. */ + size_t n = len; + ScrDyn *visible_recv = callback_recv ? callback_recv : recv; + for (size_t step = 0; step < n; step++) { + size_t i = step; + if (i >= recv->v.arr.len) continue; + ScrDyn *item = scr_dyn_retain(recv->v.arr.items[i]); + ScrDyn *r = dyn_call_cb(cb, item, i, visible_recv); + if (!r) { scr_dyn_release(item); scr_dyn_release(out); return NULL; } + if ((mid == DYN_M_map)) { + scr_dyn_arr_push(out, r); /* ownership moves in */ + r = NULL; + } else { + bool truthy = scr_dyn_truthy(r); + scr_dyn_release(r); + if ((mid == DYN_M_filter) && truthy) scr_dyn_arr_push(out, scr_dyn_retain(item)); + if ((mid == DYN_M_some) && truthy) { scr_dyn_release(item); return scr_dyn_new_bool(true); } + if ((mid == DYN_M_every) && !truthy) { scr_dyn_release(item); return scr_dyn_new_bool(false); } + if ((mid == DYN_M_find) && truthy) return item; /* +1 moves out */ + if ((mid == DYN_M_findIndex) && truthy) { scr_dyn_release(item); return scr_dyn_new_num((double)i); } + } + scr_dyn_release(item); + } + if (out) return out; + if ((mid == DYN_M_some)) return scr_dyn_new_bool(false); + if ((mid == DYN_M_every)) return scr_dyn_new_bool(true); + if ((mid == DYN_M_find)) return scr_dyn_retain(scr_dyn_undefined()); + if ((mid == DYN_M_findIndex)) return scr_dyn_new_num(-1); + return scr_dyn_retain(scr_dyn_undefined()); /* forEach */ + } + if ((mid == DYN_M_flatMap)) { + /* JS Array.prototype.flatMap over a dyn array: map + a depth-1 + * flatten. Native dyn-array results flatten element-by-element; a + * WRAPPED engine array flattens through the routed keyed reads + * (elements wrap back scalar-normalized); everything else pushes + * as a single element (JS keeps non-array callback results whole). + * JS's spec snapshots the length up front (elements appended by + * the callback are not visited). */ + if (!dyn_cb_check(args, argc)) return NULL; + ScrDyn *cb = args[0]; + ScrDyn *out = scr_dyn_new_arr(); + size_t n = recv->v.arr.len; + for (size_t i = 0; i < n && i < recv->v.arr.len; i++) { + if (!dyn_borrow_present(recv, i)) { if (scr_exc_pending()) { scr_dyn_release(out); return NULL; } continue; } + ScrDyn *item = dyn_borrow_item(recv, i); + if (!item) { scr_dyn_release(out); return NULL; } + ScrDyn *r = dyn_call_cb( + cb, item, i, callback_recv ? callback_recv : recv); + scr_dyn_release(item); + if (!r) { scr_dyn_release(out); return NULL; } + if (r->kind == SCR_DYN_TYPED_REF && scr_dyn_ref_is_array(r)) { + ScrDyn *view = scr_dyn_typed_ref_materialize(r); + scr_dyn_release(r); + r = view; + if (!r) { scr_dyn_release(out); return NULL; } + } + if (r->kind == SCR_DYN_ARR) { + for (size_t j = 0; j < r->v.arr.len; j++) { + if (dyn_borrow_present(r, j)) scr_dyn_arr_push(out, dyn_borrow_item(r, j)); + if (scr_exc_pending()) { scr_dyn_release(r); scr_dyn_release(out); return NULL; } + } + scr_dyn_release(r); + } else { + scr_dyn_arr_push(out, r); /* ownership moves in */ + } + } + return out; + } + return NULL; +} + static ScrDyn *scr_dyn_invoke_impl( ScrDyn *recv, const char *method, int mid, ScrDyn *const *args, size_t argc, const char *what, ScrDyn *callback_recv) { @@ -1367,27 +1478,16 @@ static ScrDyn *scr_dyn_invoke_impl( } return (mid == DYN_M_includes) ? scr_dyn_new_bool(false) : scr_dyn_new_num(-1); } - if ((mid == DYN_M_findLast) || (mid == DYN_M_findLastIndex)) { - if (!dyn_cb_check(args, argc)) return NULL; - ScrDyn *visible_recv = callback_recv ? callback_recv : recv; - for (size_t i = len; i > 0; i--) { - size_t index = i - 1; - ScrDyn *item = index < recv->v.arr.len - ? scr_dyn_retain(recv->v.arr.items[index]) - : scr_dyn_retain(scr_dyn_undefined()); - ScrDyn *result = dyn_call_cb(args[0], item, index, visible_recv); - if (!result) { scr_dyn_release(item); return NULL; } - bool matched = scr_dyn_truthy(result); - scr_dyn_release(result); - if (matched) { - if ((mid == DYN_M_findLast)) return item; - scr_dyn_release(item); - return scr_dyn_new_num((double)index); - } - scr_dyn_release(item); - } - return (mid == DYN_M_findLast) - ? scr_dyn_retain(scr_dyn_undefined()) : scr_dyn_new_num(-1); + if ((mid == DYN_M_forEach) || (mid == DYN_M_map) || (mid == DYN_M_filter) || + (mid == DYN_M_some) || (mid == DYN_M_every) || (mid == DYN_M_find) || + (mid == DYN_M_findIndex) || (mid == DYN_M_findLast) || (mid == DYN_M_findLastIndex) || + (mid == DYN_M_flatMap)) { + /* Every callback runs with `this` bound to the thisArg, or undefined + * when there is none. */ + scr_dyn_this_push_dyn(argc > 1 ? args[1] : scr_dyn_undefined()); + ScrDyn *result = dyn_arr_callback_method(recv, mid, args, argc, len, callback_recv); + scr_dyn_this_pop(); + return result; } if ((mid == DYN_M_join)) { ScrJsonBuf b; @@ -1550,48 +1650,6 @@ static ScrDyn *scr_dyn_invoke_impl( } return scr_dyn_retain(recv); } - if ((mid == DYN_M_forEach) || (mid == DYN_M_map) || - (mid == DYN_M_filter) || (mid == DYN_M_some) || - (mid == DYN_M_every) || (mid == DYN_M_find) || - (mid == DYN_M_findIndex)) { - if (!dyn_cb_check(args, argc)) return NULL; - ScrDyn *cb = args[0]; - ScrDyn *out = ((mid == DYN_M_map) || (mid == DYN_M_filter)) ? scr_dyn_new_arr() : NULL; - /* Array iteration methods snapshot length before the first callback. - * A shrinking callback can remove a later dense entry; an expanding - * callback must not make the new tail observable to this pass. Live - * Web-boundary capsules are the externally visible receiver, so pass - * that capsule as callback arg 3: mutations through the callback's - * array argument then commit directly to the original static array. */ - size_t n = len; - ScrDyn *visible_recv = callback_recv ? callback_recv : recv; - for (size_t step = 0; step < n; step++) { - size_t i = step; - if (i >= recv->v.arr.len) continue; - ScrDyn *item = scr_dyn_retain(recv->v.arr.items[i]); - ScrDyn *r = dyn_call_cb(cb, item, i, visible_recv); - if (!r) { scr_dyn_release(item); scr_dyn_release(out); return NULL; } - if ((mid == DYN_M_map)) { - scr_dyn_arr_push(out, r); /* ownership moves in */ - r = NULL; - } else { - bool truthy = scr_dyn_truthy(r); - scr_dyn_release(r); - if ((mid == DYN_M_filter) && truthy) scr_dyn_arr_push(out, scr_dyn_retain(item)); - if ((mid == DYN_M_some) && truthy) { scr_dyn_release(item); return scr_dyn_new_bool(true); } - if ((mid == DYN_M_every) && !truthy) { scr_dyn_release(item); return scr_dyn_new_bool(false); } - if ((mid == DYN_M_find) && truthy) return item; /* +1 moves out */ - if ((mid == DYN_M_findIndex) && truthy) { scr_dyn_release(item); return scr_dyn_new_num((double)i); } - } - scr_dyn_release(item); - } - if (out) return out; - if ((mid == DYN_M_some)) return scr_dyn_new_bool(false); - if ((mid == DYN_M_every)) return scr_dyn_new_bool(true); - if ((mid == DYN_M_find)) return scr_dyn_retain(scr_dyn_undefined()); - if ((mid == DYN_M_findIndex)) return scr_dyn_new_num(-1); - return scr_dyn_retain(scr_dyn_undefined()); /* forEach */ - } if ((mid == DYN_M_reduce) || (mid == DYN_M_reduceRight)) { if (!dyn_cb_check(args, argc)) return NULL; bool backward = (mid == DYN_M_reduceRight); @@ -1616,44 +1674,6 @@ static ScrDyn *scr_dyn_invoke_impl( scr_throw_error_msg(SCR_ERR_TYPE, message, sizeof message - 1); return NULL; } - if ((mid == DYN_M_flatMap)) { - /* JS Array.prototype.flatMap over a dyn array: map + a depth-1 - * flatten. Native dyn-array results flatten element-by-element; a - * WRAPPED engine array flattens through the routed keyed reads - * (elements wrap back scalar-normalized); everything else pushes - * as a single element (JS keeps non-array callback results whole). - * JS's spec snapshots the length up front (elements appended by - * the callback are not visited). */ - if (!dyn_cb_check(args, argc)) return NULL; - ScrDyn *cb = args[0]; - ScrDyn *out = scr_dyn_new_arr(); - size_t n = recv->v.arr.len; - for (size_t i = 0; i < n && i < recv->v.arr.len; i++) { - if (!dyn_borrow_present(recv, i)) { if (scr_exc_pending()) { scr_dyn_release(out); return NULL; } continue; } - ScrDyn *item = dyn_borrow_item(recv, i); - if (!item) { scr_dyn_release(out); return NULL; } - ScrDyn *r = dyn_call_cb( - cb, item, i, callback_recv ? callback_recv : recv); - scr_dyn_release(item); - if (!r) { scr_dyn_release(out); return NULL; } - if (r->kind == SCR_DYN_TYPED_REF && scr_dyn_ref_is_array(r)) { - ScrDyn *view = scr_dyn_typed_ref_materialize(r); - scr_dyn_release(r); - r = view; - if (!r) { scr_dyn_release(out); return NULL; } - } - if (r->kind == SCR_DYN_ARR) { - for (size_t j = 0; j < r->v.arr.len; j++) { - if (dyn_borrow_present(r, j)) scr_dyn_arr_push(out, dyn_borrow_item(r, j)); - if (scr_exc_pending()) { scr_dyn_release(r); scr_dyn_release(out); return NULL; } - } - scr_dyn_release(r); - } else { - scr_dyn_arr_push(out, r); /* ownership moves in */ - } - } - return out; - } if ((mid == DYN_M_sort)) { ScrDyn *cmp = argc > 0 ? args[0] : scr_dyn_undefined(); if (cmp->kind != SCR_DYN_UNDEF && cmp->kind != SCR_DYN_FUNC) { diff --git a/tests/corpus/array-prototype-call-native-arrays.ts b/tests/corpus/array-prototype-call-native-arrays.ts index ce198ff73..d5bfec507 100644 --- a/tests/corpus/array-prototype-call-native-arrays.ts +++ b/tests/corpus/array-prototype-call-native-arrays.ts @@ -20,3 +20,46 @@ console.log(words.join(","), Array.prototype.includes.call(words, "a")); const records: { id: number }[] = [{ id: 1 }, { id: 2 }, { id: 3 }]; Array.prototype.copyWithin.call(records, 1, 0, 1); console.log(records.map((r) => r.id).join(","), records[0] === records[1]); + +// The callback methods bind `this` to their thisArg, for native arrays +// through .call and for dynamic arrays called directly. +type Scale = { k: number; seen: number[] }; +const scale: Scale = { k: 10, seen: [] }; +const values: number[] = [1, 2, 3, 4]; +Array.prototype.forEach.call(values, function (this: Scale, v: number) { + this.seen.push(v); +}, scale); +console.log(scale.seen.join(",")); +console.log(Array.prototype.map.call(values, function (this: Scale, v: number) { + return v * this.k; +}, scale).join(",")); +console.log(Array.prototype.filter.call(values, function (this: Scale, v: number) { + return v * this.k > 15; +}, scale).join(",")); +console.log( + Array.prototype.some.call(values, function (this: Scale, v: number) { return v === this.k / 5; }, scale), + Array.prototype.every.call(values, function (this: Scale, v: number) { return v < this.k; }, scale), + Array.prototype.find.call(values, function (this: Scale, v: number) { return v * this.k === 30; }, scale), + Array.prototype.findIndex.call(values, function (this: Scale, v: number) { return v * this.k === 30; }, scale), + Array.prototype.findLast.call(values, function (this: Scale, v: number) { return v * this.k < 30; }, scale), + Array.prototype.findLastIndex.call(values, function (this: Scale, v: number) { return v * this.k < 30; }, scale), +); +console.log(Array.prototype.flatMap.call(values, function (this: unknown, v: number) { + return [v, v * (this as Scale).k]; +}, scale).join(",")); +const parsed: any = JSON.parse("[1, 2, 3]"); +const sum = { total: 0 }; +parsed.forEach(function (this: { total: number }, v: number) { + this.total += v; +}, sum); +console.log(sum.total, parsed.map(function (this: Scale, v: number) { return v * this.k; }, scale).join(",")); +class Counter { + total = 0; + addAll(items: number[]): number { + Array.prototype.forEach.call(items, (v: number) => { + this.total += v; + }); + return this.total; + } +} +console.log(new Counter().addAll(values)); From 6fa88c7243e273251e5d2814006d859213162080 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Mon, 26 Oct 2026 10:32:41 -0500 Subject: [PATCH 20/23] Observe every top-level mapping in the canonical shadow --- packages/compiler/src/frontend/type-mapper.ts | 13 +++++++++-- .../compiler/test/canon-normalize.test.ts | 23 +++++++++++++++++++ 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/packages/compiler/src/frontend/type-mapper.ts b/packages/compiler/src/frontend/type-mapper.ts index 72b29629b..b2fc774f2 100644 --- a/packages/compiler/src/frontend/type-mapper.ts +++ b/packages/compiler/src/frontend/type-mapper.ts @@ -1845,6 +1845,17 @@ function arrayClassBaseType( } export function mapType(type: ts.Type, ctx: TypeMapperCtx): IrType | null { + const shadow = ctx.canonShadow; + if (shadow === undefined || mapTypeDepth !== 0) return mapTypeEntry(type, ctx); + // Shadow mode observes every successful top-level mapping, including the + // intrinsic primitives, local class values and memo hits that return + // before the mapping proper. + const mapped = mapTypeEntry(type, ctx); + if (mapped !== null) shadow(type, mapped, ctx); + return mapped; +} + +function mapTypeEntry(type: ts.Type, ctx: TypeMapperCtx): IrType | null { if (mapTypeDepth >= MAP_TYPE_MAX_DEPTH) return null; // These intrinsic domains cannot depend on generic bindings, declaration // provenance, or recursive shape state. Enum literals retain the ordinary @@ -1915,8 +1926,6 @@ export function mapType(type: ts.Type, ctx: TypeMapperCtx): IrType | null { ) { ctx.typeMemo?.set(memoKey, mapped); } - if (mapped !== null && topLevel && ctx.canonShadow !== undefined) - ctx.canonShadow(type, mapped, ctx); return mapped; } finally { mapTypeDepth--; diff --git a/packages/compiler/test/canon-normalize.test.ts b/packages/compiler/test/canon-normalize.test.ts index c9140c6f3..c7b457dc8 100644 --- a/packages/compiler/test/canon-normalize.test.ts +++ b/packages/compiler/test/canon-normalize.test.ts @@ -297,6 +297,29 @@ describe("mapCanonical", () => { expect(strict).not.toBeNull(); expect(canonical && typeKey(canonical)).toBe(typeKey(strict!)); }); + test("shadow mode observes every successful top-level mapping", () => { + const seen: string[] = []; + const point = { kind: "object", className: "Point" } as const; + const observed: TypeMapperCtx = { + ...ctx(), + typeMemo: new Map(), + localClassValue: (type) => (type === aliases.get("Points") ? point : null), + canonShadow: (type, mapped) => seen.push(`${checker.typeToString(type)} ${typeKey(mapped)}`), + }; + // An intrinsic primitive, a mapping and its memo hit, and a local class + // value: one observation each; the nested mappings of `Same` are not + // observed. + const number = mapType(valueTypes.get("sampleNumber")!, observed); + const same = mapType(aliases.get("Same")!, observed); + expect(mapType(aliases.get("Same")!, observed)).toBe(same); + expect(mapType(aliases.get("Points")!, observed)).toBe(point); + expect(seen).toEqual([ + `number ${typeKey(number!)}`, + `Same ${typeKey(same!)}`, + `Same ${typeKey(same!)}`, + `Points ${typeKey(point)}`, + ]); + }); test("a generic alias maps under bindings, where mapType needs the instance table", () => { const { type, params } = asTypes.get("Catalog")!; const env = new Map([ From dec2b8a454b78da80343b83b9f0ade32d195a0af Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Mon, 26 Oct 2026 10:33:02 -0500 Subject: [PATCH 21/23] Record one probe for two reads of a file that saw the same content A read within IDENTITY_SETTLE_MS of a file's last write records no identity, and a later read of the same bytes in the same frontend does. FrontendInputTracker.record compared the two probes as JSON and marked the snapshot unstable. A build or coverage run that read its entry before and after that point then published no cache entry: the exact repeat of a build lowered again, and a coverage verdict was not replayed. Two file probes with the same digest now merge into one that keeps the settled identity; any other difference still marks the snapshot unstable. --- .../src/frontend/input-tracker.test.ts | 38 +++++++++++++++++++ .../compiler/src/frontend/input-tracker.ts | 25 +++++++++++- 2 files changed, 61 insertions(+), 2 deletions(-) diff --git a/packages/compiler/src/frontend/input-tracker.test.ts b/packages/compiler/src/frontend/input-tracker.test.ts index d6f9a91b7..a320eb2bd 100644 --- a/packages/compiler/src/frontend/input-tracker.test.ts +++ b/packages/compiler/src/frontend/input-tracker.test.ts @@ -316,3 +316,41 @@ test("a file changed shortly before the read is hashed on every check", async () expect(frontendInputsStillMatch(snapshot)).toBe(true); expect(reads.mock.calls.filter(([path]) => path === source)).toHaveLength(1); }); + +test("a read before and a read after a file settles record one probe", async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-inputs-")); + scratch.push(dir); + const source = join(dir, "entry.ts"); + await writeFile(source, "export const answer = 1;\n"); + const tracker = new FrontendInputTracker(); + // The first read comes right after the write, the second once the file + // has been unchanged long enough for its identity to stand for it. + tracker.run(() => trackedReadFile(source)); + vi.useFakeTimers({ toFake: ["Date"] }); + vi.setSystemTime(Date.now() + 5000); + try { + tracker.run(() => trackedReadFile(source)); + } finally { + vi.useRealTimers(); + } + const snapshot = tracker.snapshot(); + expect(snapshot.stable).toBe(true); + const probe = snapshot.probes.find((entry) => entry.op === "file"); + expect(probe?.op === "file" && probe.identity !== undefined).toBe(true); + expect(frontendInputsStillMatch(snapshot)).toBe(true); + // The identity still invalidates the snapshot on any write. + await writeFile(source, "export const answer = 2;\n"); + expect(frontendInputsStillMatch(snapshot)).toBe(false); +}); + +test("two reads of a file that saw different content prevent cache publication", async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-inputs-")); + scratch.push(dir); + const source = join(dir, "entry.ts"); + await writeFile(source, "export const answer = 1;\n"); + const tracker = new FrontendInputTracker(); + tracker.run(() => trackedReadFile(source)); + await writeFile(source, "export const answer = 2;\n"); + tracker.run(() => trackedReadFile(source)); + expect(tracker.snapshot().stable).toBe(false); +}); diff --git a/packages/compiler/src/frontend/input-tracker.ts b/packages/compiler/src/frontend/input-tracker.ts index c23a87313..18081576b 100644 --- a/packages/compiler/src/frontend/input-tracker.ts +++ b/packages/compiler/src/frontend/input-tracker.ts @@ -84,6 +84,22 @@ function readWithIdentity(path: string): { text: string; identity: string | null return { text, identity: settled && after === identity ? identity : null }; } +/** Two reads of one file that saw the same content, as one probe: a read + * before the file had been unchanged for IDENTITY_SETTLE_MS records no + * identity, and a later read of the same bytes does. The probe keeps the + * identity, which the settled read proved for that content. Null when the + * reads differ in anything else. */ +function sameFileContent( + previous: FrontendInputProbe, + probe: FrontendInputProbe, +): FrontendInputProbe | null { + if (previous.op !== "file" || probe.op !== "file" || previous.digest !== probe.digest) + return null; + if (previous.identity === undefined) return probe; + if (probe.identity === undefined || probe.identity === previous.identity) return previous; + return null; +} + function pathKind(path: string): Extract["kind"] { try { const info = statSync(path); @@ -142,12 +158,17 @@ export class FrontendInputTracker { const key = `${probe.op}\0${probe.path}`; if (probe.op === "text" && this.probes.has(key)) return; const previous = this.probes.get(key); - if (previous !== undefined && JSON.stringify(previous) !== JSON.stringify(probe)) { + const merged = previous === undefined ? null : sameFileContent(previous, probe); + if ( + merged === null && + previous !== undefined && + JSON.stringify(previous) !== JSON.stringify(probe) + ) { // An input changed while this frontend was running. The build may // still finish normally, but it is not safe to publish as a cache key. this.stable = false; } - this.probes.set(key, probe); + this.probes.set(key, merged ?? probe); if (probe.op === "file") { // A successful content read supersedes earlier existence probes for the // same path only when they also observed a file. A missing/non-file From 9b96314c20cf6bdd9757a64037522df644fc4b02 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Mon, 26 Oct 2026 10:42:41 -0500 Subject: [PATCH 22/23] Tear down self-recursive records and classes in one loop The teardown of a type with children of its own type called itself for each child: a call, its register saves and the segment bookkeeping per node, 45 instructions per node in a binary tree. A record or standalone class whose every reference field holds its own type and that has no collector header now tears down in one loop. A child whose last reference the dead node held goes onto a pending list linked through its own count word, which it no longer needs, and the loop takes the list's first node next, so nodes are visited parent first. Each node frees its block as before (dispose observers, the RC audit, blocks of other arenas), the calling thread's blocks go into the teardown segment in visit order, and the segment goes onto the free list in one step, so a structure built parent first takes the blocks in its allocation order. Shared and immortal children keep their release. The loop needs no depth budget. Other teardowns keep the recursive form. --- .../compiler/src/backend/llvm/classes.test.ts | 4 + packages/compiler/src/backend/llvm/classes.ts | 22 +++ .../backend/llvm/iterative-teardown.test.ts | 68 +++++++++ packages/compiler/src/backend/llvm/shapes.ts | 130 +++++++++++++++++- .../corpus/8400-iterative-teardown.ts.json | 6 + tests/corpus/8400-iterative-teardown.ts | 96 +++++++++++++ 6 files changed, 325 insertions(+), 1 deletion(-) create mode 100644 packages/compiler/src/backend/llvm/iterative-teardown.test.ts create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8400-iterative-teardown.ts.json create mode 100644 tests/corpus/8400-iterative-teardown.ts diff --git a/packages/compiler/src/backend/llvm/classes.test.ts b/packages/compiler/src/backend/llvm/classes.test.ts index f472eec60..e7ba72784 100644 --- a/packages/compiler/src/backend/llvm/classes.test.ts +++ b/packages/compiler/src/backend/llvm/classes.test.ts @@ -87,9 +87,13 @@ test("teardowns of types with children of their own type free their blocks as on classes: [ { name: "Linked", + // The array field keeps the recursive teardown: a type whose every + // reference field holds its own type tears down in one loop + // (iterative-teardown.test.ts). fields: [ { name: "next", type: linked }, { name: "other", type: linked }, + { name: "tags", type: { kind: "array", elem: { kind: "string" } } }, ], methods: [], loc, diff --git a/packages/compiler/src/backend/llvm/classes.ts b/packages/compiler/src/backend/llvm/classes.ts index 5ff7477f3..5941f5d5c 100644 --- a/packages/compiler/src/backend/llvm/classes.ts +++ b/packages/compiler/src/backend/llvm/classes.ts @@ -45,6 +45,7 @@ import { directDestroyTarget, segmentPrologueLines, segmentTeardownLines, + iterativeTeardownDefinitions, segmentedTeardown, usedDirectTargets, type DirectDestroy, @@ -1334,6 +1335,15 @@ export function emitClassShapes( } else { const freeBody: string[] = []; teardown(freeBody); + // A segmented class whose every reference field holds its own type and + // that has no collector header tears down in one loop (shapes.ts + // iterativeTeardownDefinitions). + const iterative = + segmented && + !traced && + refFields.every( + (f) => directDestroyTarget(host, f.type, directTargets)?.destroy === destroy, + ); defs.push( ...releaseBody( host, @@ -1345,6 +1355,18 @@ export function emitClassShapes( lifetime, chained, segmented, + false, + iterative + ? iterativeTeardownDefinitions( + host, + destroy, + struct, + refFields.map((f) => f.index), + `@${mangleClassRelease(cls.name)}`, + "weak", + count, + ) + : null, ), ``, ); diff --git a/packages/compiler/src/backend/llvm/iterative-teardown.test.ts b/packages/compiler/src/backend/llvm/iterative-teardown.test.ts new file mode 100644 index 000000000..befa258e8 --- /dev/null +++ b/packages/compiler/src/backend/llvm/iterative-teardown.test.ts @@ -0,0 +1,68 @@ +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { expect, test } from "vitest"; +import { compile, deserializeModule } from "../../index.js"; +import { emitLlvmModule } from "./emitter.js"; + +async function moduleFor(source: string) { + const dir = await mkdtemp(join(tmpdir(), "scriptc-iterative-teardown-")); + try { + const entry = join(dir, "main.ts"), + output = join(dir, "main.ir.json"); + await writeFile(entry, source); + const result = await compile(entry, { outDir: dir, outPath: output, outputKind: "ir" }); + if (!result.ok) + throw new Error(result.diagnostics.map((item) => `${item.code}: ${item.message}`).join("\n")); + return deserializeModule(await readFile(output, "utf8")); + } finally { + await rm(dir, { recursive: true, force: true }); + } +} + +const SOURCE = ` +interface TreeNode { left: TreeNode | null; right: TreeNode | null } +class ListNode { + next: ListNode | null; + value: number; + constructor(next: ListNode | null, value: number) { this.next = next; this.value = value; } +} +interface Labeled { child: Labeled | null; label: string[] } +function tree(d: number): TreeNode { return d > 0 ? { left: tree(d - 1), right: tree(d - 1) } : { left: null, right: null }; } +function size(n: TreeNode | null): number { return n === null ? 0 : 1 + size(n.left) + size(n.right); } +let list: ListNode | null = null; +for (let i = 0; i < 10; i++) list = new ListNode(list, i); +let labeled: Labeled | null = { child: { child: null, label: ["b"] }, label: ["a"] }; +console.log(size(tree(4)), list!.value, labeled!.label.length); +list = null; +labeled = null; +`; + +/** The definitions whose symbols end in `suffix`. */ +function definitions(ir: string, suffix: string): string[] { + return [ + ...ir.matchAll(new RegExp(`^define internal [^\\n]*@([\\w.$%-]+${suffix})\\(`, "gm")), + ].map((m) => m[1]!); +} + +test("self-recursive records and classes tear down in one loop", async () => { + const ir = emitLlvmModule(await moduleFor(SOURCE), { targetTriple: "x86_64-unknown-linux-gnu" }); + const loops = definitions(ir, "_destroy_it"); + // The tree record and the list class; the labeled record also holds an + // array, so it keeps the recursive teardown. + expect(loops.length).toBe(2); + expect(loops.some((name) => name.includes("ListNode"))).toBe(true); + for (const name of loops) { + const body = new RegExp( + `^define internal void @${name.replace(/[$.]/g, "\\$&")}\\(ptr %root\\)[^]*?^}`, + "m", + ).exec(ir)![0]; + // No call to a teardown: dead children go onto the pending list. + expect(body).not.toMatch(/call [^\n]*_destroy/); + expect(body).toContain("inttoptr"); + const base = name.slice(0, -"_it".length); + expect(ir).toContain(`define internal void @${base}(ptr %o)`); + expect(ir).not.toContain(`@${base}_s(`); + } + expect(definitions(ir, "_destroy_s").length).toBe(1); +}); diff --git a/packages/compiler/src/backend/llvm/shapes.ts b/packages/compiler/src/backend/llvm/shapes.ts index 5b76886b2..ea429d71a 100644 --- a/packages/compiler/src/backend/llvm/shapes.ts +++ b/packages/compiler/src/backend/llvm/shapes.ts @@ -35,6 +35,7 @@ import { emitObjectAlloc, emitObjectFree, emitSegmentFree, + type FreeObservers, CYC_APPEND, cycleAppendDecls, cycleAppendHelper, @@ -1820,6 +1821,115 @@ function sizeLiteral(sizeType: RcCount, value: bigint): string { return BigInt.asIntN(sizeType === "i64" ? 64 : 32, value).toString(); } +/** The teardown of a type whose reference fields all hold its own type + * (`fields`: their struct indices; `release`: the type's inlinable release; + * no collector header): one loop instead of a call per node. A child whose + * last reference the dead node held joins a pending list linked through its + * own count word, which it no longer needs; the loop takes the list's first + * node next, so nodes are visited parent first, fields in order. Each node + * releases its block as emitObjectFree does, keeping a block of the calling + * thread's arena in a segment linked through its first word, and the segment + * goes onto the free list in one step at the end (emitSegmentFree): the next + * structure built parent first takes the blocks in the order it allocates + * them, as segmentTeardownLines arranges for the recursive form. The loop + * needs no depth budget. `_d` (the direct-chain entry other teardowns call) + * and the plain entry both run it. A class that keeps its id in the count + * word counts in the low half (`count`); a dead node's whole word is free. */ +export function iterativeTeardownDefinitions( + host: ShapeHost, + destroy: string, + struct: string, + fields: readonly number[], + release: string, + observers: FreeObservers, + count: RcCount = host.sizeType, +): string[] { + const S = host.sizeType; + const lines = [ + `define internal void @${destroy}_it(ptr %root) ${FN_ATTRS} {`, + `entry:`, + ` %sg.own = alloca ptr`, + ` br label %it.node`, + `it.node:`, + ` %o = phi ptr [ %root, %entry ], [ %it.top, %it.next ]`, + ` %it.pend = phi ptr [ null, %entry ], [ %it.rest, %it.next ]`, + ` %it.first = phi ptr [ null, %entry ], [ %it.fa, %it.next ]`, + ` %it.last = phi ptr [ null, %entry ], [ %it.la, %it.next ]`, + ...fields.flatMap((index, i) => [ + ` %it.f${i} = getelementptr inbounds %${struct}, ptr %o, i64 0, i32 ${index}`, + ` %it.v${i} = load ptr, ptr %it.f${i}`, + ]), + ` store ptr null, ptr %sg.own`, + ...emitObjectFree(host, false, observers, "%sg.own"), + ` br label %it.freed`, + `it.freed:`, + ` %it.k = load ptr, ptr %sg.own`, + ` %it.kept = icmp ne ptr %it.k, null`, + ` br i1 %it.kept, label %it.app, label %it.kids`, + `it.app:`, + ` %it.none = icmp eq ptr %it.first, null`, + ` br i1 %it.none, label %it.appd, label %it.link`, + `it.link:`, + ` store ptr %it.k, ptr %it.last`, + ` br label %it.appd`, + `it.appd:`, + ` %it.nf = select i1 %it.none, ptr %it.k, ptr %it.first`, + ` br label %it.kids`, + `it.kids:`, + ` %it.fa = phi ptr [ %it.nf, %it.appd ], [ %it.first, %it.freed ]`, + ` %it.la = phi ptr [ %it.k, %it.appd ], [ %it.last, %it.freed ]`, + ]; + // The fields go onto the list last first, so the first field is next. + let pend = "%it.pend"; + let from = "it.kids"; + for (let i = fields.length - 1; i >= 0; i--) { + const v = `%it.v${i}`; + lines.push( + ` %it.nn${i} = icmp ne ptr ${v}, null`, + ` br i1 %it.nn${i}, label %it.chk${i}, label %it.j${i}`, + `it.chk${i}:`, + ` %it.rc${i} = load ${count}, ptr ${v}`, + ` %it.last${i} = icmp eq ${count} %it.rc${i}, 1`, + ` br i1 %it.last${i}, label %it.push${i}, label %it.sh${i}`, + `it.push${i}:`, + ` %it.pl${i} = ptrtoint ptr ${pend} to ${S}`, + ` store ${S} %it.pl${i}, ptr ${v}`, + ` br label %it.j${i}`, + `it.sh${i}:`, + ` call void ${release}(ptr ${v}) ; a shared or immortal child`, + ` br label %it.j${i}`, + `it.j${i}:`, + ` %it.p${i} = phi ptr [ ${pend}, %${from} ], [ ${v}, %it.push${i} ], [ ${pend}, %it.sh${i} ]`, + ); + pend = `%it.p${i}`; + from = `it.j${i}`; + } + lines.push( + ` %it.empty = icmp eq ptr ${pend}, null`, + ` br i1 %it.empty, label %it.done, label %it.next`, + `it.next:`, + ` %it.top = getelementptr i8, ptr ${pend}, i64 0`, + ` %it.lw = load ${S}, ptr %it.top`, + ` %it.rest = inttoptr ${S} %it.lw to ptr`, + ` br label %it.node`, + `it.done:`, + ...emitSegmentFree(host, "sgf", "%it.fa", "%it.la"), + ` ret void`, + `}`, + `define internal void @${destroy}_d(ptr %o, i32 %d) ${FN_ATTRS} {`, + `entry:`, + ` call void @${destroy}_it(ptr %o)`, + ` ret void`, + `}`, + `define internal void @${destroy}(ptr %o) ${FN_ATTRS} {`, + `entry:`, + ` call void @${destroy}_it(ptr %o)`, + ` ret void`, + `}`, + ); + return lines; +} + /** Common NULL/immortal/decrement skeleton for ordinary object releases. * `freeBody` owns the zero-ref teardown and must leave the current block at * its end; traced objects also get the possible-cycle-root branch. @@ -1838,6 +1948,7 @@ export function releaseBody( chained = false, segmented = false, frozen = false, + destroyDefs: readonly string[] | null = null, ): string[] { const S = host.sizeType; if (bounded) host.declare(`declare void @scr_rc_destroy(ptr, ptr)`); @@ -1845,7 +1956,9 @@ export function releaseBody( // A freezable record's count is the word without its frozen bit. const notFrozen = sizeLiteral(S, ~recordFrozenBit(S)); const lines = [ - ...(bounded ? destroyDefinitions(destroy, freeBody, chained, segmented ? host : null) : []), + ...(bounded + ? (destroyDefs ?? destroyDefinitions(destroy, freeBody, chained, segmented ? host : null)) + : []), ...releaseFastPath(host, fnName, `${fnName}_slow`, traced, comment, immortal, S, frozen), `define internal void @${fnName}_slow(ptr %o) noinline ${FN_ATTRS} {${comment ? ` ; ${comment}` : ""}`, `entry:`, @@ -2141,6 +2254,11 @@ export function emitRecordShapes( ), ); else freeBody.push(...emitObjectFree(host, traced, observers)); + // A segmented shape whose every reference field holds its own type, with + // no collector header and no frozen bit in its count, tears down in one + // loop (iterativeTeardownDefinitions). + const iterative = + segmented && !traced && shape.freezable !== true && ownType.length === refMembers.length; defs.push( ...releaseBody( host, @@ -2153,6 +2271,16 @@ export function emitRecordShapes( chained, segmented, shape.freezable === true, + iterative + ? iterativeTeardownDefinitions( + host, + destroy, + struct, + refMembers.map((m) => m.index), + `@${mangleRecordRelease(shape.id)}`, + observers, + ) + : null, ), ``, ); diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8400-iterative-teardown.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8400-iterative-teardown.ts.json new file mode 100644 index 000000000..80ad38bdf --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8400-iterative-teardown.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/8400-iterative-teardown.ts" + ], + "diags": [] +} diff --git a/tests/corpus/8400-iterative-teardown.ts b/tests/corpus/8400-iterative-teardown.ts new file mode 100644 index 000000000..69ec13824 --- /dev/null +++ b/tests/corpus/8400-iterative-teardown.ts @@ -0,0 +1,96 @@ +// Teardown of self-recursive records and classes: shared and immortal +// children, deep chains, weak-collection keys, and trees rebuilt from freed +// blocks. + +interface TreeNode { + left: TreeNode | null; + right: TreeNode | null; +} + +class ListNode { + next: ListNode | null; + value: number; + constructor(next: ListNode | null, value: number) { + this.next = next; + this.value = value; + } +} + +class Pair { + a: Pair | null; + b: Pair | null; + constructor(a: Pair | null, b: Pair | null) { + this.a = a; + this.b = b; + } +} + +function build(depth: number): TreeNode { + return depth > 0 ? { left: build(depth - 1), right: build(depth - 1) } : { left: null, right: null }; +} + +function count(node: TreeNode | null): number { + if (node === null) return 0; + return 1 + count(node.left) + count(node.right); +} + +// Trees torn down and rebuilt from their freed blocks. +let total = 0; +for (let depth = 0; depth <= 12; depth++) { + for (let i = 0; i < 4; i++) total += count(build(depth)); +} +console.log("trees", total); + +// A child shared by two parents survives the first parent's teardown. +const shared = build(3); +let left: TreeNode | null = { left: shared, right: build(2) }; +let right: TreeNode | null = { left: build(1), right: shared }; +left = null; +console.log("shared after first parent", count(shared), count(right)); +right = null; +console.log("shared after second parent", count(shared)); + +// A node held from outside keeps its subtree when the tree around it dies. +let outer: TreeNode | null = build(6); +const kept = outer.left!.right!; +outer = null; +console.log("kept subtree", count(kept)); + +// A chain far deeper than any call stack. +let head: ListNode | null = null; +for (let i = 0; i < 2000000; i++) head = new ListNode(head, i); +let sum = 0; +for (let n = head; n !== null; n = n.next) sum += n.value % 7; +head = null; +console.log("chain", sum); + +// A list whose middle node is still referenced keeps the tail from it. +let list: ListNode | null = null; +let middle: ListNode | null = null; +for (let i = 0; i < 1000; i++) { + list = new ListNode(list, i); + if (i === 500) middle = list; +} +list = null; +let tail = 0; +for (let n = middle; n !== null; n = n.next) tail++; +console.log("tail from middle", tail); +middle = null; + +// Nodes used as WeakMap keys: their entries go away when the tree dies. +const weak = new WeakMap(); +let root: Pair | null = new Pair(new Pair(null, null), new Pair(new Pair(null, null), null)); +const probe = root.b!; +weak.set(root, 1); +weak.set(root.a!, 2); +weak.set(probe, 3); +root = null; +console.log("weak probe", weak.get(probe), weak.has(probe)); + +// Immortal leaves shared by many nodes. +const leaf: TreeNode = { left: null, right: null }; +let wide: TreeNode | null = { left: leaf, right: leaf }; +for (let i = 0; i < 100; i++) wide = { left: wide, right: leaf }; +console.log("wide", count(wide)); +wide = null; +console.log("leaf", count(leaf)); From e7f3ab29c289111d9327a7b2c6de5d25b6e33bd0 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Mon, 26 Oct 2026 10:59:32 -0500 Subject: [PATCH 23/23] Track a stream's 'data' listeners instead of looking them up per chunk Each chunk a readable side delivered looked up its 'data' listeners by name twice: once to decide whether a flowing stream emits directly, and again in emit('data'), which scans the stream's event names even when a piped stream has no 'data' listener at all. The stream now keeps a data_listeners flag, set from the emitter registry's empty/nonempty transitions through scr_emitter_observe, and reads it on those paths; emit('data') is skipped when the stream has no 'data' listener. --- packages/runtime/src/scr_stream.c | 39 ++++++++++++++++++++++++++----- 1 file changed, 33 insertions(+), 6 deletions(-) diff --git a/packages/runtime/src/scr_stream.c b/packages/runtime/src/scr_stream.c index 2f4ad185e..fdd7179e5 100644 --- a/packages/runtime/src/scr_stream.c +++ b/packages/runtime/src/scr_stream.c @@ -119,6 +119,10 @@ struct ScrStreamState { bool reading, in_read_sync, ended, end_emitted, end_scheduled; bool in_read_call; /* inside the user's _read (in_read_sync also starts true) */ bool need_readable, emitted_readable, readable_listening; + /* the stream has 'data' listeners: kept by scr_stream_listener_observe + * from the registry's empty/nonempty transitions, so the per-chunk + * paths read it instead of looking the name up */ + bool data_listeners; bool resume_scheduled, maybe_more_scheduled; /* The buffer's representation: encoded means every entry is a ScrStr, * otherwise every entry is a ScrBytes (scr_stream_rbuf_push checks @@ -881,10 +885,14 @@ static void scr_stream_emit_data(ScrStream *s, void *chunk, ScrEntryKind kind) { } else { b = chunk; } - ScrStr *n = (ScrStr *)&scr_stream_lit_data; - scr_emitter_emit((ScrEmitter *)s, n, b, str, value); - scr_str_release(n); - if (scr_exc_pending()) return; + /* emit('data') with no listeners does nothing; a piped stream delivers + * through st->pipes below */ + if (st->r.data_listeners) { + ScrStr *n = (ScrStr *)&scr_stream_lit_data; + scr_emitter_emit((ScrEmitter *)s, n, b, str, value); + scr_str_release(n); + if (scr_exc_pending()) return; + } /* pipe delivery, with backpressure: a full destination pauses us until * its 'drain'. Encoded sources write the string (utf8 bytes at the * destination — Node writes the decoded string through). */ @@ -1033,7 +1041,7 @@ static bool scr_stream_add_chunk(ScrStream *s, ScrBytes *chunk, bool front) { } if (entry_len > 0 && st->r.flowing == 1 && st->r.length == 0 && !st->r.in_read_sync && !front && - (scr_emitter_has((ScrEmitter *)s, "data") || st->pipes.n > 0)) { + (st->r.data_listeners || st->pipes.n > 0)) { /* the direct-emit fast path: flowing, empty buffer, async push */ if (st->r.emitted_readable) st->r.emitted_readable = false; ScrEntryKind kind = scr_stream_rkind(st); @@ -2002,7 +2010,7 @@ static int scr_stream_add_entry(ScrStream *s, void *entry /*moves*/, ScrEntryKin } st->r.reading = false; if (st->r.flowing == 1 && st->r.length == 0 && !st->r.in_read_sync && - (scr_emitter_has((ScrEmitter *)s, "data") || st->pipes.n > 0)) { + (st->r.data_listeners || st->pipes.n > 0)) { if (st->r.emitted_readable) st->r.emitted_readable = false; scr_stream_emit_data(s, entry, kind); scr_stream_entry_release_kind(entry, kind); @@ -2501,6 +2509,20 @@ static ScrStreamState *scr_stream_state_new(bool has_r, bool has_w, return st; } +/* The registry's empty/nonempty transitions for the names the per-chunk + * paths test (scr_emitter_observe): no script runs here. */ +static void scr_stream_listener_observe(void *context, const ScrStr *name, bool present) { + ScrStream *s = context; + if (s->st != NULL && name->len == 4 && memcmp(name->data, "data", 4) == 0) + s->st->r.data_listeners = present; +} + +/* Starts tracking the stream's 'data' listeners, once its state exists. */ +static void scr_stream_watch_listeners(ScrStream *s) { + s->st->r.data_listeners = scr_emitter_has((ScrEmitter *)s, "data"); + scr_emitter_observe((ScrEmitter *)s, &scr_stream_listener_observe, s); +} + static ScrStream *scr_stream_alloc(const ScrVt *vt, const char *cls, bool has_r, bool has_w, double rhwm, double whwm, bool auto_destroy, bool emit_close, bool allow_half_open) { @@ -2529,6 +2551,7 @@ static ScrStream *scr_stream_alloc(const ScrVt *vt, const char *cls, bool has_r, scr_emitter_reserve(em, "end"); scr_emitter_reserve(em, "readable"); } + scr_stream_watch_listeners(s); scr_obj_alloc_note(); return s; } @@ -2651,6 +2674,7 @@ void scr_stream_init_readable(ScrStream *s, double hwm, bool auto_destroy, bool ScrClosure *destroy, ScrStreamErrInv destroy_inv) { ScrStreamState *st = scr_stream_state_new(true, false, hwm, -1, auto_destroy, emit_close, true); s->st = st; + scr_stream_watch_listeners(s); st->r.read_cb = read; st->r.read_inv = read_inv; st->destroy_cb = destroy; @@ -2663,6 +2687,7 @@ void scr_stream_init_writable(ScrStream *s, double hwm, bool auto_destroy, bool ScrClosure *destroy, ScrStreamErrInv destroy_inv) { ScrStreamState *st = scr_stream_state_new(false, true, -1, hwm, auto_destroy, emit_close, true); s->st = st; + scr_stream_watch_listeners(s); st->w.write_cb = write; st->w.write_inv = write_inv; st->w.final_cb = final_cb; @@ -2680,6 +2705,7 @@ void scr_stream_init_duplex(ScrStream *s, double rhwm, double whwm, bool auto_de ScrStreamState *st = scr_stream_state_new(readable_side, writable_side, rhwm, whwm, auto_destroy, emit_close, allow_half_open); s->st = st; + scr_stream_watch_listeners(s); if (!readable_side) { st->r.ended = true; st->r.end_emitted = true; @@ -2706,6 +2732,7 @@ static void scr_stream_init_transformish(ScrStream *s, bool passthrough, ScrStreamState *st = scr_stream_state_new(true, true, rhwm, whwm, auto_destroy, emit_close, allow_half_open); s->st = st; + scr_stream_watch_listeners(s); st->is_transform = !passthrough; st->passthrough = passthrough; /* the Transform constructor's readable-half stance (see new_transformish) */