From 0d4954f40c62619fbae706d6b1a0eec7100e6856 Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Sat, 24 Oct 2026 21:21:11 -0700 Subject: [PATCH 1/4] Never let the shape overlay break compilation With the overlay on, a lowering refusal no longer fails the build. The annotations the refusal depends on are dropped and the program is loaded and lowered again, so those declarations compile untyped, as they do without the overlay: - Each overlay annotation carries an id (`@scriptcShape #`). - shape-inference/fallback.ts blames the annotations of the declarations enclosing a refusal and of the declarations its expression names, following unannotated functions' returns. - Rounds that stop converging drop whole files' overlays, then the overlay for the build. SC6103 reports what was dropped. Also: - The tsc gate's project world is built over the files as written; it used to see the overlay's files, which failed the typecheck of three npm cases. - Arrays a builtin creates (match results, `new Array(n)`) are no longer typed by the overlay: they may have holes, and the library types them. - compile() accepts `shapeOverlay: true`. tests/harness/npm-shape-overlay.test.ts runs the whole npm suite with the overlay on (npm.test.ts's body moves to npm-suite.ts): every case builds and runs as in the default build, and a refused build that compiles with the overlay matches Node. Co-Authored-By: Claude Opus 5.5 --- packages/compiler/src/compile-types.ts | 3 +- .../src/frontend/lowering/call-signatures.ts | 6 +- .../src/frontend/lowering/lower-classes.ts | 5 +- packages/compiler/src/frontend/pipeline.ts | 90 +++++- packages/compiler/src/frontend/program.ts | 41 ++- .../src/frontend/shape-inference/fallback.ts | 242 +++++++++++++++ .../src/frontend/shape-inference/index.ts | 113 ++++++- .../src/frontend/shape-inference/overlay.ts | 69 +++-- .../compiler/src/frontend/ts7/program-host.ts | 5 + packages/compiler/src/index.ts | 12 +- scripts/ci-test-plan.mjs | 1 + tests/harness/npm-shape-overlay.test.ts | 7 + tests/harness/npm-suite.ts | 280 ++++++++++++++++++ tests/harness/npm.test.ts | 261 +--------------- 14 files changed, 829 insertions(+), 306 deletions(-) create mode 100644 packages/compiler/src/frontend/shape-inference/fallback.ts create mode 100644 tests/harness/npm-shape-overlay.test.ts create mode 100644 tests/harness/npm-suite.ts diff --git a/packages/compiler/src/compile-types.ts b/packages/compiler/src/compile-types.ts index 2c9ba44d7..c0b4797a4 100644 --- a/packages/compiler/src/compile-types.ts +++ b/packages/compiler/src/compile-types.ts @@ -56,7 +56,8 @@ export interface CompileBaseOptions { /** Eligible packages compile automatically. Explicit names override automatic selection heuristics. */ npmStatic?: readonly string[] | "auto"; /** Shape inference's typed overlay for opted-in packages' JavaScript. - * False disables it (`--no-shape-overlay`); a package opts out with + * True enables it, false disables it (`--no-shape-overlay`); unset + * follows SCRIPTC_SHAPE_OVERLAY. A package opts out with * `"scriptc": { "packages": { "": { "shapes": false } } }`. */ shapeOverlay?: boolean; /** Outbound native FFI manifest. Its signature-only TypeScript bindings diff --git a/packages/compiler/src/frontend/lowering/call-signatures.ts b/packages/compiler/src/frontend/lowering/call-signatures.ts index 9d8096c62..3b2a4bd39 100644 --- a/packages/compiler/src/frontend/lowering/call-signatures.ts +++ b/packages/compiler/src/frontend/lowering/call-signatures.ts @@ -182,7 +182,11 @@ export function checkedJsDocumentedParameter( lowerer: Lowerer, param: ts.ParameterDeclaration, ): boolean { - if (param.parent !== undefined && shapeOverlayAnnotated(param.parent) && documentedParameter(param)) + if ( + param.parent !== undefined && + shapeOverlayAnnotated(param.parent) && + documentedParameter(param) + ) return true; return lowerer.checkedJsFile(ts.getSourceFileOfNode(param)) && documentedParameter(param); } diff --git a/packages/compiler/src/frontend/lowering/lower-classes.ts b/packages/compiler/src/frontend/lowering/lower-classes.ts index 573bb0c69..b2036dc8a 100644 --- a/packages/compiler/src/frontend/lowering/lower-classes.ts +++ b/packages/compiler/src/frontend/lowering/lower-classes.ts @@ -1156,7 +1156,10 @@ function jsFieldInitializerType(lowerer: Lowerer, type: IrType, expression: ts.E // Shape inference's overlay types a field as a record or array only when // the analysis proves its values never reach code that could observe a // copy, so those fields keep their native layout. - if ((type.kind === "array" || type.kind === "record") && !shapeOverlayAnnotated(expression.parent?.parent ?? expression)) + if ( + (type.kind === "array" || type.kind === "record") && + !shapeOverlayAnnotated(expression.parent?.parent ?? expression) + ) return DYN; // JavaScript documentation cannot guarantee that a call or an indexed // read returns a present number. Numeric fields receiving those values diff --git a/packages/compiler/src/frontend/pipeline.ts b/packages/compiler/src/frontend/pipeline.ts index 92bafe54d..48d4161f9 100644 --- a/packages/compiler/src/frontend/pipeline.ts +++ b/packages/compiler/src/frontend/pipeline.ts @@ -1,6 +1,11 @@ -import { shapeOverlayReport } from "./shape-inference/index.js"; /** Shared frontend orchestration. Hosts supply a program loader so Node * and native callers use the same npm selection, preflight and lowering. */ +import { + beginShapeOverlayBuild, + excludeRefusedOverlay, + shapeOverlayActive, + shapeOverlayReport, +} from "./shape-inference/index.js"; import type { ScrDiagnostic } from "../diagnostics/diagnostic.js"; import type { SrcLoc } from "../ir/ir.js"; import type { NpmStaticStatus } from "../coverage/report.js"; @@ -445,6 +450,10 @@ export function runFrontend( } } +/** Rounds of dropping single refused overlay annotations before the + * fallback drops whole files' overlays. */ +const OVERLAY_ANNOTATION_ROUNDS = 4; + function loadFrontend( entryPath: string, loadProgram: ProgramLoader, @@ -458,6 +467,7 @@ function loadFrontend( // separate compiles in a long-lived process. A cache miss must observe // package.json edits before it can publish a new early-library entry. clearResolveCaches(); + beginShapeOverlayBuild(); const statuses: NpmStaticStatus[] = []; const npmSites = new Map(); // Explicit library attempts bypass only the auto-selection heuristics. @@ -650,7 +660,9 @@ function loadFrontend( } } - const finalLoad = load; + // The lowering may reload once the overlay drops refused annotations. + let finalLoad = load; + const finalNpmStatic = [...effective]; return { preflight, entryText: () => finalLoad.entry.text, @@ -688,24 +700,74 @@ function loadFrontend( ].map((sf) => [sf.fileName, sf.text]), ), lower: (opts) => { - const lowered = lowerToIr(finalLoad.program, finalLoad.entry, finalLoad.moduleOrder, { - ...opts, - frontendServices: finalLoad.services, - startupCrash: finalLoad.startupCrash ?? null, - inlineRequireCycleCalls: finalLoad.inlineRequireCycleCalls, - moduleEdges: finalLoad.moduleEdges, - projectCheckJs: finalLoad.typeWorld?.projectCheckJs ?? false, - externalTypes: finalLoad.externalTypes, - externalTypeSpecifiersByFile: finalLoad.externalTypeSpecifiersByFile, - }); + const lowerOnce = (): LowerResult => + lowerToIr(finalLoad.program, finalLoad.entry, finalLoad.moduleOrder, { + ...opts, + frontendServices: finalLoad.services, + startupCrash: finalLoad.startupCrash ?? null, + inlineRequireCycleCalls: finalLoad.inlineRequireCycleCalls, + moduleEdges: finalLoad.moduleEdges, + projectCheckJs: finalLoad.typeWorld?.projectCheckJs ?? false, + externalTypes: finalLoad.externalTypes, + externalTypeSpecifiersByFile: finalLoad.externalTypeSpecifiersByFile, + }); + let lowered = lowerOnce(); + // Shape inference never breaks compilation (shape-inference/ + // fallback.ts): refusals drop the overlay annotations they depend on + // and the program reloads. Rounds that stop converging escalate to + // the overlay of every file involved, then to no overlay at all. + // Refusals deferred to run time (runtime fences) drop only the + // annotations they are blamed on: a fence the overlay did not cause + // stays as it is without the overlay. + let level: "annotations" | "files" | "all" | "fences" = "annotations"; + for (let round = 0; shapeOverlayActive(); round++) { + const failed = lowered.diagnostics.length > 0; + const refusals = failed ? lowered.diagnostics : lowered.runtimeFences; + if (refusals.length === 0) break; + let changed: boolean; + if (!failed) { + level = "fences"; + changed = + round < OVERLAY_ANNOTATION_ROUNDS && + excludeRefusedOverlay(finalLoad.program, refusals, level); + } else { + if (level === "fences") level = "annotations"; + if (round >= OVERLAY_ANNOTATION_ROUNDS && level === "annotations") level = "files"; + else if (round > OVERLAY_ANNOTATION_ROUNDS) level = "all"; + changed = excludeRefusedOverlay(finalLoad.program, refusals, level); + while (!changed && level !== "all") { + level = level === "annotations" ? "files" : "all"; + changed = excludeRefusedOverlay(finalLoad.program, refusals, level); + } + } + if (process.env["SCRIPTC_DEBUG_SHAPE_FALLBACK"] !== undefined) + process.stderr.write( + `[shape-fallback] round ${round} level ${level}: ${refusals.length} refusals (${refusals + .slice(0, 3) + .map((d) => `${d.loc.file.split("/").pop()}:${d.loc.start} ${d.message.slice(0, 60)}`) + .join("; ")}), changed ${changed}\n`, + ); + if (!changed) break; + finalLoad.dispose(); + finalLoad = loadProgram(entryPath, { npmStatic: finalNpmStatic, externalTypes }); + // The preflight fills the module order; the tsc world reads the + // files as written, so it answers as the first load did. + const reloadPreflight = checkPreflight(finalLoad); + if (reloadPreflight.length > 0) + return { ...lowered, module: null, diagnostics: reloadPreflight }; + lowered = lowerOnce(); + } // The declaration-typed boundary's notes (SC6102) join the // performance warnings; they never fail a build. - const notes = [...(finalLoad.declBoundaryNotes ?? []), ...shapeOverlayReport(finalLoad.entry.fileName)]; + const notes = [ + ...(finalLoad.declBoundaryNotes ?? []), + ...shapeOverlayReport(finalLoad.entry.fileName), + ]; if (notes.length === 0) return lowered; return { ...lowered, performance: [...notes, ...(lowered.performance ?? [])] }; }, npmStatic: statuses, npmImportSites: npmSites, - dispose: finalLoad.dispose, + dispose: () => finalLoad.dispose(), }; } diff --git a/packages/compiler/src/frontend/program.ts b/packages/compiler/src/frontend/program.ts index d44c837af..2dfc46bdc 100644 --- a/packages/compiler/src/frontend/program.ts +++ b/packages/compiler/src/frontend/program.ts @@ -1012,6 +1012,7 @@ function loadProgram7( // A load without the overlay must not see an earlier load's overlay // files (the registry is per process). resetShapeOverlay(); + const overlayOriginals = new Map(); if (overlayer !== undefined && jsx.refusal === null) { // Rounds: structural rewrites, then the typed overlay of the rebuilt // program (shape-inference/index.ts). @@ -1019,15 +1020,23 @@ function loadProgram7( const overlay = overlayer(program, round); if (overlay === null || overlay.size === 0) break; const before = program; - for (const [file, text] of overlay) host.addVirtualFile(file, text); - program = host.createProgram([...programRoots, overridesDtsPath()], options); + // The tsc gate reads the files as written: its diagnostics are + // taken before the overlay's files replace them on the host, and + // its project world (load.projectWorld()) is built over the files + // as written. if (tscWorld === undefined) tscWorld = { program: before, diagnostics: ts.getPreEmitDiagnostics(before), projectWorld: null, + declWorldFallback: true, }; - else if (tscWorld.program !== before) before.dispose(); + for (const file of overlay.keys()) + if (!overlayOriginals.has(file)) + overlayOriginals.set(file, before.getSourceFile(file)?.text ?? null); + for (const [file, text] of overlay) host.addVirtualFile(file, text); + program = host.createProgram([...programRoots, overridesDtsPath()], options); + if (tscWorld.program !== before) before.dispose(); if (shapeOverlayDone()) break; } } @@ -1051,7 +1060,10 @@ function loadProgram7( externalTypeSpecifiersByFile, ...(tscWorld === undefined ? {} : { jsxSources: jsx.sources }), ...(tscWorld === undefined ? {} : { tscWorld }), - projectWorld: () => (projectWorld ??= host.createProgram(programRoots, options)), + projectWorld: () => + (projectWorld ??= withFilesAsWritten(host, overlayOriginals, () => + host.createProgram(programRoots, options), + )), disposeAll: () => { try { projectWorld?.dispose(); @@ -1079,6 +1091,27 @@ export interface ProgramLoadOptions { externalTypes?: readonly (readonly [string, string])[]; } +/** Runs `build` with the overlay's files served as written (their original + * texts), then serves the overlay's texts again. */ +function withFilesAsWritten( + host: ts.Ts7Host, + originals: ReadonlyMap, + build: () => T, +): T { + if (originals.size === 0) return build(); + const overlaid = new Map(); + for (const [file, text] of originals) { + const current = host.virtualFileText(file); + if (current !== undefined) overlaid.set(file, current); + if (text !== null) host.addVirtualFile(file, text); + } + try { + return build(); + } finally { + for (const [file, text] of overlaid) host.addVirtualFile(file, text); + } +} + export function loadProgram( entryPath: string, services: FrontendServices, diff --git a/packages/compiler/src/frontend/shape-inference/fallback.ts b/packages/compiler/src/frontend/shape-inference/fallback.ts new file mode 100644 index 000000000..6a2824990 --- /dev/null +++ b/packages/compiler/src/frontend/shape-inference/fallback.ts @@ -0,0 +1,242 @@ +/* Shape inference never breaks compilation: when the lowering refuses code + * with the overlay on, the overlay annotations the refusal depends on are + * dropped and the program is loaded and lowered again, so those + * declarations compile untyped (as they do without the overlay). + * + * Attribution reads the refused program. A refusal depends on: + * - the annotations of the declarations enclosing it (an annotated + * function's parameter types, a typed local, a receiver cast); + * - the annotations of the declarations its expression names (a method + * whose annotated return type the refused member access reads, a field + * whose annotated type the access sees), followed through the return + * expressions of unannotated functions, whose result types the checker + * infers from what they return. + * + * pipeline.ts escalates when a round does not converge: first the + * overlay of every file involved, then the whole overlay for the build. */ +import * as ts from "../ts7/adapter.js"; +import type { ScrDiagnostic } from "../../diagnostics/diagnostic.js"; +import { OVERLAY_MARK, overlayAnnotationIds } from "./overlay.js"; + +export interface OverlayAttribution { + /** Annotation ids per overlay file. */ + readonly annotations: Map>; + /** Overlay files a refusal involves (where it sits, or what it names). */ + readonly files: Set; + /** Refusals with no overlay annotation or file to blame. */ + readonly unattributed: number; +} + +/** How far the attribution follows unannotated functions' return + * expressions. */ +const RETURN_DEPTH = 3; + +export function attributeOverlayRefusals( + program: ts.Program, + diagnostics: readonly ScrDiagnostic[], + overlayFiles: ReadonlySet, +): OverlayAttribution { + const annotations = new Map>(); + const files = new Set(); + let unattributed = 0; + let checker: ts.TypeChecker | null = null; + // Blamed annotations and files so far (a refusal that adds none is + // unattributed). + let blamed = 0; + const addFile = (file: string): void => { + if (files.has(file)) return; + files.add(file); + blamed++; + }; + const add = (file: string, id: number): void => { + let set = annotations.get(file); + if (set === undefined) annotations.set(file, (set = new Set())); + if (!set.has(id)) blamed++; + set.add(id); + addFile(file); + }; + // The ids in the trivia before a node (where the overlay writes them). + const marksOf = (node: ts.Node): number[] => { + const sf = node.getSourceFile(); + if (!overlayFiles.has(sf.fileName)) return []; + const trivia = sf.text.slice(node.pos, ts.tokenStartOfNode(node)); + return trivia.includes(OVERLAY_MARK) ? overlayAnnotationIds(trivia) : []; + }; + const blameNode = (node: ts.Node): boolean => { + const ids = marksOf(node); + for (const id of ids) add(node.getSourceFile().fileName, id); + return ids.length > 0; + }; + // A declaration's annotation: on the declaration, or on the statement + // holding it (`/** @type */ const x = …`, `/** @type */ this.f = …`). + const visitedDecls = new Set(); + const blameDeclaration = (decl: ts.Node, depth: number): void => { + if (visitedDecls.has(decl)) return; + visitedDecls.add(decl); + const sf = decl.getSourceFile(); + if (!overlayFiles.has(sf.fileName)) return; + addFile(sf.fileName); + let marked = false; + for (let n: ts.Node | undefined = decl, i = 0; n !== undefined && i < 4; n = n.parent, i++) { + if (blameNode(n)) marked = true; + if (ts.isStatement(n) || isClassMember(n) || ts.isSourceFile(n)) break; + } + // An unannotated function's result type comes from its returns. + if (!marked && depth < RETURN_DEPTH && ts.isFunctionLike(decl)) { + const body = (decl as { body?: ts.Node }).body; + if (body !== undefined) { + if (!ts.isBlock(body)) blameReferences(body, depth + 1); + else + visitReturns(body, (e) => { + blameReferences(e, depth + 1); + }); + } + } + }; + // Every declaration an expression's names resolve to. + const blameReferences = (root: ts.Node, depth: number): void => { + checker ??= program.getTypeChecker(); + const visit = (n: ts.Node): void => { + if (ts.isIdentifier(n) || ts.isPrivateIdentifier(n)) { + let symbol: ts.Symbol | undefined; + try { + symbol = checker!.getSymbolAtLocation(n); + } catch { + symbol = undefined; + } + for (const d of symbol?.declarations ?? []) blameDeclaration(d, depth); + } + ts.forEachChild(n, (c) => { + visit(c); + return undefined; + }); + }; + visit(root); + }; + + // Calls of each function declared in an overlay file, by declaration. + let calls: Map | null = null; + const callsOf = (fn: ts.Node): readonly ts.CallExpression[] => { + if (calls === null) { + calls = new Map(); + checker ??= program.getTypeChecker(); + for (const file of overlayFiles) { + const sf = program.getSourceFile(file); + if (sf === undefined) continue; + const visit = (n: ts.Node): void => { + if (ts.isCallExpression(n) && ts.isIdentifier(n.expression)) { + let decl: ts.Node | undefined; + try { + decl = checker!.getSymbolAtLocation(n.expression)?.declarations[0]; + } catch { + decl = undefined; + } + if (decl !== undefined) { + let list = calls!.get(decl); + if (list === undefined) calls!.set(decl, (list = [])); + list.push(n); + } + } + ts.forEachChild(n, (c) => { + visit(c); + return undefined; + }); + }; + visit(sf); + } + } + return calls.get(fn) ?? []; + }; + + for (const d of diagnostics) { + const sf = program.getSourceFile(d.loc.file); + if (sf === undefined) { + unattributed++; + continue; + } + const before = blamed; + if (overlayFiles.has(sf.fileName)) addFile(sf.fileName); + const node = innermostNodeAt(sf, d.loc.start, Math.max(d.loc.end, d.loc.start)); + // Enclosing annotated declarations, then the names of the refused + // expression, or failing that of the statement around it. + let statement: ts.Node = node; + for (let n: ts.Node | undefined = node; n !== undefined; n = n.parent) { + blameNode(n); + if (statement === node && (ts.isStatement(n) || isClassMember(n))) statement = n; + } + const own = blamed; + blameReferences(node, 0); + if (blamed === own && statement !== node) blameReferences(statement, 0); + // A function without parameter annotations is instantiated per call + // site from its arguments' types: blame what the arguments name. + if (blamed === own) { + const fn = enclosingFunction(node); + if (fn !== undefined) + for (const call of callsOf(fn)) + for (const arg of call.arguments.nodes) blameReferences(arg, 1); + } + // Still nothing: the lowering of the enclosing function changed as a + // whole (an instantiation the annotations inside it made possible). + if (blamed === before) { + const fn = enclosingFunctionLike(node); + if (fn !== undefined && overlayFiles.has(fn.getSourceFile().fileName)) { + const visit = (n: ts.Node): void => { + blameNode(n); + ts.forEachChild(n, (c) => { + visit(c); + return undefined; + }); + }; + visit(fn); + } + } + if (blamed === before) unattributed++; + } + return { annotations, files, unattributed }; +} + +function enclosingFunction(node: ts.Node): ts.Node | undefined { + for (let n: ts.Node | undefined = node.parent; n !== undefined; n = n.parent) + if (ts.isFunctionDeclaration(n)) return n; + else if (ts.isFunctionLike(n)) return undefined; + return undefined; +} + +function enclosingFunctionLike(node: ts.Node): ts.Node | undefined { + for (let n: ts.Node | undefined = node; n !== undefined; n = n.parent) + if (ts.isFunctionLike(n)) return n; + return undefined; +} + +function visitReturns(body: ts.Node, f: (e: ts.Expression) => void): void { + const visit = (n: ts.Node): void => { + if (ts.isFunctionLike(n) || ts.isClassLikeDeclaration(n)) return; + if (ts.isReturnStatement(n) && n.expression !== undefined) f(n.expression); + ts.forEachChild(n, (c) => { + visit(c); + return undefined; + }); + }; + ts.forEachChild(body, (c) => { + visit(c); + return undefined; + }); +} + +/** The innermost node whose token span covers [start, end). */ +function innermostNodeAt(sf: ts.SourceFile, start: number, end: number): ts.Node { + let node: ts.Node = sf; + for (;;) { + let next: ts.Node | undefined; + ts.forEachChild(node, (c) => { + if (next === undefined && ts.tokenStartOfNode(c) <= start && c.end >= end) next = c; + return undefined; + }); + if (next === undefined) return node; + node = next; + } +} + +function isClassMember(n: ts.Node): boolean { + return n.parent !== undefined && ts.isClassLikeDeclaration(n.parent) && n !== n.parent.name; +} diff --git a/packages/compiler/src/frontend/shape-inference/index.ts b/packages/compiler/src/frontend/shape-inference/index.ts index 233e62bf9..1db90850e 100644 --- a/packages/compiler/src/frontend/shape-inference/index.ts +++ b/packages/compiler/src/frontend/shape-inference/index.ts @@ -7,7 +7,8 @@ import type { ScrDiagnostic } from "../../diagnostics/diagnostic.js"; import { npmStaticPackageOfPath } from "../npm-static.js"; import { isShapelessPackage } from "../package-shims.js"; import { analyzeShapes, isPackageJsFile } from "./analyze.js"; -import { OVERLAY_MARK, planShapeOverlay } from "./overlay.js"; +import { OVERLAY_MARK, planShapeOverlay, type OverlayExclusions } from "./overlay.js"; +import { attributeOverlayRefusals } from "./fallback.js"; import { planStaticObjectRewrites } from "./static-objects.js"; import { isJsSourceFile } from "../program.js"; import { buildShapeReport } from "./report.js"; @@ -58,6 +59,7 @@ export function shapeOverlayAnnotationCount(): number { } export function resetShapeOverlay(): void { + staticRewriteFiles = new Set(); staticObjectStats = { classes: 0, functions: 0, directCalls: 0 }; overlayDone = false; overlayFiles = new Set(); @@ -73,12 +75,90 @@ export function shapeOverlayEnabled(): boolean { return process.env["SCRIPTC_SHAPE_OVERLAY"] === "1"; } +/* The build's fallback state (fallback.ts): annotations and files whose + * lowering was refused, and whether the build dropped the overlay. It + * outlives the loads of one build. */ +let exclusions: { annotations: Map>; files: Set } = { + annotations: new Map(), + files: new Set(), +}; +let overlayDropped = false; +let droppedAnnotations = 0; + +/** Starts a build: no annotation is excluded yet. */ +export function beginShapeOverlayBuild(): void { + exclusions = { annotations: new Map(), files: new Set() }; + overlayDropped = false; + droppedAnnotations = 0; +} + +/** Whether the current load has overlay files or structural rewrites. */ +export function shapeOverlayActive(): boolean { + const { classes, functions } = staticObjectStats; + return overlayFiles.size > 0 || classes + functions > 0; +} + +/** The overlay's answer to refused lowering diagnostics: it excludes what + * the refusals depend on, at the given level, and says whether a reload + * can change anything. + * - "annotations": the blamed annotations (or, for a blamed file without + * blamed annotations, that file); + * - "fences": like "annotations", for refusals deferred to run time, and + * never a whole file; + * - "files": every overlay file involved; + * - "all": the whole overlay. */ +export function excludeRefusedOverlay( + program: ts.Program, + diagnostics: readonly ScrDiagnostic[], + level: "annotations" | "files" | "all" | "fences", +): boolean { + if (overlayDropped) return false; + const files = new Set(overlayFiles); + // Files with structural rewrites only also count as overlay files. + for (const f of staticRewriteFiles) files.add(f); + if (level === "all") { + overlayDropped = true; + droppedAnnotations = overlayAnnotations; + return true; + } + const blame = attributeOverlayRefusals(program, diagnostics, files); + let changed = false; + for (const file of blame.files) { + const ids = blame.annotations.get(file); + // Deferred refusals drop only the annotations they are blamed on. + if (level === "fences" && ids === undefined) continue; + if (level === "files" || ids === undefined) { + if (!exclusions.files.has(file)) { + exclusions.files.add(file); + changed = true; + } + continue; + } + let set = exclusions.annotations.get(file); + if (set === undefined) exclusions.annotations.set(file, (set = new Set())); + for (const id of ids) { + if (set.has(id)) continue; + set.add(id); + droppedAnnotations++; + changed = true; + } + } + return changed; +} + +let staticRewriteFiles: ReadonlySet = new Set(); + +function overlayExclusions(): OverlayExclusions { + return exclusions; +} + /** loadProgram7's overlayer: analyze, plan, and register the overlay files. */ export function shapeOverlayer( program: ts.Program, round: number, ): ReadonlyMap | null { if (round === 0) resetShapeOverlay(); + if (overlayDropped) return null; const analysis = analyzeShapes(program, program.getTypeChecker()); if (analysis.budgetExceeded) return null; // Package JavaScript of packages that keep shapes on, and the program's @@ -92,7 +172,11 @@ export function shapeOverlayer( // Round 0: the structural rewrites (static-objects.ts); the program is // rebuilt from them and analyzed again before the typed overlay. if (round === 0 && process.env["SCRIPTC_SHAPE_STATIC_OBJECTS"] !== "0") { - const rewrites = planStaticObjectRewrites(analysis, jsFiles); + const rewrites = planStaticObjectRewrites( + analysis, + jsFiles.filter((sf) => !exclusions.files.has(sf.fileName)), + ); + staticRewriteFiles = new Set(rewrites.files.keys()); staticObjectStats = { classes: rewrites.classes, functions: rewrites.functions, @@ -107,7 +191,7 @@ export function shapeOverlayer( } if (round > 1) return null; const report = buildShapeReport(analysis, "1"); - const overlay = planShapeOverlay(analysis, report.sites, jsFiles); + const overlay = planShapeOverlay(analysis, report.sites, jsFiles, overlayExclusions()); overlayFiles = new Set(overlay.files.keys()); overlayAnnotations = overlay.annotations; overlayDeadBranches = overlay.deadBranches; @@ -181,6 +265,26 @@ export function shapeInferredFactCheckCount(): number { * checks of inferred facts the build inserted. A note, never a failure. */ export function shapeOverlayReport(entryFile: string): ScrDiagnostic[] { const { classes, functions, directCalls } = staticObjectStats; + // Annotations whose lowering was refused (fallback.ts); they compile + // untyped. + const fallback = overlayDropped + ? "the lowering refused code the overlay typed, so the build compiles without it" + : droppedAnnotations + exclusions.files.size > 0 + ? `it dropped ${droppedAnnotations} annotation${droppedAnnotations === 1 ? "" : "s"}` + + (exclusions.files.size > 0 + ? ` and the overlay of ${exclusions.files.size} file${exclusions.files.size === 1 ? "" : "s"}` + : "") + + " whose lowering was refused" + : null; + if (overlayDropped) + return [ + { + code: "SC6103", + message: `shape inference: ${fallback}`, + loc: { file: entryFile, start: 0, end: 0 }, + hint: "--no-shape-overlay compiles without the overlay", + }, + ]; if (overlayFiles.size === 0 && classes + functions === 0) return []; const packages = new Set( [...overlayFiles].map((f) => npmStaticPackageOfPath(f) ?? "the program"), @@ -197,7 +301,8 @@ export function shapeOverlayReport(entryFile: string): ScrDiagnostic[] { : "") + (classes + functions > 0 ? `; it compiled ${classes} constructor function${classes === 1 ? "" : "s"} as classes and ${functions} namespace function${functions === 1 ? "" : "s"} as direct functions (${directCalls} direct call${directCalls === 1 ? "" : "s"})` - : ""), + : "") + + (fallback !== null ? `; ${fallback}` : ""), loc: { file: entryFile, start: 0, end: 0 }, hint: "a failing check reports a scriptc analysis bug; --no-shape-overlay compiles without the overlay", }, diff --git a/packages/compiler/src/frontend/shape-inference/overlay.ts b/packages/compiler/src/frontend/shape-inference/overlay.ts index 37b30b1f6..1cf035bff 100644 --- a/packages/compiler/src/frontend/shape-inference/overlay.ts +++ b/packages/compiler/src/frontend/shape-inference/overlay.ts @@ -54,10 +54,28 @@ function list(arr: { readonly nodes: readonly T[] } | undefin let debugIndex = 0; +/** Annotations a build dropped because their lowering was refused: per + * file, the annotation ids (positions in the overlay's input text); a file + * in `files` gets no overlay at all. */ +export interface OverlayExclusions { + readonly annotations: ReadonlyMap>; + readonly files: ReadonlySet; +} + +export const NO_EXCLUSIONS: OverlayExclusions = { annotations: new Map(), files: new Set() }; + +/** The ids of the overlay annotations in a comment or trivia text. */ +export function overlayAnnotationIds(text: string): number[] { + const ids: number[] = []; + for (const m of text.matchAll(/@scriptcShape #(\d+)/g)) ids.push(Number(m[1])); + return ids; +} + export function planShapeOverlay( analysis: ShapeAnalysis, verdicts: readonly SiteVerdict[], packageFiles: readonly ts.SourceFile[], + exclusions: OverlayExclusions = NO_EXCLUSIONS, ): ShapeOverlay { const pSites = new Map(); const siteVerdicts = new Set(verdicts.map((v) => v.site.atom)); @@ -96,9 +114,20 @@ export function planShapeOverlay( } } for (const sf of packageFiles) { + if (exclusions.files.has(sf.fileName)) continue; const typedefs = new Map(); const typedefText: string[] = []; const edits: Edit[] = []; + const dropped = exclusions.annotations.get(sf.fileName); + // One annotation: a JSDoc comment at `pos` carrying MARK and the + // annotation's id (its position), so a refused lowering can name it and + // the next load drop it. `close`: the text that closes a type cast. + const annotate = (pos: number, tags: string, close?: number): void => { + if (dropped?.has(pos) === true) return; + edits.push({ pos, text: `/** ${tags} ${MARK} #${pos} */ ${close !== undefined ? "(" : ""}` }); + if (close !== undefined) edits.push({ pos: close, text: ")" }); + annotations++; + }; const className = (decl: ts.Node): string | null => { const own = (decl as ts.ClassLikeDeclaration).name; @@ -111,7 +140,8 @@ export function planShapeOverlay( own === undefined || !ts.isClassDeclaration(decl) || decl.parent !== other || - !overlaid.has(other) + !overlaid.has(other) || + exclusions.files.has(other.fileName) ) return null; const alias = `__shape_class_${own.text}`; @@ -172,6 +202,15 @@ export function planShapeOverlay( objects++; if (analysis.escaped.has(a) || (siteVerdicts.has(a) && !pSites.has(a))) return inner ? "any" : null; + // Only arrays the program writes as literals: a builtin's array + // (a match result, `new Array(n)`) may have holes, and the + // checker already types it from the library. + if ( + a.node === undefined || + !ts.isArrayLiteralExpression(a.node) || + a.node.elements.nodes.some((e) => e.kind === ts.SyntaxKind.OmittedExpression) + ) + return inner ? "any" : null; const el = analysis.elements.get(a); if (el !== undefined) for (const x of analysis.nodes[el]!.atoms) elements.add(x); hasArray = true; @@ -334,9 +373,7 @@ export function planShapeOverlay( !declaresMember(cls, node.expression.name.text) ) { const self = node.expression.expression; - edits.push({ pos: ts.tokenStartOfNode(self), text: `/** @type {${t}} ${MARK} */ (` }); - edits.push({ pos: self.end, text: ")" }); - annotations++; + annotate(ts.tokenStartOfNode(self), `@type {${t}}`, self.end); } ts.forEachChild(node, (c) => { scan(c); @@ -429,7 +466,7 @@ export function planShapeOverlay( const t = typeOf(paramValues(fn, index), false, 0); if (t !== null) tags.push(`@param {${t}} ${p.name.text}`); }); - return tags.length > 0 ? `${tags.join(" ")} ${MARK}` : null; + return tags.length > 0 ? tags.join(" ") : null; }; const visit = (node: ts.Node): void => { @@ -447,10 +484,7 @@ export function planShapeOverlay( ) { if (node.body !== undefined) { const tags = paramTags(node); - if (tags !== null) { - edits.push({ pos: ts.tokenStartOfNode(node), text: `/** ${tags} */ ` }); - annotations++; - } + if (tags !== null) annotate(ts.tokenStartOfNode(node), tags); } // `this.f = …` fields of a constructor's top-level statements. if (ts.isConstructorDeclaration(node) && node.body !== undefined) { @@ -473,10 +507,7 @@ export function planShapeOverlay( if (seen.has(name)) continue; seen.add(name); const t = cls !== undefined ? fieldType(cls, name) : null; - if (t !== null) { - edits.push({ pos: ts.tokenStartOfNode(st), text: `/** @type {${t}} ${MARK} */ ` }); - annotations++; - } + if (t !== null) annotate(ts.tokenStartOfNode(st), `@type {${t}}`); } } } @@ -492,10 +523,7 @@ export function planShapeOverlay( const init = d.initializer; if (ts.isIdentifier(d.name) && init !== undefined && !literalInitializer(init)) { const t = typeOf(analysis.declValues(d), false, 0); - if (t !== null) { - edits.push({ pos: ts.tokenStartOfNode(node), text: `/** @type {${t}} ${MARK} */ ` }); - annotations++; - } + if (t !== null) annotate(ts.tokenStartOfNode(node), `@type {${t}}`); } } ts.forEachChild(node, (c) => { @@ -525,6 +553,7 @@ export function planShapeOverlay( const ranges: [number, number][] = []; for (const branch of dead) { const start = ts.tokenStartOfNode(branch); + if (dropped?.has(start) === true) continue; if (ranges.some(([lo, hi]) => start >= lo && start < hi)) continue; ranges.push([start, branch.end]); } @@ -540,7 +569,11 @@ export function planShapeOverlay( `scriptc internal error: shape inference proved the branch at ${rel}:${line} unreachable, but it ran. ` + `This is a bug in scriptc's shape analysis, not in the program. Rebuild with --no-shape-overlay, ` + `or set "scriptc": {"packages": {"${pkg}": {"shapes": false}}} in package.json.`; - edits.push({ pos: lo, end: hi, text: `{ throw new Error(${JSON.stringify(message)}); }` }); + edits.push({ + pos: lo, + end: hi, + text: `{ /** ${MARK} #${lo} */ throw new Error(${JSON.stringify(message)}); }`, + }); deadBranches++; } } diff --git a/packages/compiler/src/frontend/ts7/program-host.ts b/packages/compiler/src/frontend/ts7/program-host.ts index ec143e401..3c3e81d9c 100644 --- a/packages/compiler/src/frontend/ts7/program-host.ts +++ b/packages/compiler/src/frontend/ts7/program-host.ts @@ -190,6 +190,11 @@ export class Ts7Host { }); } + /** The text of a registered in-memory file, or undefined. */ + virtualFileText(path: string): string | undefined { + return this.virtualFiles.get(tsgoPath(path)); + } + /** Registers an in-memory file served to tsgo by the virtual-FS hooks. */ addVirtualFile(path: string, content: string): void { this.ensureOpen(); diff --git a/packages/compiler/src/index.ts b/packages/compiler/src/index.ts index c0f0e0649..c46d50374 100644 --- a/packages/compiler/src/index.ts +++ b/packages/compiler/src/index.ts @@ -443,10 +443,11 @@ export async function compile( opts: CompileRequestOptions, ): Promise { clearCompileSessionCaches(); - // `shapeOverlay: false` reaches the frontend (and its child process) - // through the environment the overlay reads. + // `shapeOverlay` reaches the frontend (and its child process) through + // the environment the overlay reads. const overlayEnv = process.env["SCRIPTC_SHAPE_OVERLAY"]; - if (opts.shapeOverlay === false) process.env["SCRIPTC_SHAPE_OVERLAY"] = "0"; + if (opts.shapeOverlay !== undefined) + process.env["SCRIPTC_SHAPE_OVERLAY"] = opts.shapeOverlay ? "1" : "0"; const frontendInputs = new FrontendInputTracker(); const warnings: CompileWarningSink = { warnings: [], sourceTexts: null }; // compileTracked starts it for a build with no earlier build of the same invocation. @@ -459,7 +460,7 @@ export async function compile( } finally { setLibDiagnosticsStore(null); frontendProcess?.release(); - if (opts.shapeOverlay === false) { + if (opts.shapeOverlay !== undefined) { if (overlayEnv === undefined) delete process.env["SCRIPTC_SHAPE_OVERLAY"]; else process.env["SCRIPTC_SHAPE_OVERLAY"] = overlayEnv; } @@ -1200,7 +1201,8 @@ function configureLibDiagnostics(root: string | null): void { export async function runFrontendProcess(requestPath: string, resultPath: string): Promise { const request = JSON.parse(await readFile(requestPath, "utf8")) as FrontendProcessRequest; // The process started before the build's options were known. - if (request.opts.shapeOverlay === false) process.env["SCRIPTC_SHAPE_OVERLAY"] = "0"; + if (request.opts.shapeOverlay !== undefined) + process.env["SCRIPTC_SHAPE_OVERLAY"] = request.opts.shapeOverlay ? "1" : "0"; clearCompileSessionCaches(); setProvenanceSources(request.provenance); // The compile that started this process prepared the cache root. diff --git a/scripts/ci-test-plan.mjs b/scripts/ci-test-plan.mjs index 184f04d1c..a0fb3f354 100644 --- a/scripts/ci-test-plan.mjs +++ b/scripts/ci-test-plan.mjs @@ -7,6 +7,7 @@ export const corpusFiles = [ "tests/harness/differential.test.ts", "tests/harness/llvm-differential.test.ts", "tests/harness/npm.test.ts", + "tests/harness/npm-shape-overlay.test.ts", "tests/harness/server.test.ts", "tests/harness/test262.test.ts", "tests/harness/three.test.ts", diff --git a/tests/harness/npm-shape-overlay.test.ts b/tests/harness/npm-shape-overlay.test.ts new file mode 100644 index 000000000..4db61fbe6 --- /dev/null +++ b/tests/harness/npm-shape-overlay.test.ts @@ -0,0 +1,7 @@ +/* Shape inference's overlay never breaks compilation: every npm case + * compiles with the overlay on and builds and runs exactly as it does in the + * default build (npm-suite.ts's KNOWN). An overlay annotation whose lowering + * is refused is dropped, so its declaration compiles untyped. */ +import { defineNpmSuite } from "./npm-suite.js"; + +defineNpmSuite({ label: "npm native, shape overlay", shapeOverlay: true }); diff --git a/tests/harness/npm-suite.ts b/tests/harness/npm-suite.ts new file mode 100644 index 000000000..85785364f --- /dev/null +++ b/tests/harness/npm-suite.ts @@ -0,0 +1,280 @@ +/* The npm suite (npm.test.ts runs it as built by default, + * npm-shape-overlay.test.ts with shape inference's overlay on). + * + * npm packages compiled natively: every program under tests/fixtures/npm/cases + * (plus the commander fixtures in npm-cases.ts) builds with the default + * static npm compile and runs against Node. A case matches Node (stdout and + * exit status for every argv list, stderr when it opts in with + * `// @stderr`) unless KNOWN records how it differs: + * + * - build: the compile refuses it with this diagnostic code; + * - run: it compiles, and the program's output names this diagnostic code + * (a refusal deferred to the operation that reached it), for every argv + * list or for the listed `argvs`, while the other lists match Node; + * - divergence: it runs but does not match Node yet. These run as expected + * failures, so a fix that makes one match fails here until KNOWN drops it. + * + * The fixture node_modules are committed test data. Binaries compile the + * package sources at build time and read nothing from node_modules at run + * time. + */ +import { execFile } from "node:child_process"; +import { createHash } from "node:crypto"; +import { globSync, mkdirSync, readFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { promisify } from "node:util"; +import { describe, expect, test } from "vitest"; +import { compile } from "@scriptc/compiler"; +import { npmCases, type NpmCase } from "./npm-cases.js"; +import { shardSelect, shardSuffix } from "./shard.js"; + +const execFileAsync = promisify(execFile); +const repoRoot = join(import.meta.dirname, "../.."); +const fixturesRoot = join(repoRoot, "tests/fixtures"); +const cacheDir = join(repoRoot, "node_modules/.cache/scriptc-tests"); +const sanitize = process.env["SCRIPTC_SAN"] === "1"; + +type Known = { build: string } | { run: string; argvs?: string[][] } | { divergence: string }; + +const KNOWN: Record = { + "any-chaining": { build: "SC2003" }, + "assert-shims": { run: "SC1043" }, + "buffer-shims": { run: "SC2020" }, + "bytes-exit": { build: "SC2004" }, + "cjs-esm-named": { build: "SC2013" }, + "cjs-lexer-aligned": { build: "SC2013" }, + "commander-calc": { run: "SC1090", argvs: [["--help"]] }, + "commander-calc-async": { run: "SC1090", argvs: [["--help"]] }, + "create-require": { run: "SC1090" }, + "dotfolder": { run: "SC1090" }, + "dual-package-kinds": { build: "SC2013" }, + "dynamic-import": { build: "SC2012" }, + "dynamic-import-local": { build: "SC2012" }, + "esbuild-require": { build: "SC2013" }, + "esm-missing-default": { build: "SC1090" }, + "esm-missing-export": { build: "SC1090" }, + "fetch-destructure": { run: "SC1031" }, + "fs-shims": { run: "SC1101" }, + "import-refusal": { build: "SC2020" }, + "island-registry": { build: "SC2011" }, + "island-timers": { + divergence: + "Timeout handles are numbers (typeof) and lose their ref flag once fired or cleared", + }, + "island-web-plumbing": { run: "SC2020" }, + "lazy-traps": { build: "SC2013" }, + "misc-shims": { run: "SC2020" }, + "namespace": { build: "SC1013" }, + "node24-async-events": { run: "SC2020" }, + "node24-commonjs-metadata": { run: "SC2020" }, + "node24-crypto-kdf": { run: "SC2020" }, + "node24-deep-equality": { run: "SC2020" }, + "node24-diagnostics-async": { run: "SC1090" }, + "node24-diagnostics-tracing": { run: "SC1090" }, + "node24-env": { run: "SC1090" }, + "node24-esm-metadata": { run: "SC1090" }, + "node24-event-emitter": { run: "SC2020" }, + "node24-file-url-conversion": { build: "SC1090" }, + "node24-fs-descriptors": { run: "SC2020" }, + "node24-mime": { run: "SC2020" }, + "node24-mime-errors": { run: "SC1090" }, + "node24-no-error-assertions": { run: "SC2020" }, + "node24-parseargs": { run: "SC1090" }, + "node24-promise-timers": { run: "SC2020" }, + "node24-string-decoder": { run: "SC2020" }, + "node24-style-text": { run: "SC2020" }, + "node24-system-errors": { run: "SC1090" }, + "node24-text-encoding": { run: "SC2020" }, + "node24-text-utilities": { run: "SC2020" }, + "node24-timers-unref": { run: "SC2020" }, + "node24-url-factories": { run: "SC2020" }, + "node24-url-mutation": { run: "SC1090" }, + "node24-util-adapters": { run: "Object.create with this value is not supported yet" }, + "node24-util-types": { run: "SC1090" }, + "node24-web-stream-encoding": { run: "SC1090" }, + "node24-web-stream-errors": { run: "SC1090" }, + "node24-web-stream-lifecycle": { run: "SC1090" }, + "node24-web-stream-pipe-abort": { run: "SC2020" }, + "node24-web-stream-pipe-errors": { run: "SC1090" }, + "node24-web-stream-pipes": { run: "SC1090" }, + "node24-web-stream-queues": { run: "SC1090" }, + "node24-web-stream-readable-lifecycle": { run: "SC2020" }, + "node24-web-stream-readers": { run: "SC1090" }, + "node24-web-stream-tee": { run: "SC2020" }, + "node24-web-stream-transforms": { run: "SC1090" }, + "package-imports": { build: "SC2013" }, + "promise-bridge": { build: "SC1090" }, + "self-name": { build: "SC2013" }, + "stream-shims": { run: "SC2020" }, + "util-shims": { run: "SC2020" }, + "web-streams": { run: "SC1090" }, + "workspace-copied": { build: "SC2013" }, + "workspace-unclean": { build: "SC2013" }, +}; + +/** Per-case timeouts past the default 120 s: the undici cases compile the + * built-in undici implementation's module graph, the zod 3 cases (zod3, + * zod3-adoption, zod3-boundary, zod3-infer) compile zod 3's 3,700-line + * types module, and the zod 4 cases (zod4-boundary, zod4-record, + * hono-zod-validator, hono-zod-default-hook) + * compile zod 4's core, classic API and locales. A sanitized build of any + * of them takes minutes on a CI runner. */ +const TIMEOUTS: Record = { + "hono-zod-default-hook": 900_000, + "hono-zod-validator": 900_000, + undici5: 600_000, + undici7: 600_000, + "zod4-boundary": 900_000, + "zod4-record": 900_000, + zod3: 600_000, + "zod3-adoption": 600_000, + "zod3-boundary": 600_000, + "zod3-infer": 600_000, +}; + +interface RunResult { + stdout: Buffer; + stderr: Buffer; + exitCode: number; +} + +function programStderr(stderr: Buffer): Buffer { + if (!sanitize) return stderr; + // Linux ASan emits this instrumentation warning at the first fiber swap. + // Latin-1 preserves every other byte, including non-UTF-8 program output. + return Buffer.from( + stderr + .toString("latin1") + .replace( + /^==\d+==WARNING: ASan doesn't fully support makecontext\/swapcontext functions and may produce false positives in some cases!\n/gm, + "", + ), + "latin1", + ); +} + +async function runBinary(cmd: string, args: string[], cwd: string): Promise { + try { + // stdin closes immediately, as in differential.test.ts: a program that + // reads fd 0 to EOF must not hang on an open pipe. + const pending = execFileAsync(cmd, args, { cwd, encoding: "buffer" }); + pending.child.stdin?.end(); + const { stdout, stderr } = await pending; + return { stdout, stderr, exitCode: 0 }; + } catch (err) { + const e = err as { code?: unknown; signal?: unknown; stdout?: Buffer; stderr?: Buffer }; + if (!Buffer.isBuffer(e.stdout) || !Buffer.isBuffer(e.stderr)) throw err; + const exitCode = typeof e.code === "number" ? e.code : -1; + if (exitCode === -1 && typeof e.signal !== "string") throw err; + return { stdout: e.stdout, stderr: e.stderr, exitCode }; + } +} + +type Built = { ok: true; binary: string } | { ok: false; codes: string[]; text: string }; + +/** Compile once per (entry, lane); the cache key hashes the program and the + * fixture packages so edits to either rebuild. */ +async function build(entry: string, shapeOverlay: boolean | undefined): Promise { + const hash = createHash("sha256"); + const inputs = [ + entry, + ...globSync(join(dirname(entry), "*.{ts,mts,cts,js,mjs,cjs,json}")), + ...globSync(join(fixturesRoot, "npm/node_modules/**/*.{js,mjs,cjs,json,d.ts,node}")), + ].sort(); + for (const f of inputs) hash.update(f).update(readFileSync(f)); + const key = hash + .update(sanitize ? "san" : "plain") + .update(shapeOverlay === undefined ? "" : `overlay=${shapeOverlay}`) + .digest("hex") + .slice(0, 16); + const outDir = join(cacheDir, `npm-${key}`); + mkdirSync(outDir, { recursive: true }); + const result = await compile(entry, { + outPath: join(outDir, "program"), + outDir, + sanitize, + ...(shapeOverlay === undefined ? {} : { shapeOverlay }), + }); + if (result.ok) return { ok: true, binary: result.binaryPath }; + return { + ok: false, + codes: result.diagnostics.map((d) => d.code), + text: result.diagnostics.map((d) => `${d.code}: ${d.message}`).join("\n"), + }; +} + +async function expectMatchesNode(c: NpmCase, binary: string): Promise { + const cwd = dirname(c.entry); + for (const argv of c.argvs ?? [[]]) { + const [nodeRes, nativeRes] = await Promise.all([ + runBinary(process.execPath, [c.entry, ...argv], cwd), + runBinary(binary, argv, cwd), + ]); + const label = argv.join(" "); + if (!nodeRes.stdout.equals(nativeRes.stdout)) { + expect(nativeRes.stdout.toString("utf8"), label).toBe(nodeRes.stdout.toString("utf8")); + expect.unreachable("stdout differed at byte level but not after utf8 decode"); + } + expect(nativeRes.exitCode, label).toBe(nodeRes.exitCode); + if (c.compareStderr) expect(programStderr(nativeRes.stderr), label).toEqual(nodeRes.stderr); + } +} + +/** Defines the suite. `shapeOverlay` is passed to every compile (unset: the + * default build). With the overlay on, every case must still build and run + * as KNOWN records for the default build, except that a refused build may + * compile, and then matches Node. */ +export function defineNpmSuite(options: { label: string; shapeOverlay?: boolean }): void { + const { label, shapeOverlay } = options; + const cases = shardSelect(npmCases(fixturesRoot), (c) => c.name); + test("every KNOWN entry names a case", () => { + const names = new Set(npmCases(fixturesRoot).map((c) => c.name)); + expect(Object.keys(KNOWN).filter((name) => !names.has(name))).toEqual([]); + }); + + describe(`${label} (${cases.length} programs${sanitize ? ", sanitized" : ""}${shardSuffix()})`, () => { + for (const c of cases) { + const known = KNOWN[c.name]; + if (known !== undefined && "divergence" in known) { + test.fails(`${c.name} (diverges: ${known.divergence})`, async () => { + const built = await build(c.entry, shapeOverlay); + if (!built.ok) throw new Error(built.text); + await expectMatchesNode(c, built.binary); + }, 120_000); + continue; + } + test( + c.name, + async () => { + const built = await build(c.entry, shapeOverlay); + if (known !== undefined && "build" in known) { + // The overlay may type code the default build refuses: such a + // program must then match Node. + if (shapeOverlay === true && built.ok) { + await expectMatchesNode(c, built.binary); + return; + } + expect(built.ok, "the compile refuses the program").toBe(false); + if (!built.ok) expect(built.codes, built.text).toContain(known.build); + return; + } + if (!built.ok) throw new Error(`npm fixture failed to compile:\n${built.text}`); + if (known !== undefined && "run" in known) { + for (const argv of known.argvs ?? c.argvs ?? [[]]) { + const res = await runBinary(built.binary, argv, dirname(c.entry)); + const output = res.stdout.toString("utf8") + res.stderr.toString("utf8"); + expect(output, `${argv.join(" ")}: the deferred refusal`).toContain(known.run); + } + if (known.argvs === undefined) return; + const refused = new Set(known.argvs.map((argv) => JSON.stringify(argv))); + const argvs = (c.argvs ?? [[]]).filter((argv) => !refused.has(JSON.stringify(argv))); + await expectMatchesNode({ ...c, argvs }, built.binary); + return; + } + await expectMatchesNode(c, built.binary); + }, + TIMEOUTS[c.name] ?? 120_000, + ); + } + }); +} diff --git a/tests/harness/npm.test.ts b/tests/harness/npm.test.ts index 27b27b8e3..dc64d8b86 100644 --- a/tests/harness/npm.test.ts +++ b/tests/harness/npm.test.ts @@ -1,259 +1,4 @@ -/* npm packages compiled natively: every program under tests/fixtures/npm/cases - * (plus the commander fixtures in npm-cases.ts) builds with the default - * static npm compile and runs against Node. A case matches Node (stdout and - * exit status for every argv list, stderr when it opts in with - * `// @stderr`) unless KNOWN records how it differs: - * - * - build: the compile refuses it with this diagnostic code; - * - run: it compiles, and the program's output names this diagnostic code - * (a refusal deferred to the operation that reached it), for every argv - * list or for the listed `argvs`, while the other lists match Node; - * - divergence: it runs but does not match Node yet. These run as expected - * failures, so a fix that makes one match fails here until KNOWN drops it. - * - * The fixture node_modules are committed test data. Binaries compile the - * package sources at build time and read nothing from node_modules at run - * time. - */ -import { execFile } from "node:child_process"; -import { createHash } from "node:crypto"; -import { globSync, mkdirSync, readFileSync } from "node:fs"; -import { dirname, join } from "node:path"; -import { promisify } from "node:util"; -import { describe, expect, test } from "vitest"; -import { compile } from "@scriptc/compiler"; -import { npmCases, type NpmCase } from "./npm-cases.js"; -import { shardSelect, shardSuffix } from "./shard.js"; +/* npm packages compiled natively as built by default (npm-suite.ts). */ +import { defineNpmSuite } from "./npm-suite.js"; -const execFileAsync = promisify(execFile); -const repoRoot = join(import.meta.dirname, "../.."); -const fixturesRoot = join(repoRoot, "tests/fixtures"); -const cacheDir = join(repoRoot, "node_modules/.cache/scriptc-tests"); -const sanitize = process.env["SCRIPTC_SAN"] === "1"; - -type Known = { build: string } | { run: string; argvs?: string[][] } | { divergence: string }; - -const KNOWN: Record = { - "any-chaining": { build: "SC2003" }, - "assert-shims": { run: "SC1043" }, - "buffer-shims": { run: "SC2020" }, - "bytes-exit": { build: "SC2004" }, - "cjs-esm-named": { build: "SC2013" }, - "cjs-lexer-aligned": { build: "SC2013" }, - "commander-calc": { run: "SC1090", argvs: [["--help"]] }, - "commander-calc-async": { run: "SC1090", argvs: [["--help"]] }, - "create-require": { run: "SC1090" }, - "dotfolder": { run: "SC1090" }, - "dual-package-kinds": { build: "SC2013" }, - "dynamic-import": { build: "SC2012" }, - "dynamic-import-local": { build: "SC2012" }, - "esbuild-require": { build: "SC2013" }, - "esm-missing-default": { build: "SC1090" }, - "esm-missing-export": { build: "SC1090" }, - "fetch-destructure": { run: "SC1031" }, - "fs-shims": { run: "SC1101" }, - "import-refusal": { build: "SC2020" }, - "island-registry": { build: "SC2011" }, - "island-timers": { - divergence: - "Timeout handles are numbers (typeof) and lose their ref flag once fired or cleared", - }, - "island-web-plumbing": { run: "SC2020" }, - "lazy-traps": { build: "SC2013" }, - "misc-shims": { run: "SC2020" }, - "namespace": { build: "SC1013" }, - "node24-async-events": { run: "SC2020" }, - "node24-commonjs-metadata": { run: "SC2020" }, - "node24-crypto-kdf": { run: "SC2020" }, - "node24-deep-equality": { run: "SC2020" }, - "node24-diagnostics-async": { run: "SC1090" }, - "node24-diagnostics-tracing": { run: "SC1090" }, - "node24-env": { run: "SC1090" }, - "node24-esm-metadata": { run: "SC1090" }, - "node24-event-emitter": { run: "SC2020" }, - "node24-file-url-conversion": { build: "SC1090" }, - "node24-fs-descriptors": { run: "SC2020" }, - "node24-mime": { run: "SC2020" }, - "node24-mime-errors": { run: "SC1090" }, - "node24-no-error-assertions": { run: "SC2020" }, - "node24-parseargs": { run: "SC1090" }, - "node24-promise-timers": { run: "SC2020" }, - "node24-string-decoder": { run: "SC2020" }, - "node24-style-text": { run: "SC2020" }, - "node24-system-errors": { run: "SC1090" }, - "node24-text-encoding": { run: "SC2020" }, - "node24-text-utilities": { run: "SC2020" }, - "node24-timers-unref": { run: "SC2020" }, - "node24-url-factories": { run: "SC2020" }, - "node24-url-mutation": { run: "SC1090" }, - "node24-util-adapters": { run: "Object.create with this value is not supported yet" }, - "node24-util-types": { run: "SC1090" }, - "node24-web-stream-encoding": { run: "SC1090" }, - "node24-web-stream-errors": { run: "SC1090" }, - "node24-web-stream-lifecycle": { run: "SC1090" }, - "node24-web-stream-pipe-abort": { run: "SC2020" }, - "node24-web-stream-pipe-errors": { run: "SC1090" }, - "node24-web-stream-pipes": { run: "SC1090" }, - "node24-web-stream-queues": { run: "SC1090" }, - "node24-web-stream-readable-lifecycle": { run: "SC2020" }, - "node24-web-stream-readers": { run: "SC1090" }, - "node24-web-stream-tee": { run: "SC2020" }, - "node24-web-stream-transforms": { run: "SC1090" }, - "package-imports": { build: "SC2013" }, - "promise-bridge": { build: "SC1090" }, - "self-name": { build: "SC2013" }, - "stream-shims": { run: "SC2020" }, - "util-shims": { run: "SC2020" }, - "web-streams": { run: "SC1090" }, - "workspace-copied": { build: "SC2013" }, - "workspace-unclean": { build: "SC2013" }, -}; - -const cases = shardSelect(npmCases(fixturesRoot), (c) => c.name); - -/** Per-case timeouts past the default 120 s: the undici cases compile the - * built-in undici implementation's module graph, the zod 3 cases (zod3, - * zod3-adoption, zod3-boundary, zod3-infer) compile zod 3's 3,700-line - * types module, and the zod 4 cases (zod4-boundary, zod4-record, - * hono-zod-validator, hono-zod-default-hook) - * compile zod 4's core, classic API and locales. A sanitized build of any - * of them takes minutes on a CI runner. */ -const TIMEOUTS: Record = { - "hono-zod-default-hook": 900_000, - "hono-zod-validator": 900_000, - undici5: 600_000, - undici7: 600_000, - "zod4-boundary": 900_000, - "zod4-record": 900_000, - zod3: 600_000, - "zod3-adoption": 600_000, - "zod3-boundary": 600_000, - "zod3-infer": 600_000, -}; - -interface RunResult { - stdout: Buffer; - stderr: Buffer; - exitCode: number; -} - -function programStderr(stderr: Buffer): Buffer { - if (!sanitize) return stderr; - // Linux ASan emits this instrumentation warning at the first fiber swap. - // Latin-1 preserves every other byte, including non-UTF-8 program output. - return Buffer.from( - stderr - .toString("latin1") - .replace( - /^==\d+==WARNING: ASan doesn't fully support makecontext\/swapcontext functions and may produce false positives in some cases!\n/gm, - "", - ), - "latin1", - ); -} - -async function runBinary(cmd: string, args: string[], cwd: string): Promise { - try { - // stdin closes immediately, as in differential.test.ts: a program that - // reads fd 0 to EOF must not hang on an open pipe. - const pending = execFileAsync(cmd, args, { cwd, encoding: "buffer" }); - pending.child.stdin?.end(); - const { stdout, stderr } = await pending; - return { stdout, stderr, exitCode: 0 }; - } catch (err) { - const e = err as { code?: unknown; signal?: unknown; stdout?: Buffer; stderr?: Buffer }; - if (!Buffer.isBuffer(e.stdout) || !Buffer.isBuffer(e.stderr)) throw err; - const exitCode = typeof e.code === "number" ? e.code : -1; - if (exitCode === -1 && typeof e.signal !== "string") throw err; - return { stdout: e.stdout, stderr: e.stderr, exitCode }; - } -} - -type Built = { ok: true; binary: string } | { ok: false; codes: string[]; text: string }; - -/** Compile once per (entry, lane); the cache key hashes the program and the - * fixture packages so edits to either rebuild. */ -async function build(entry: string): Promise { - const hash = createHash("sha256"); - const inputs = [ - entry, - ...globSync(join(dirname(entry), "*.{ts,mts,cts,js,mjs,cjs,json}")), - ...globSync(join(fixturesRoot, "npm/node_modules/**/*.{js,mjs,cjs,json,d.ts,node}")), - ].sort(); - for (const f of inputs) hash.update(f).update(readFileSync(f)); - const key = hash - .update(sanitize ? "san" : "plain") - .digest("hex") - .slice(0, 16); - const outDir = join(cacheDir, `npm-${key}`); - mkdirSync(outDir, { recursive: true }); - const result = await compile(entry, { outPath: join(outDir, "program"), outDir, sanitize }); - if (result.ok) return { ok: true, binary: result.binaryPath }; - return { - ok: false, - codes: result.diagnostics.map((d) => d.code), - text: result.diagnostics.map((d) => `${d.code}: ${d.message}`).join("\n"), - }; -} - -async function expectMatchesNode(c: NpmCase, binary: string): Promise { - const cwd = dirname(c.entry); - for (const argv of c.argvs ?? [[]]) { - const [nodeRes, nativeRes] = await Promise.all([ - runBinary(process.execPath, [c.entry, ...argv], cwd), - runBinary(binary, argv, cwd), - ]); - const label = argv.join(" "); - if (!nodeRes.stdout.equals(nativeRes.stdout)) { - expect(nativeRes.stdout.toString("utf8"), label).toBe(nodeRes.stdout.toString("utf8")); - expect.unreachable("stdout differed at byte level but not after utf8 decode"); - } - expect(nativeRes.exitCode, label).toBe(nodeRes.exitCode); - if (c.compareStderr) expect(programStderr(nativeRes.stderr), label).toEqual(nodeRes.stderr); - } -} - -test("every KNOWN entry names a case", () => { - const names = new Set(npmCases(fixturesRoot).map((c) => c.name)); - expect(Object.keys(KNOWN).filter((name) => !names.has(name))).toEqual([]); -}); - -describe(`npm native (${cases.length} programs${sanitize ? ", sanitized" : ""}${shardSuffix()})`, () => { - for (const c of cases) { - const known = KNOWN[c.name]; - if (known !== undefined && "divergence" in known) { - test.fails(`${c.name} (diverges: ${known.divergence})`, async () => { - const built = await build(c.entry); - if (!built.ok) throw new Error(built.text); - await expectMatchesNode(c, built.binary); - }, 120_000); - continue; - } - test( - c.name, - async () => { - const built = await build(c.entry); - if (known !== undefined && "build" in known) { - expect(built.ok, "the compile refuses the program").toBe(false); - if (!built.ok) expect(built.codes, built.text).toContain(known.build); - return; - } - if (!built.ok) throw new Error(`npm fixture failed to compile:\n${built.text}`); - if (known !== undefined && "run" in known) { - for (const argv of known.argvs ?? c.argvs ?? [[]]) { - const res = await runBinary(built.binary, argv, dirname(c.entry)); - const output = res.stdout.toString("utf8") + res.stderr.toString("utf8"); - expect(output, `${argv.join(" ")}: the deferred refusal`).toContain(known.run); - } - if (known.argvs === undefined) return; - const refused = new Set(known.argvs.map((argv) => JSON.stringify(argv))); - const argvs = (c.argvs ?? [[]]).filter((argv) => !refused.has(JSON.stringify(argv))); - await expectMatchesNode({ ...c, argvs }, built.binary); - return; - } - await expectMatchesNode(c, built.binary); - }, - TIMEOUTS[c.name] ?? 120_000, - ); - } -}); +defineNpmSuite({ label: "npm native" }); From 88717dd3f57c26a32fd73d4b6a1f5a6a65d0cdc4 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Sun, 25 Oct 2026 12:16:30 -0500 Subject: [PATCH 2/4] Address review feedback --- packages/compiler/src/frontend/program.ts | 12 ++++--- .../frontend/shape-inference/registry.test.ts | 34 +++++++++++++++++++ 2 files changed, 42 insertions(+), 4 deletions(-) create mode 100644 packages/compiler/src/frontend/shape-inference/registry.test.ts diff --git a/packages/compiler/src/frontend/program.ts b/packages/compiler/src/frontend/program.ts index 2dfc46bdc..c29b78e7a 100644 --- a/packages/compiler/src/frontend/program.ts +++ b/packages/compiler/src/frontend/program.ts @@ -1008,10 +1008,8 @@ function loadProgram7( } } // Shape inference's overlay: package files typed from proven facts. The - // tsc gate keeps reading the files as written. - // A load without the overlay must not see an earlier load's overlay - // files (the registry is per process). - resetShapeOverlay(); + // tsc gate keeps reading the files as written. loadProgram resets the + // overlay registry before the lowering world's load. const overlayOriginals = new Map(); if (overlayer !== undefined && jsx.refusal === null) { // Rounds: structural rewrites, then the typed overlay of the rebuilt @@ -1320,6 +1318,12 @@ export function loadProgram( return projection; }; try { + // The overlay registry is per process and describes the lowering + // world's load: a load without the overlay must not see an earlier + // load's overlay files. The authored world (authored()) is a second load + // of the same build, which preflight can create after the overlay + // rounds, so it leaves the registry as this load set it. + resetShapeOverlay(); const load = loadProgram7( host, entryPath, diff --git a/packages/compiler/src/frontend/shape-inference/registry.test.ts b/packages/compiler/src/frontend/shape-inference/registry.test.ts new file mode 100644 index 000000000..31c07daea --- /dev/null +++ b/packages/compiler/src/frontend/shape-inference/registry.test.ts @@ -0,0 +1,34 @@ +/* loadProgram resets the shape overlay's registry for the lowering world's + * load only. The authored world that preflight can create afterwards + * (load.projectWorld()) is a second load of the same build: it leaves the + * lowering load's overlay registered, so the fallback (fallback.ts) still + * finds the overlay behind a refusal. */ +import { join } from "node:path"; +import { expect, test } from "vitest"; +import { loadProgram } from "../program-node.js"; +import { shapeOverlayActive, shapeOverlayFileNames } from "./index.js"; + +const entry = join( + import.meta.dirname, + "../../../../../tests/fixtures/npm/cases/static-objects/js-main.cjs", +); + +test("the authored world leaves the lowering load's overlay registered", () => { + const saved = process.env["SCRIPTC_SHAPE_OVERLAY"]; + process.env["SCRIPTC_SHAPE_OVERLAY"] = "1"; + try { + const load = loadProgram(entry, { npmStatic: ["static-ns"] }); + try { + expect(shapeOverlayActive()).toBe(true); + const files = shapeOverlayFileNames(); + load.projectWorld(); + expect(shapeOverlayActive()).toBe(true); + expect(shapeOverlayFileNames()).toEqual(files); + } finally { + load.dispose(); + } + } finally { + if (saved === undefined) delete process.env["SCRIPTC_SHAPE_OVERLAY"]; + else process.env["SCRIPTC_SHAPE_OVERLAY"] = saved; + } +}); From 7be83d8e114c353faa92ffef8352f2e780fb0e31 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Sun, 25 Oct 2026 15:47:57 -0500 Subject: [PATCH 3/4] Fix the merge and address review feedback --- .gitignore | 2 ++ packages/compiler/node_modules | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) delete mode 120000 packages/compiler/node_modules diff --git a/.gitignore b/.gitignore index 81b92ddd5..88dfdc35b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,6 @@ node_modules/ +# A symlinked node_modules is a file to git, which node_modules/ does not match. +/packages/*/node_modules .pnpm-store/ # test-fixture packages are TEST DATA, committed on purpose !tests/diagnostics/npm-imports/node_modules/ diff --git a/packages/compiler/node_modules b/packages/compiler/node_modules deleted file mode 120000 index 6b01b0eda..000000000 --- a/packages/compiler/node_modules +++ /dev/null @@ -1 +0,0 @@ -../../../shape-reg/packages/compiler/node_modules \ No newline at end of file From 6aaaea04e4c59ffa38f40e9cf44758c2d3b7b3bb Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Sun, 25 Oct 2026 20:31:37 -0500 Subject: [PATCH 4/4] Keep one source label per node in the shape analyzer --- .../src/frontend/shape-inference/analyze.ts | 98 ++++++++++--------- 1 file changed, 53 insertions(+), 45 deletions(-) diff --git a/packages/compiler/src/frontend/shape-inference/analyze.ts b/packages/compiler/src/frontend/shape-inference/analyze.ts index 2a0ee09a1..bde5e58dd 100644 --- a/packages/compiler/src/frontend/shape-inference/analyze.ts +++ b/packages/compiler/src/frontend/shape-inference/analyze.ts @@ -317,6 +317,14 @@ export function analyzeShapes(program: ts.Program, checker: ts.TypeChecker): Sha } class Analyzer { + /** locLabel per node: sink reasons and atom labels name the same sites + * over and over, and each label reads the file's text for its line. */ + private readonly locLabels = new Map(); + private loc(node: Node): string { + let label = this.locLabels.get(node); + if (label === undefined) this.locLabels.set(node, (label = locLabel(node))); + return label; + } readonly atoms = new AtomTable(); readonly nodes: ValueNode[] = []; readonly functions = new Map(); @@ -830,7 +838,7 @@ class Analyzer { if (this.traceTop) this.nodeLabel.set( id, - `binding ${(decl as { name?: { text?: string } }).name?.text ?? "?"}@${locLabel(decl)}${this.ctx !== "" ? ` [${this.ctx}]` : ""}`, + `binding ${(decl as { name?: { text?: string } }).name?.text ?? "?"}@${this.loc(decl)}${this.ctx !== "" ? ` [${this.ctx}]` : ""}`, ); } return id; @@ -908,7 +916,7 @@ class Analyzer { const nameNode = (fn as { name?: Node }).name; const name = nameNode !== undefined ? (propertyNameText(nameNode) ?? "") : null; const owner = ownerClass !== null ? this.className(ownerClass) : null; - const label = `${owner !== null ? `${owner}.` : ""}${name ?? (ts.isConstructorDeclaration(node) ? "constructor" : "")}@${locLabel(node)}`; + const label = `${owner !== null ? `${owner}.` : ""}${name ?? (ts.isConstructorDeclaration(node) ? "constructor" : "")}@${this.loc(node)}`; const info: FunctionInfo = { node, ctx, @@ -952,7 +960,7 @@ class Analyzer { if (p.dotDotDotToken !== undefined) { const arr = this.atoms.intern(`rest:${this.nodes.length}`, () => ({ kind: "array", - label: `rest@${locLabel(p)}`, + label: `rest@${this.loc(p)}`, node: p, })); this.add(this.binding(p), arr); @@ -998,7 +1006,7 @@ class Analyzer { const p = decl.parent; if (p !== undefined && ts.isVariableDeclaration(p) && ts.isIdentifier(p.name)) return p.name.text; - return ``; + return ``; } classAtoms(decl: Node): { cls: Atom; inst: Atom } { @@ -1039,7 +1047,7 @@ class Analyzer { ctx: "", root: null, walked: true, - label: `${label}.${suffix}@${locLabel(decl)}`, + label: `${label}.${suffix}@${this.loc(decl)}`, params: [], rest: -1, thisNode: this.newNode(), @@ -1187,7 +1195,7 @@ class Analyzer { if (this.isFactory(info.node)) { const d = this.atoms.intern(`dict:${this.nodeKey(site)}`, () => ({ kind: "dictionary", - label: `Object.create@${locLabel(site)}`, + label: `Object.create@${this.loc(site)}`, node: site, })); this.add(dst, d); @@ -1253,7 +1261,7 @@ class Analyzer { syntheticArray(site: Node, tag: string): Atom { return this.atoms.intern(`arr:${tag}:${this.nodeKey(site)}`, () => ({ kind: "array", - label: `${tag}[]@${locLabel(site)}`, + label: `${tag}[]@${this.loc(site)}`, node: site, })); } @@ -1361,7 +1369,7 @@ class Analyzer { return; } if (ts.isThrowStatement(s)) { - this.sink(this.expr(s.expression), `thrown at ${locLabel(s)}`); + this.sink(this.expr(s.expression), `thrown at ${this.loc(s)}`); return; } if ( @@ -1437,7 +1445,7 @@ class Analyzer { Symbol.iterator member have a computed key; they are G sites. */ } else { this.add(dst, this.top); - if (a.kind !== "top") this.escape(a, `iterated at ${locLabel(site)}`); + if (a.kind !== "top") this.escape(a, `iterated at ${this.loc(site)}`); } }); } @@ -1453,7 +1461,7 @@ class Analyzer { if (this.traceTop && !this.nodeLabel.has(id)) this.nodeLabel.set( id, - `expr ${e.getText().slice(0, 40).replace(/\s+/g, " ")}@${locLabel(e)}`, + `expr ${e.getText().slice(0, 40).replace(/\s+/g, " ")}@${this.loc(e)}`, ); return id; } @@ -1586,7 +1594,7 @@ class Analyzer { if (ts.isTaggedTemplateExpression(e)) return this.taggedTemplate(e); if (ts.isYieldExpression(e)) { if (e.expression !== undefined) - this.sink(this.expr(e.expression), `yielded at ${locLabel(e)}`); + this.sink(this.expr(e.expression), `yielded at ${this.loc(e)}`); return this.constNode(this.top); } if (ts.isMetaProperty(e)) return this.constNode(this.top); @@ -1841,7 +1849,7 @@ class Analyzer { const spreads: number[] = []; const inPackage = isPackageJsFile(e.getSourceFile()); const label = (): string => - `{${keys.slice(0, 6).join(",")}${keys.length > 6 ? ",…" : ""}${keysKnown ? "" : ",…"}}@${locLabel(e)}`; + `{${keys.slice(0, 6).join(",")}${keys.length > 6 ? ",…" : ""}${keysKnown ? "" : ",…"}}@${this.loc(e)}`; // keys first, so the label is complete for (const p of list(e.properties)) { if (ts.isSpreadAssignment(p)) { @@ -2053,7 +2061,7 @@ class Analyzer { arrayLiteral(e: ts.ArrayLiteralExpression): number { const atom = this.atoms.intern(`arr:${this.nodeKey(e)}`, () => ({ kind: "array", - label: `[]@${locLabel(e)}`, + label: `[]@${this.loc(e)}`, node: e, })); const n = this.newNode(); @@ -2175,7 +2183,7 @@ class Analyzer { if (ts.isIdentifier(t)) { const decl = this.resolve(t); if (decl === undefined || decl.getSourceFile().isDeclarationFile) { - this.sink(value, `global write at ${locLabel(t)}`); + this.sink(value, `global write at ${this.loc(t)}`); return; } this.edge(value, this.binding(decl)); @@ -2421,7 +2429,7 @@ class Analyzer { switch (a.kind) { case "top": this.noteTopWrite(name, site); - this.sink(value, `stored into ⊤.${name} at ${locLabel(site)}`); + this.sink(value, `stored into ⊤.${name} at ${this.loc(site)}`); return; case "instance": { const member = this.lookupMember(a.node!, name, false); @@ -2448,7 +2456,7 @@ class Analyzer { case "prim": return; default: - this.sink(value, `stored into ${a.label} at ${locLabel(site)}`); + this.sink(value, `stored into ${a.label} at ${this.loc(site)}`); } }); } @@ -2497,7 +2505,7 @@ class Analyzer { switch (a.kind) { case "top": this.noteTopWrite(null, site); - this.sink(value, `stored into ⊤[k] at ${locLabel(site)}`); + this.sink(value, `stored into ⊤[k] at ${this.loc(site)}`); return; case "array": // An index, or a named property under an unknown key. @@ -2515,7 +2523,7 @@ class Analyzer { return; default: // As a store under a name: a builtin, promise or collection keeps it where it isn't tracked. - this.sink(value, `stored into ${a.label}[k] at ${locLabel(site)}`); + this.sink(value, `stored into ${a.label}[k] at ${this.loc(site)}`); } }); } @@ -2591,7 +2599,7 @@ class Analyzer { if (inst.kind === "instance") this.callConstructorChain(base, inst, args, spread, e); }); } else if (base === undefined) - for (const a of args) this.sink(a, `super(...) at ${locLabel(e)}`); + for (const a of args) this.sink(a, `super(...) at ${this.loc(e)}`); return dst; } // CommonJS `require("")` of an analyzed module: its exports (the @@ -2686,9 +2694,9 @@ class Analyzer { case "prim": return; default: - for (const x of args) this.sink(x, `argument of ⊤ call at ${locLabel(site)}`); + for (const x of args) this.sink(x, `argument of ⊤ call at ${this.loc(site)}`); if (recv !== null && a.kind === "top") - this.sink(recv, `receiver of ⊤ call at ${locLabel(site)}`); + this.sink(recv, `receiver of ⊤ call at ${this.loc(site)}`); this.add(dst, this.top); } }); @@ -2731,7 +2739,7 @@ class Analyzer { this.constructBuiltin(a.path ?? "", args, dst, e); return; default: - for (const x of args) this.sink(x, `argument of ⊤ constructor at ${locLabel(e)}`); + for (const x of args) this.sink(x, `argument of ⊤ constructor at ${this.loc(e)}`); this.add(dst, this.top); } }); @@ -2746,7 +2754,7 @@ class Analyzer { case "WeakSet": { const c = this.atoms.intern(`coll:${this.nodeKey(site)}`, () => ({ kind: "collection", - label: `${path}@${locLabel(site)}`, + label: `${path}@${this.loc(site)}`, node: site, })); this.add(dst, c); @@ -2760,7 +2768,7 @@ class Analyzer { this.add(dst, this.builtin("Error.instance")); return; case "Promise": { - const p = this.promiseAtom(site, locLabel(site)); + const p = this.promiseAtom(site, this.loc(site)); this.add(this.field(p, "value"), this.top); this.add(dst, p); if (args[0] !== undefined) { @@ -2793,11 +2801,11 @@ class Analyzer { // The handler's traps run on every operation with the target. if (args[0] !== undefined) this.ops.push({ kind: "proxy", recv: args[0], name: null, node: site }); - for (const x of args) this.sink(x, `argument of new Proxy at ${locLabel(site)}`); + for (const x of args) this.sink(x, `argument of new Proxy at ${this.loc(site)}`); this.add(dst, this.top); return; default: - for (const x of args) this.sink(x, `argument of new ${path} at ${locLabel(site)}`); + for (const x of args) this.sink(x, `argument of new ${path} at ${this.loc(site)}`); this.add(dst, this.top); } } @@ -2824,13 +2832,13 @@ class Analyzer { this.callFunction(this.infoOfAtom(f)!, null, cbArgs, false, ret ?? this.newNode(), site); this.forceLoop = saved; } else if (f.kind === "top" || f.kind === "builtin") { - for (const x of cbArgs) this.sink(x, `argument of ⊤ callback at ${locLabel(site)}`); + for (const x of cbArgs) this.sink(x, `argument of ⊤ callback at ${this.loc(site)}`); if (ret !== null) this.add(ret, this.top); } }); }; const escapeAll = (): void => { - for (const x of args) this.sink(x, `argument of ${path} at ${locLabel(site)}`); + for (const x of args) this.sink(x, `argument of ${path} at ${this.loc(site)}`); this.add(dst, this.top); }; // Methods of builtin receivers. @@ -2948,7 +2956,7 @@ class Analyzer { return; } escapeAll(); - this.escape(recv, `array.${method} at ${locLabel(site)}`); + this.escape(recv, `array.${method} at ${this.loc(site)}`); return; } if (path.startsWith("string.") || path.startsWith("number.") || path.startsWith("boolean.")) { @@ -2977,7 +2985,7 @@ class Analyzer { return; } if (path.startsWith("promise.")) { - const out = this.promiseAtom(site, locLabel(site)); + const out = this.promiseAtom(site, this.loc(site)); const value = this.field(recv, "value"); const ret = this.newNode(); if (method === "then") { @@ -3047,7 +3055,7 @@ class Analyzer { if (info !== undefined && args.length > 1) { this.activate(info); for (const p of info.params) this.add(p, this.top); - for (const x of args.slice(1)) this.sink(x, `bound argument at ${locLabel(site)}`); + for (const x of args.slice(1)) this.sink(x, `bound argument at ${this.loc(site)}`); } this.add(dst, recv); return; @@ -3145,7 +3153,7 @@ class Analyzer { for (const s of args.slice(1)) this.copyFields(s, t, site); if (t.kind === "top") for (const s of args.slice(1)) - this.sink(s, `Object.assign into ⊤ at ${locLabel(site)}`); + this.sink(s, `Object.assign into ⊤ at ${this.loc(site)}`); }); this.edge(arg(0), dst); return; @@ -3154,20 +3162,20 @@ class Analyzer { case "Object.defineProperties": case "Reflect.defineProperty": this.ops.push({ kind: "defineProperty", recv: arg(0), name: null, node: site }); - this.sink(arg(2), `property descriptor at ${locLabel(site)}`); - this.sink(arg(1), `property descriptor at ${locLabel(site)}`); + this.sink(arg(2), `property descriptor at ${this.loc(site)}`); + this.sink(arg(1), `property descriptor at ${this.loc(site)}`); this.edge(arg(0), dst); return; case "Object.setPrototypeOf": case "Reflect.setPrototypeOf": this.ops.push({ kind: "setPrototypeOf", recv: arg(0), name: null, node: site }); - this.sink(arg(1), `prototype at ${locLabel(site)}`); + this.sink(arg(1), `prototype at ${this.loc(site)}`); this.edge(arg(0), dst); return; case "Object.create": { const d = this.atoms.intern(`dict:${this.nodeKey(site)}`, () => ({ kind: "dictionary", - label: `Object.create@${locLabel(site)}`, + label: `Object.create@${this.loc(site)}`, node: site, })); this.add(dst, d); @@ -3176,7 +3184,7 @@ class Analyzer { case "Object.fromEntries": { const d = this.atoms.intern(`dict:${this.nodeKey(site)}`, () => ({ kind: "dictionary", - label: `fromEntries@${locLabel(site)}`, + label: `fromEntries@${this.loc(site)}`, node: site, })); const pairs = this.newNode(); @@ -3225,7 +3233,7 @@ class Analyzer { site, ); else if (f.kind === "top") - this.sink(revived, `JSON.parse reviver at ${locLabel(site)}`); + this.sink(revived, `JSON.parse reviver at ${this.loc(site)}`); }); this.edge(revived, dst); return; @@ -3252,7 +3260,7 @@ class Analyzer { site, ); else if (f.kind === "top") - this.sink(serialized, `JSON.stringify replacer at ${locLabel(site)}`); + this.sink(serialized, `JSON.stringify replacer at ${this.loc(site)}`); }); prim("string"); prim("undefined"); @@ -3281,7 +3289,7 @@ class Analyzer { if (c.kind === "builtin" && c.path === "Proxy") this.ops.push({ kind: "proxy", recv: listed, name: null, node: site }); }); - this.sink(listed, `argument of ${path} at ${locLabel(site)}`); + this.sink(listed, `argument of ${path} at ${this.loc(site)}`); escapeAll(); return; } @@ -3310,23 +3318,23 @@ class Analyzer { prim("bigint"); return; case "Promise.resolve": { - const p = this.promiseAtom(site, locLabel(site)); + const p = this.promiseAtom(site, this.loc(site)); if (args[0] !== undefined) this.awaitInto(args[0], this.field(p, "value")); else this.add(this.field(p, "value"), this.prim("undefined")); this.add(dst, p); return; } case "Promise.reject": { - const p = this.promiseAtom(site, locLabel(site)); + const p = this.promiseAtom(site, this.loc(site)); this.add(dst, p); - if (args[0] !== undefined) this.sink(args[0], `rejection at ${locLabel(site)}`); + if (args[0] !== undefined) this.sink(args[0], `rejection at ${this.loc(site)}`); return; } case "Promise.all": case "Promise.allSettled": case "Promise.race": case "Promise.any": { - const p = this.promiseAtom(site, locLabel(site)); + const p = this.promiseAtom(site, this.loc(site)); const items = this.newNode(); if (args[0] !== undefined) this.iterateInto(args[0], items, site); if (path === "Promise.all") { @@ -3340,7 +3348,7 @@ class Analyzer { return; } case "promise.resolver": - for (const x of args) this.sink(x, `promise resolution at ${locLabel(site)}`); + for (const x of args) this.sink(x, `promise resolution at ${this.loc(site)}`); return; } if (path.startsWith("console.")) {