diff --git a/.gitignore b/.gitignore index 81b92ddd5..88dfdc35b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,6 @@ node_modules/ +# A symlinked node_modules is a file to git, which node_modules/ does not match. +/packages/*/node_modules .pnpm-store/ # test-fixture packages are TEST DATA, committed on purpose !tests/diagnostics/npm-imports/node_modules/ diff --git a/packages/compiler/src/compile-types.ts b/packages/compiler/src/compile-types.ts index da78905b3..afe3ce1b6 100644 --- a/packages/compiler/src/compile-types.ts +++ b/packages/compiler/src/compile-types.ts @@ -68,7 +68,8 @@ export interface CompileBaseOptions { /** Eligible packages compile automatically. Explicit names override automatic selection heuristics. */ npmStatic?: readonly string[] | "auto"; /** Shape inference's typed overlay for opted-in packages' JavaScript. - * False disables it (`--no-shape-overlay`); a package opts out with + * True enables it, false disables it (`--no-shape-overlay`); unset + * follows SCRIPTC_SHAPE_OVERLAY. A package opts out with * `"scriptc": { "packages": { "": { "shapes": false } } }`. */ shapeOverlay?: boolean; /** Outbound native FFI manifest. Its signature-only TypeScript bindings diff --git a/packages/compiler/src/frontend/pipeline.ts b/packages/compiler/src/frontend/pipeline.ts index 1255db8a1..b80ce32d0 100644 --- a/packages/compiler/src/frontend/pipeline.ts +++ b/packages/compiler/src/frontend/pipeline.ts @@ -1,6 +1,11 @@ -import { shapeOverlayReport } from "./shape-inference/index.js"; /** Shared frontend orchestration. Hosts supply a program loader so Node * and native callers use the same npm selection, preflight and lowering. */ +import { + beginShapeOverlayBuild, + excludeRefusedOverlay, + shapeOverlayActive, + shapeOverlayReport, +} from "./shape-inference/index.js"; import type { ScrDiagnostic } from "../diagnostics/diagnostic.js"; import type { SrcLoc } from "../ir/ir.js"; import type { NpmStaticStatus } from "../coverage/report.js"; @@ -496,6 +501,10 @@ export function runFrontend( } } +/** Rounds of dropping single refused overlay annotations before the + * fallback drops whole files' overlays. */ +const OVERLAY_ANNOTATION_ROUNDS = 4; + function loadFrontend( entryPath: string, loadProgram: ProgramLoader, @@ -510,6 +519,7 @@ function loadFrontend( // separate compiles in a long-lived process. A cache miss must observe // package.json edits before it can publish a new early-library entry. clearResolveCaches(); + beginShapeOverlayBuild(); const statuses: NpmStaticStatus[] = []; const npmSites = new Map(); // Node island detection (report-only): how each package was refused, and @@ -730,7 +740,9 @@ function loadFrontend( } } - const finalLoad = load; + // The lowering may reload once the overlay drops refused annotations. + let finalLoad = load; + const finalNpmStatic = [...effective]; // An unknown key or value in the project's scriptc configuration makes // the Node island report unavailable (SC8004). The settings configure // only that report: a build warns about them and goes on. @@ -790,16 +802,63 @@ function loadFrontend( return texts; }, lower: (opts) => { - const lowered = lowerToIr(finalLoad.program, finalLoad.entry, finalLoad.moduleOrder, { - ...opts, - frontendServices: finalLoad.services, - startupCrash: finalLoad.startupCrash ?? null, - inlineRequireCycleCalls: finalLoad.inlineRequireCycleCalls, - moduleEdges: finalLoad.moduleEdges, - projectCheckJs: finalLoad.typeWorld?.projectCheckJs ?? false, - externalTypes: finalLoad.externalTypes, - externalTypeSpecifiersByFile: finalLoad.externalTypeSpecifiersByFile, - }); + const lowerOnce = (): LowerResult => + lowerToIr(finalLoad.program, finalLoad.entry, finalLoad.moduleOrder, { + ...opts, + frontendServices: finalLoad.services, + startupCrash: finalLoad.startupCrash ?? null, + inlineRequireCycleCalls: finalLoad.inlineRequireCycleCalls, + moduleEdges: finalLoad.moduleEdges, + projectCheckJs: finalLoad.typeWorld?.projectCheckJs ?? false, + externalTypes: finalLoad.externalTypes, + externalTypeSpecifiersByFile: finalLoad.externalTypeSpecifiersByFile, + }); + let lowered = lowerOnce(); + // Shape inference never breaks compilation (shape-inference/ + // fallback.ts): refusals drop the overlay annotations they depend on + // and the program reloads. Rounds that stop converging escalate to + // the overlay of every file involved, then to no overlay at all. + // Refusals deferred to run time (runtime fences) drop only the + // annotations they are blamed on: a fence the overlay did not cause + // stays as it is without the overlay. + let level: "annotations" | "files" | "all" | "fences" = "annotations"; + for (let round = 0; shapeOverlayActive(); round++) { + const failed = lowered.diagnostics.length > 0; + const refusals = failed ? lowered.diagnostics : lowered.runtimeFences; + if (refusals.length === 0) break; + let changed: boolean; + if (!failed) { + level = "fences"; + changed = + round < OVERLAY_ANNOTATION_ROUNDS && + excludeRefusedOverlay(finalLoad.program, refusals, level); + } else { + if (level === "fences") level = "annotations"; + if (round >= OVERLAY_ANNOTATION_ROUNDS && level === "annotations") level = "files"; + else if (round > OVERLAY_ANNOTATION_ROUNDS) level = "all"; + changed = excludeRefusedOverlay(finalLoad.program, refusals, level); + while (!changed && level !== "all") { + level = level === "annotations" ? "files" : "all"; + changed = excludeRefusedOverlay(finalLoad.program, refusals, level); + } + } + if (process.env["SCRIPTC_DEBUG_SHAPE_FALLBACK"] !== undefined) + process.stderr.write( + `[shape-fallback] round ${round} level ${level}: ${refusals.length} refusals (${refusals + .slice(0, 3) + .map((d) => `${d.loc.file.split("/").pop()}:${d.loc.start} ${d.message.slice(0, 60)}`) + .join("; ")}), changed ${changed}\n`, + ); + if (!changed) break; + finalLoad.dispose(); + finalLoad = loadProgram(entryPath, { npmStatic: finalNpmStatic, externalTypes }); + // The preflight fills the module order; the tsc world reads the + // files as written, so it answers as the first load did. + const reloadPreflight = checkPreflight(finalLoad); + if (reloadPreflight.length > 0) + return { ...lowered, module: null, diagnostics: reloadPreflight }; + lowered = lowerOnce(); + } // The declaration-typed boundary's notes (SC6102) join the // performance warnings; they never fail a build. const notes = [ @@ -834,6 +893,6 @@ function loadFrontend( refusals, }; }, - dispose: finalLoad.dispose, + dispose: () => finalLoad.dispose(), }; } diff --git a/packages/compiler/src/frontend/program.ts b/packages/compiler/src/frontend/program.ts index 67a333189..904d30d5b 100644 --- a/packages/compiler/src/frontend/program.ts +++ b/packages/compiler/src/frontend/program.ts @@ -1012,10 +1012,9 @@ function loadProgram7( } } // Shape inference's overlay: package files typed from proven facts. The - // tsc gate keeps reading the files as written. - // A load without the overlay must not see an earlier load's overlay - // files (the registry is per process). - resetShapeOverlay(); + // tsc gate keeps reading the files as written. loadProgram resets the + // overlay registry before the lowering world's load. + const overlayOriginals = new Map(); if (overlayer !== undefined && jsx.refusal === null) { // Rounds: structural rewrites, then the typed overlay of the rebuilt // program (shape-inference/index.ts). @@ -1023,15 +1022,23 @@ function loadProgram7( const overlay = overlayer(program, round); if (overlay === null || overlay.size === 0) break; const before = program; - for (const [file, text] of overlay) host.addVirtualFile(file, text); - program = host.createProgram([...programRoots, overridesDtsPath()], options); + // The tsc gate reads the files as written: its diagnostics are + // taken before the overlay's files replace them on the host, and + // its project world (load.projectWorld()) is built over the files + // as written. if (tscWorld === undefined) tscWorld = { program: before, diagnostics: ts.getPreEmitDiagnostics(before), projectWorld: null, + declWorldFallback: true, }; - else if (tscWorld.program !== before) before.dispose(); + for (const file of overlay.keys()) + if (!overlayOriginals.has(file)) + overlayOriginals.set(file, before.getSourceFile(file)?.text ?? null); + for (const [file, text] of overlay) host.addVirtualFile(file, text); + program = host.createProgram([...programRoots, overridesDtsPath()], options); + if (tscWorld.program !== before) before.dispose(); if (shapeOverlayDone()) break; } } @@ -1055,7 +1062,10 @@ function loadProgram7( externalTypeSpecifiersByFile, ...(tscWorld === undefined ? {} : { jsxSources: jsx.sources }), ...(tscWorld === undefined ? {} : { tscWorld }), - projectWorld: () => (projectWorld ??= host.createProgram(programRoots, options)), + projectWorld: () => + (projectWorld ??= withFilesAsWritten(host, overlayOriginals, () => + host.createProgram(programRoots, options), + )), disposeAll: () => { try { projectWorld?.dispose(); @@ -1083,6 +1093,27 @@ export interface ProgramLoadOptions { externalTypes?: readonly (readonly [string, string])[]; } +/** Runs `build` with the overlay's files served as written (their original + * texts), then serves the overlay's texts again. */ +function withFilesAsWritten( + host: ts.Ts7Host, + originals: ReadonlyMap, + build: () => T, +): T { + if (originals.size === 0) return build(); + const overlaid = new Map(); + for (const [file, text] of originals) { + const current = host.virtualFileText(file); + if (current !== undefined) overlaid.set(file, current); + if (text !== null) host.addVirtualFile(file, text); + } + try { + return build(); + } finally { + for (const [file, text] of overlaid) host.addVirtualFile(file, text); + } +} + export function loadProgram( entryPath: string, services: FrontendServices, @@ -1296,6 +1327,12 @@ export function loadProgram( return projection; }; try { + // The overlay registry is per process and describes the lowering + // world's load: a load without the overlay must not see an earlier + // load's overlay files. The authored world (authored()) is a second load + // of the same build, which preflight can create after the overlay + // rounds, so it leaves the registry as this load set it. + resetShapeOverlay(); const load = loadProgram7( host, entryPath, diff --git a/packages/compiler/src/frontend/shape-inference/analyze.ts b/packages/compiler/src/frontend/shape-inference/analyze.ts index 2a0ee09a1..bde5e58dd 100644 --- a/packages/compiler/src/frontend/shape-inference/analyze.ts +++ b/packages/compiler/src/frontend/shape-inference/analyze.ts @@ -317,6 +317,14 @@ export function analyzeShapes(program: ts.Program, checker: ts.TypeChecker): Sha } class Analyzer { + /** locLabel per node: sink reasons and atom labels name the same sites + * over and over, and each label reads the file's text for its line. */ + private readonly locLabels = new Map(); + private loc(node: Node): string { + let label = this.locLabels.get(node); + if (label === undefined) this.locLabels.set(node, (label = locLabel(node))); + return label; + } readonly atoms = new AtomTable(); readonly nodes: ValueNode[] = []; readonly functions = new Map(); @@ -830,7 +838,7 @@ class Analyzer { if (this.traceTop) this.nodeLabel.set( id, - `binding ${(decl as { name?: { text?: string } }).name?.text ?? "?"}@${locLabel(decl)}${this.ctx !== "" ? ` [${this.ctx}]` : ""}`, + `binding ${(decl as { name?: { text?: string } }).name?.text ?? "?"}@${this.loc(decl)}${this.ctx !== "" ? ` [${this.ctx}]` : ""}`, ); } return id; @@ -908,7 +916,7 @@ class Analyzer { const nameNode = (fn as { name?: Node }).name; const name = nameNode !== undefined ? (propertyNameText(nameNode) ?? "") : null; const owner = ownerClass !== null ? this.className(ownerClass) : null; - const label = `${owner !== null ? `${owner}.` : ""}${name ?? (ts.isConstructorDeclaration(node) ? "constructor" : "")}@${locLabel(node)}`; + const label = `${owner !== null ? `${owner}.` : ""}${name ?? (ts.isConstructorDeclaration(node) ? "constructor" : "")}@${this.loc(node)}`; const info: FunctionInfo = { node, ctx, @@ -952,7 +960,7 @@ class Analyzer { if (p.dotDotDotToken !== undefined) { const arr = this.atoms.intern(`rest:${this.nodes.length}`, () => ({ kind: "array", - label: `rest@${locLabel(p)}`, + label: `rest@${this.loc(p)}`, node: p, })); this.add(this.binding(p), arr); @@ -998,7 +1006,7 @@ class Analyzer { const p = decl.parent; if (p !== undefined && ts.isVariableDeclaration(p) && ts.isIdentifier(p.name)) return p.name.text; - return ``; + return ``; } classAtoms(decl: Node): { cls: Atom; inst: Atom } { @@ -1039,7 +1047,7 @@ class Analyzer { ctx: "", root: null, walked: true, - label: `${label}.${suffix}@${locLabel(decl)}`, + label: `${label}.${suffix}@${this.loc(decl)}`, params: [], rest: -1, thisNode: this.newNode(), @@ -1187,7 +1195,7 @@ class Analyzer { if (this.isFactory(info.node)) { const d = this.atoms.intern(`dict:${this.nodeKey(site)}`, () => ({ kind: "dictionary", - label: `Object.create@${locLabel(site)}`, + label: `Object.create@${this.loc(site)}`, node: site, })); this.add(dst, d); @@ -1253,7 +1261,7 @@ class Analyzer { syntheticArray(site: Node, tag: string): Atom { return this.atoms.intern(`arr:${tag}:${this.nodeKey(site)}`, () => ({ kind: "array", - label: `${tag}[]@${locLabel(site)}`, + label: `${tag}[]@${this.loc(site)}`, node: site, })); } @@ -1361,7 +1369,7 @@ class Analyzer { return; } if (ts.isThrowStatement(s)) { - this.sink(this.expr(s.expression), `thrown at ${locLabel(s)}`); + this.sink(this.expr(s.expression), `thrown at ${this.loc(s)}`); return; } if ( @@ -1437,7 +1445,7 @@ class Analyzer { Symbol.iterator member have a computed key; they are G sites. */ } else { this.add(dst, this.top); - if (a.kind !== "top") this.escape(a, `iterated at ${locLabel(site)}`); + if (a.kind !== "top") this.escape(a, `iterated at ${this.loc(site)}`); } }); } @@ -1453,7 +1461,7 @@ class Analyzer { if (this.traceTop && !this.nodeLabel.has(id)) this.nodeLabel.set( id, - `expr ${e.getText().slice(0, 40).replace(/\s+/g, " ")}@${locLabel(e)}`, + `expr ${e.getText().slice(0, 40).replace(/\s+/g, " ")}@${this.loc(e)}`, ); return id; } @@ -1586,7 +1594,7 @@ class Analyzer { if (ts.isTaggedTemplateExpression(e)) return this.taggedTemplate(e); if (ts.isYieldExpression(e)) { if (e.expression !== undefined) - this.sink(this.expr(e.expression), `yielded at ${locLabel(e)}`); + this.sink(this.expr(e.expression), `yielded at ${this.loc(e)}`); return this.constNode(this.top); } if (ts.isMetaProperty(e)) return this.constNode(this.top); @@ -1841,7 +1849,7 @@ class Analyzer { const spreads: number[] = []; const inPackage = isPackageJsFile(e.getSourceFile()); const label = (): string => - `{${keys.slice(0, 6).join(",")}${keys.length > 6 ? ",…" : ""}${keysKnown ? "" : ",…"}}@${locLabel(e)}`; + `{${keys.slice(0, 6).join(",")}${keys.length > 6 ? ",…" : ""}${keysKnown ? "" : ",…"}}@${this.loc(e)}`; // keys first, so the label is complete for (const p of list(e.properties)) { if (ts.isSpreadAssignment(p)) { @@ -2053,7 +2061,7 @@ class Analyzer { arrayLiteral(e: ts.ArrayLiteralExpression): number { const atom = this.atoms.intern(`arr:${this.nodeKey(e)}`, () => ({ kind: "array", - label: `[]@${locLabel(e)}`, + label: `[]@${this.loc(e)}`, node: e, })); const n = this.newNode(); @@ -2175,7 +2183,7 @@ class Analyzer { if (ts.isIdentifier(t)) { const decl = this.resolve(t); if (decl === undefined || decl.getSourceFile().isDeclarationFile) { - this.sink(value, `global write at ${locLabel(t)}`); + this.sink(value, `global write at ${this.loc(t)}`); return; } this.edge(value, this.binding(decl)); @@ -2421,7 +2429,7 @@ class Analyzer { switch (a.kind) { case "top": this.noteTopWrite(name, site); - this.sink(value, `stored into ⊤.${name} at ${locLabel(site)}`); + this.sink(value, `stored into ⊤.${name} at ${this.loc(site)}`); return; case "instance": { const member = this.lookupMember(a.node!, name, false); @@ -2448,7 +2456,7 @@ class Analyzer { case "prim": return; default: - this.sink(value, `stored into ${a.label} at ${locLabel(site)}`); + this.sink(value, `stored into ${a.label} at ${this.loc(site)}`); } }); } @@ -2497,7 +2505,7 @@ class Analyzer { switch (a.kind) { case "top": this.noteTopWrite(null, site); - this.sink(value, `stored into ⊤[k] at ${locLabel(site)}`); + this.sink(value, `stored into ⊤[k] at ${this.loc(site)}`); return; case "array": // An index, or a named property under an unknown key. @@ -2515,7 +2523,7 @@ class Analyzer { return; default: // As a store under a name: a builtin, promise or collection keeps it where it isn't tracked. - this.sink(value, `stored into ${a.label}[k] at ${locLabel(site)}`); + this.sink(value, `stored into ${a.label}[k] at ${this.loc(site)}`); } }); } @@ -2591,7 +2599,7 @@ class Analyzer { if (inst.kind === "instance") this.callConstructorChain(base, inst, args, spread, e); }); } else if (base === undefined) - for (const a of args) this.sink(a, `super(...) at ${locLabel(e)}`); + for (const a of args) this.sink(a, `super(...) at ${this.loc(e)}`); return dst; } // CommonJS `require("")` of an analyzed module: its exports (the @@ -2686,9 +2694,9 @@ class Analyzer { case "prim": return; default: - for (const x of args) this.sink(x, `argument of ⊤ call at ${locLabel(site)}`); + for (const x of args) this.sink(x, `argument of ⊤ call at ${this.loc(site)}`); if (recv !== null && a.kind === "top") - this.sink(recv, `receiver of ⊤ call at ${locLabel(site)}`); + this.sink(recv, `receiver of ⊤ call at ${this.loc(site)}`); this.add(dst, this.top); } }); @@ -2731,7 +2739,7 @@ class Analyzer { this.constructBuiltin(a.path ?? "", args, dst, e); return; default: - for (const x of args) this.sink(x, `argument of ⊤ constructor at ${locLabel(e)}`); + for (const x of args) this.sink(x, `argument of ⊤ constructor at ${this.loc(e)}`); this.add(dst, this.top); } }); @@ -2746,7 +2754,7 @@ class Analyzer { case "WeakSet": { const c = this.atoms.intern(`coll:${this.nodeKey(site)}`, () => ({ kind: "collection", - label: `${path}@${locLabel(site)}`, + label: `${path}@${this.loc(site)}`, node: site, })); this.add(dst, c); @@ -2760,7 +2768,7 @@ class Analyzer { this.add(dst, this.builtin("Error.instance")); return; case "Promise": { - const p = this.promiseAtom(site, locLabel(site)); + const p = this.promiseAtom(site, this.loc(site)); this.add(this.field(p, "value"), this.top); this.add(dst, p); if (args[0] !== undefined) { @@ -2793,11 +2801,11 @@ class Analyzer { // The handler's traps run on every operation with the target. if (args[0] !== undefined) this.ops.push({ kind: "proxy", recv: args[0], name: null, node: site }); - for (const x of args) this.sink(x, `argument of new Proxy at ${locLabel(site)}`); + for (const x of args) this.sink(x, `argument of new Proxy at ${this.loc(site)}`); this.add(dst, this.top); return; default: - for (const x of args) this.sink(x, `argument of new ${path} at ${locLabel(site)}`); + for (const x of args) this.sink(x, `argument of new ${path} at ${this.loc(site)}`); this.add(dst, this.top); } } @@ -2824,13 +2832,13 @@ class Analyzer { this.callFunction(this.infoOfAtom(f)!, null, cbArgs, false, ret ?? this.newNode(), site); this.forceLoop = saved; } else if (f.kind === "top" || f.kind === "builtin") { - for (const x of cbArgs) this.sink(x, `argument of ⊤ callback at ${locLabel(site)}`); + for (const x of cbArgs) this.sink(x, `argument of ⊤ callback at ${this.loc(site)}`); if (ret !== null) this.add(ret, this.top); } }); }; const escapeAll = (): void => { - for (const x of args) this.sink(x, `argument of ${path} at ${locLabel(site)}`); + for (const x of args) this.sink(x, `argument of ${path} at ${this.loc(site)}`); this.add(dst, this.top); }; // Methods of builtin receivers. @@ -2948,7 +2956,7 @@ class Analyzer { return; } escapeAll(); - this.escape(recv, `array.${method} at ${locLabel(site)}`); + this.escape(recv, `array.${method} at ${this.loc(site)}`); return; } if (path.startsWith("string.") || path.startsWith("number.") || path.startsWith("boolean.")) { @@ -2977,7 +2985,7 @@ class Analyzer { return; } if (path.startsWith("promise.")) { - const out = this.promiseAtom(site, locLabel(site)); + const out = this.promiseAtom(site, this.loc(site)); const value = this.field(recv, "value"); const ret = this.newNode(); if (method === "then") { @@ -3047,7 +3055,7 @@ class Analyzer { if (info !== undefined && args.length > 1) { this.activate(info); for (const p of info.params) this.add(p, this.top); - for (const x of args.slice(1)) this.sink(x, `bound argument at ${locLabel(site)}`); + for (const x of args.slice(1)) this.sink(x, `bound argument at ${this.loc(site)}`); } this.add(dst, recv); return; @@ -3145,7 +3153,7 @@ class Analyzer { for (const s of args.slice(1)) this.copyFields(s, t, site); if (t.kind === "top") for (const s of args.slice(1)) - this.sink(s, `Object.assign into ⊤ at ${locLabel(site)}`); + this.sink(s, `Object.assign into ⊤ at ${this.loc(site)}`); }); this.edge(arg(0), dst); return; @@ -3154,20 +3162,20 @@ class Analyzer { case "Object.defineProperties": case "Reflect.defineProperty": this.ops.push({ kind: "defineProperty", recv: arg(0), name: null, node: site }); - this.sink(arg(2), `property descriptor at ${locLabel(site)}`); - this.sink(arg(1), `property descriptor at ${locLabel(site)}`); + this.sink(arg(2), `property descriptor at ${this.loc(site)}`); + this.sink(arg(1), `property descriptor at ${this.loc(site)}`); this.edge(arg(0), dst); return; case "Object.setPrototypeOf": case "Reflect.setPrototypeOf": this.ops.push({ kind: "setPrototypeOf", recv: arg(0), name: null, node: site }); - this.sink(arg(1), `prototype at ${locLabel(site)}`); + this.sink(arg(1), `prototype at ${this.loc(site)}`); this.edge(arg(0), dst); return; case "Object.create": { const d = this.atoms.intern(`dict:${this.nodeKey(site)}`, () => ({ kind: "dictionary", - label: `Object.create@${locLabel(site)}`, + label: `Object.create@${this.loc(site)}`, node: site, })); this.add(dst, d); @@ -3176,7 +3184,7 @@ class Analyzer { case "Object.fromEntries": { const d = this.atoms.intern(`dict:${this.nodeKey(site)}`, () => ({ kind: "dictionary", - label: `fromEntries@${locLabel(site)}`, + label: `fromEntries@${this.loc(site)}`, node: site, })); const pairs = this.newNode(); @@ -3225,7 +3233,7 @@ class Analyzer { site, ); else if (f.kind === "top") - this.sink(revived, `JSON.parse reviver at ${locLabel(site)}`); + this.sink(revived, `JSON.parse reviver at ${this.loc(site)}`); }); this.edge(revived, dst); return; @@ -3252,7 +3260,7 @@ class Analyzer { site, ); else if (f.kind === "top") - this.sink(serialized, `JSON.stringify replacer at ${locLabel(site)}`); + this.sink(serialized, `JSON.stringify replacer at ${this.loc(site)}`); }); prim("string"); prim("undefined"); @@ -3281,7 +3289,7 @@ class Analyzer { if (c.kind === "builtin" && c.path === "Proxy") this.ops.push({ kind: "proxy", recv: listed, name: null, node: site }); }); - this.sink(listed, `argument of ${path} at ${locLabel(site)}`); + this.sink(listed, `argument of ${path} at ${this.loc(site)}`); escapeAll(); return; } @@ -3310,23 +3318,23 @@ class Analyzer { prim("bigint"); return; case "Promise.resolve": { - const p = this.promiseAtom(site, locLabel(site)); + const p = this.promiseAtom(site, this.loc(site)); if (args[0] !== undefined) this.awaitInto(args[0], this.field(p, "value")); else this.add(this.field(p, "value"), this.prim("undefined")); this.add(dst, p); return; } case "Promise.reject": { - const p = this.promiseAtom(site, locLabel(site)); + const p = this.promiseAtom(site, this.loc(site)); this.add(dst, p); - if (args[0] !== undefined) this.sink(args[0], `rejection at ${locLabel(site)}`); + if (args[0] !== undefined) this.sink(args[0], `rejection at ${this.loc(site)}`); return; } case "Promise.all": case "Promise.allSettled": case "Promise.race": case "Promise.any": { - const p = this.promiseAtom(site, locLabel(site)); + const p = this.promiseAtom(site, this.loc(site)); const items = this.newNode(); if (args[0] !== undefined) this.iterateInto(args[0], items, site); if (path === "Promise.all") { @@ -3340,7 +3348,7 @@ class Analyzer { return; } case "promise.resolver": - for (const x of args) this.sink(x, `promise resolution at ${locLabel(site)}`); + for (const x of args) this.sink(x, `promise resolution at ${this.loc(site)}`); return; } if (path.startsWith("console.")) { diff --git a/packages/compiler/src/frontend/shape-inference/fallback.ts b/packages/compiler/src/frontend/shape-inference/fallback.ts new file mode 100644 index 000000000..6a2824990 --- /dev/null +++ b/packages/compiler/src/frontend/shape-inference/fallback.ts @@ -0,0 +1,242 @@ +/* Shape inference never breaks compilation: when the lowering refuses code + * with the overlay on, the overlay annotations the refusal depends on are + * dropped and the program is loaded and lowered again, so those + * declarations compile untyped (as they do without the overlay). + * + * Attribution reads the refused program. A refusal depends on: + * - the annotations of the declarations enclosing it (an annotated + * function's parameter types, a typed local, a receiver cast); + * - the annotations of the declarations its expression names (a method + * whose annotated return type the refused member access reads, a field + * whose annotated type the access sees), followed through the return + * expressions of unannotated functions, whose result types the checker + * infers from what they return. + * + * pipeline.ts escalates when a round does not converge: first the + * overlay of every file involved, then the whole overlay for the build. */ +import * as ts from "../ts7/adapter.js"; +import type { ScrDiagnostic } from "../../diagnostics/diagnostic.js"; +import { OVERLAY_MARK, overlayAnnotationIds } from "./overlay.js"; + +export interface OverlayAttribution { + /** Annotation ids per overlay file. */ + readonly annotations: Map>; + /** Overlay files a refusal involves (where it sits, or what it names). */ + readonly files: Set; + /** Refusals with no overlay annotation or file to blame. */ + readonly unattributed: number; +} + +/** How far the attribution follows unannotated functions' return + * expressions. */ +const RETURN_DEPTH = 3; + +export function attributeOverlayRefusals( + program: ts.Program, + diagnostics: readonly ScrDiagnostic[], + overlayFiles: ReadonlySet, +): OverlayAttribution { + const annotations = new Map>(); + const files = new Set(); + let unattributed = 0; + let checker: ts.TypeChecker | null = null; + // Blamed annotations and files so far (a refusal that adds none is + // unattributed). + let blamed = 0; + const addFile = (file: string): void => { + if (files.has(file)) return; + files.add(file); + blamed++; + }; + const add = (file: string, id: number): void => { + let set = annotations.get(file); + if (set === undefined) annotations.set(file, (set = new Set())); + if (!set.has(id)) blamed++; + set.add(id); + addFile(file); + }; + // The ids in the trivia before a node (where the overlay writes them). + const marksOf = (node: ts.Node): number[] => { + const sf = node.getSourceFile(); + if (!overlayFiles.has(sf.fileName)) return []; + const trivia = sf.text.slice(node.pos, ts.tokenStartOfNode(node)); + return trivia.includes(OVERLAY_MARK) ? overlayAnnotationIds(trivia) : []; + }; + const blameNode = (node: ts.Node): boolean => { + const ids = marksOf(node); + for (const id of ids) add(node.getSourceFile().fileName, id); + return ids.length > 0; + }; + // A declaration's annotation: on the declaration, or on the statement + // holding it (`/** @type */ const x = …`, `/** @type */ this.f = …`). + const visitedDecls = new Set(); + const blameDeclaration = (decl: ts.Node, depth: number): void => { + if (visitedDecls.has(decl)) return; + visitedDecls.add(decl); + const sf = decl.getSourceFile(); + if (!overlayFiles.has(sf.fileName)) return; + addFile(sf.fileName); + let marked = false; + for (let n: ts.Node | undefined = decl, i = 0; n !== undefined && i < 4; n = n.parent, i++) { + if (blameNode(n)) marked = true; + if (ts.isStatement(n) || isClassMember(n) || ts.isSourceFile(n)) break; + } + // An unannotated function's result type comes from its returns. + if (!marked && depth < RETURN_DEPTH && ts.isFunctionLike(decl)) { + const body = (decl as { body?: ts.Node }).body; + if (body !== undefined) { + if (!ts.isBlock(body)) blameReferences(body, depth + 1); + else + visitReturns(body, (e) => { + blameReferences(e, depth + 1); + }); + } + } + }; + // Every declaration an expression's names resolve to. + const blameReferences = (root: ts.Node, depth: number): void => { + checker ??= program.getTypeChecker(); + const visit = (n: ts.Node): void => { + if (ts.isIdentifier(n) || ts.isPrivateIdentifier(n)) { + let symbol: ts.Symbol | undefined; + try { + symbol = checker!.getSymbolAtLocation(n); + } catch { + symbol = undefined; + } + for (const d of symbol?.declarations ?? []) blameDeclaration(d, depth); + } + ts.forEachChild(n, (c) => { + visit(c); + return undefined; + }); + }; + visit(root); + }; + + // Calls of each function declared in an overlay file, by declaration. + let calls: Map | null = null; + const callsOf = (fn: ts.Node): readonly ts.CallExpression[] => { + if (calls === null) { + calls = new Map(); + checker ??= program.getTypeChecker(); + for (const file of overlayFiles) { + const sf = program.getSourceFile(file); + if (sf === undefined) continue; + const visit = (n: ts.Node): void => { + if (ts.isCallExpression(n) && ts.isIdentifier(n.expression)) { + let decl: ts.Node | undefined; + try { + decl = checker!.getSymbolAtLocation(n.expression)?.declarations[0]; + } catch { + decl = undefined; + } + if (decl !== undefined) { + let list = calls!.get(decl); + if (list === undefined) calls!.set(decl, (list = [])); + list.push(n); + } + } + ts.forEachChild(n, (c) => { + visit(c); + return undefined; + }); + }; + visit(sf); + } + } + return calls.get(fn) ?? []; + }; + + for (const d of diagnostics) { + const sf = program.getSourceFile(d.loc.file); + if (sf === undefined) { + unattributed++; + continue; + } + const before = blamed; + if (overlayFiles.has(sf.fileName)) addFile(sf.fileName); + const node = innermostNodeAt(sf, d.loc.start, Math.max(d.loc.end, d.loc.start)); + // Enclosing annotated declarations, then the names of the refused + // expression, or failing that of the statement around it. + let statement: ts.Node = node; + for (let n: ts.Node | undefined = node; n !== undefined; n = n.parent) { + blameNode(n); + if (statement === node && (ts.isStatement(n) || isClassMember(n))) statement = n; + } + const own = blamed; + blameReferences(node, 0); + if (blamed === own && statement !== node) blameReferences(statement, 0); + // A function without parameter annotations is instantiated per call + // site from its arguments' types: blame what the arguments name. + if (blamed === own) { + const fn = enclosingFunction(node); + if (fn !== undefined) + for (const call of callsOf(fn)) + for (const arg of call.arguments.nodes) blameReferences(arg, 1); + } + // Still nothing: the lowering of the enclosing function changed as a + // whole (an instantiation the annotations inside it made possible). + if (blamed === before) { + const fn = enclosingFunctionLike(node); + if (fn !== undefined && overlayFiles.has(fn.getSourceFile().fileName)) { + const visit = (n: ts.Node): void => { + blameNode(n); + ts.forEachChild(n, (c) => { + visit(c); + return undefined; + }); + }; + visit(fn); + } + } + if (blamed === before) unattributed++; + } + return { annotations, files, unattributed }; +} + +function enclosingFunction(node: ts.Node): ts.Node | undefined { + for (let n: ts.Node | undefined = node.parent; n !== undefined; n = n.parent) + if (ts.isFunctionDeclaration(n)) return n; + else if (ts.isFunctionLike(n)) return undefined; + return undefined; +} + +function enclosingFunctionLike(node: ts.Node): ts.Node | undefined { + for (let n: ts.Node | undefined = node; n !== undefined; n = n.parent) + if (ts.isFunctionLike(n)) return n; + return undefined; +} + +function visitReturns(body: ts.Node, f: (e: ts.Expression) => void): void { + const visit = (n: ts.Node): void => { + if (ts.isFunctionLike(n) || ts.isClassLikeDeclaration(n)) return; + if (ts.isReturnStatement(n) && n.expression !== undefined) f(n.expression); + ts.forEachChild(n, (c) => { + visit(c); + return undefined; + }); + }; + ts.forEachChild(body, (c) => { + visit(c); + return undefined; + }); +} + +/** The innermost node whose token span covers [start, end). */ +function innermostNodeAt(sf: ts.SourceFile, start: number, end: number): ts.Node { + let node: ts.Node = sf; + for (;;) { + let next: ts.Node | undefined; + ts.forEachChild(node, (c) => { + if (next === undefined && ts.tokenStartOfNode(c) <= start && c.end >= end) next = c; + return undefined; + }); + if (next === undefined) return node; + node = next; + } +} + +function isClassMember(n: ts.Node): boolean { + return n.parent !== undefined && ts.isClassLikeDeclaration(n.parent) && n !== n.parent.name; +} diff --git a/packages/compiler/src/frontend/shape-inference/index.ts b/packages/compiler/src/frontend/shape-inference/index.ts index 233e62bf9..1db90850e 100644 --- a/packages/compiler/src/frontend/shape-inference/index.ts +++ b/packages/compiler/src/frontend/shape-inference/index.ts @@ -7,7 +7,8 @@ import type { ScrDiagnostic } from "../../diagnostics/diagnostic.js"; import { npmStaticPackageOfPath } from "../npm-static.js"; import { isShapelessPackage } from "../package-shims.js"; import { analyzeShapes, isPackageJsFile } from "./analyze.js"; -import { OVERLAY_MARK, planShapeOverlay } from "./overlay.js"; +import { OVERLAY_MARK, planShapeOverlay, type OverlayExclusions } from "./overlay.js"; +import { attributeOverlayRefusals } from "./fallback.js"; import { planStaticObjectRewrites } from "./static-objects.js"; import { isJsSourceFile } from "../program.js"; import { buildShapeReport } from "./report.js"; @@ -58,6 +59,7 @@ export function shapeOverlayAnnotationCount(): number { } export function resetShapeOverlay(): void { + staticRewriteFiles = new Set(); staticObjectStats = { classes: 0, functions: 0, directCalls: 0 }; overlayDone = false; overlayFiles = new Set(); @@ -73,12 +75,90 @@ export function shapeOverlayEnabled(): boolean { return process.env["SCRIPTC_SHAPE_OVERLAY"] === "1"; } +/* The build's fallback state (fallback.ts): annotations and files whose + * lowering was refused, and whether the build dropped the overlay. It + * outlives the loads of one build. */ +let exclusions: { annotations: Map>; files: Set } = { + annotations: new Map(), + files: new Set(), +}; +let overlayDropped = false; +let droppedAnnotations = 0; + +/** Starts a build: no annotation is excluded yet. */ +export function beginShapeOverlayBuild(): void { + exclusions = { annotations: new Map(), files: new Set() }; + overlayDropped = false; + droppedAnnotations = 0; +} + +/** Whether the current load has overlay files or structural rewrites. */ +export function shapeOverlayActive(): boolean { + const { classes, functions } = staticObjectStats; + return overlayFiles.size > 0 || classes + functions > 0; +} + +/** The overlay's answer to refused lowering diagnostics: it excludes what + * the refusals depend on, at the given level, and says whether a reload + * can change anything. + * - "annotations": the blamed annotations (or, for a blamed file without + * blamed annotations, that file); + * - "fences": like "annotations", for refusals deferred to run time, and + * never a whole file; + * - "files": every overlay file involved; + * - "all": the whole overlay. */ +export function excludeRefusedOverlay( + program: ts.Program, + diagnostics: readonly ScrDiagnostic[], + level: "annotations" | "files" | "all" | "fences", +): boolean { + if (overlayDropped) return false; + const files = new Set(overlayFiles); + // Files with structural rewrites only also count as overlay files. + for (const f of staticRewriteFiles) files.add(f); + if (level === "all") { + overlayDropped = true; + droppedAnnotations = overlayAnnotations; + return true; + } + const blame = attributeOverlayRefusals(program, diagnostics, files); + let changed = false; + for (const file of blame.files) { + const ids = blame.annotations.get(file); + // Deferred refusals drop only the annotations they are blamed on. + if (level === "fences" && ids === undefined) continue; + if (level === "files" || ids === undefined) { + if (!exclusions.files.has(file)) { + exclusions.files.add(file); + changed = true; + } + continue; + } + let set = exclusions.annotations.get(file); + if (set === undefined) exclusions.annotations.set(file, (set = new Set())); + for (const id of ids) { + if (set.has(id)) continue; + set.add(id); + droppedAnnotations++; + changed = true; + } + } + return changed; +} + +let staticRewriteFiles: ReadonlySet = new Set(); + +function overlayExclusions(): OverlayExclusions { + return exclusions; +} + /** loadProgram7's overlayer: analyze, plan, and register the overlay files. */ export function shapeOverlayer( program: ts.Program, round: number, ): ReadonlyMap | null { if (round === 0) resetShapeOverlay(); + if (overlayDropped) return null; const analysis = analyzeShapes(program, program.getTypeChecker()); if (analysis.budgetExceeded) return null; // Package JavaScript of packages that keep shapes on, and the program's @@ -92,7 +172,11 @@ export function shapeOverlayer( // Round 0: the structural rewrites (static-objects.ts); the program is // rebuilt from them and analyzed again before the typed overlay. if (round === 0 && process.env["SCRIPTC_SHAPE_STATIC_OBJECTS"] !== "0") { - const rewrites = planStaticObjectRewrites(analysis, jsFiles); + const rewrites = planStaticObjectRewrites( + analysis, + jsFiles.filter((sf) => !exclusions.files.has(sf.fileName)), + ); + staticRewriteFiles = new Set(rewrites.files.keys()); staticObjectStats = { classes: rewrites.classes, functions: rewrites.functions, @@ -107,7 +191,7 @@ export function shapeOverlayer( } if (round > 1) return null; const report = buildShapeReport(analysis, "1"); - const overlay = planShapeOverlay(analysis, report.sites, jsFiles); + const overlay = planShapeOverlay(analysis, report.sites, jsFiles, overlayExclusions()); overlayFiles = new Set(overlay.files.keys()); overlayAnnotations = overlay.annotations; overlayDeadBranches = overlay.deadBranches; @@ -181,6 +265,26 @@ export function shapeInferredFactCheckCount(): number { * checks of inferred facts the build inserted. A note, never a failure. */ export function shapeOverlayReport(entryFile: string): ScrDiagnostic[] { const { classes, functions, directCalls } = staticObjectStats; + // Annotations whose lowering was refused (fallback.ts); they compile + // untyped. + const fallback = overlayDropped + ? "the lowering refused code the overlay typed, so the build compiles without it" + : droppedAnnotations + exclusions.files.size > 0 + ? `it dropped ${droppedAnnotations} annotation${droppedAnnotations === 1 ? "" : "s"}` + + (exclusions.files.size > 0 + ? ` and the overlay of ${exclusions.files.size} file${exclusions.files.size === 1 ? "" : "s"}` + : "") + + " whose lowering was refused" + : null; + if (overlayDropped) + return [ + { + code: "SC6103", + message: `shape inference: ${fallback}`, + loc: { file: entryFile, start: 0, end: 0 }, + hint: "--no-shape-overlay compiles without the overlay", + }, + ]; if (overlayFiles.size === 0 && classes + functions === 0) return []; const packages = new Set( [...overlayFiles].map((f) => npmStaticPackageOfPath(f) ?? "the program"), @@ -197,7 +301,8 @@ export function shapeOverlayReport(entryFile: string): ScrDiagnostic[] { : "") + (classes + functions > 0 ? `; it compiled ${classes} constructor function${classes === 1 ? "" : "s"} as classes and ${functions} namespace function${functions === 1 ? "" : "s"} as direct functions (${directCalls} direct call${directCalls === 1 ? "" : "s"})` - : ""), + : "") + + (fallback !== null ? `; ${fallback}` : ""), loc: { file: entryFile, start: 0, end: 0 }, hint: "a failing check reports a scriptc analysis bug; --no-shape-overlay compiles without the overlay", }, diff --git a/packages/compiler/src/frontend/shape-inference/overlay.ts b/packages/compiler/src/frontend/shape-inference/overlay.ts index a0ae9ec2a..039f3e46b 100644 --- a/packages/compiler/src/frontend/shape-inference/overlay.ts +++ b/packages/compiler/src/frontend/shape-inference/overlay.ts @@ -54,10 +54,28 @@ function list(arr: { readonly nodes: readonly T[] } | undefin let debugIndex = 0; +/** Annotations a build dropped because their lowering was refused: per + * file, the annotation ids (positions in the overlay's input text); a file + * in `files` gets no overlay at all. */ +export interface OverlayExclusions { + readonly annotations: ReadonlyMap>; + readonly files: ReadonlySet; +} + +export const NO_EXCLUSIONS: OverlayExclusions = { annotations: new Map(), files: new Set() }; + +/** The ids of the overlay annotations in a comment or trivia text. */ +export function overlayAnnotationIds(text: string): number[] { + const ids: number[] = []; + for (const m of text.matchAll(/@scriptcShape #(\d+)/g)) ids.push(Number(m[1])); + return ids; +} + export function planShapeOverlay( analysis: ShapeAnalysis, verdicts: readonly SiteVerdict[], packageFiles: readonly ts.SourceFile[], + exclusions: OverlayExclusions = NO_EXCLUSIONS, ): ShapeOverlay { const pSites = new Map(); const siteVerdicts = new Set(verdicts.map((v) => v.site.atom)); @@ -96,9 +114,20 @@ export function planShapeOverlay( } } for (const sf of packageFiles) { + if (exclusions.files.has(sf.fileName)) continue; const typedefs = new Map(); const typedefText: string[] = []; const edits: Edit[] = []; + const dropped = exclusions.annotations.get(sf.fileName); + // One annotation: a JSDoc comment at `pos` carrying MARK and the + // annotation's id (its position), so a refused lowering can name it and + // the next load drop it. `close`: the text that closes a type cast. + const annotate = (pos: number, tags: string, close?: number): void => { + if (dropped?.has(pos) === true) return; + edits.push({ pos, text: `/** ${tags} ${MARK} #${pos} */ ${close !== undefined ? "(" : ""}` }); + if (close !== undefined) edits.push({ pos: close, text: ")" }); + annotations++; + }; const className = (decl: ts.Node): string | null => { const own = (decl as ts.ClassLikeDeclaration).name; @@ -111,7 +140,8 @@ export function planShapeOverlay( own === undefined || !ts.isClassDeclaration(decl) || decl.parent !== other || - !overlaid.has(other) + !overlaid.has(other) || + exclusions.files.has(other.fileName) ) return null; const alias = `__shape_class_${own.text}`; @@ -172,6 +202,15 @@ export function planShapeOverlay( objects++; if (analysis.escaped.has(a) || (siteVerdicts.has(a) && !pSites.has(a))) return inner ? "any" : null; + // Only arrays the program writes as literals: a builtin's array + // (a match result, `new Array(n)`) may have holes, and the + // checker already types it from the library. + if ( + a.node === undefined || + !ts.isArrayLiteralExpression(a.node) || + a.node.elements.nodes.some((e) => e.kind === ts.SyntaxKind.OmittedExpression) + ) + return inner ? "any" : null; const el = analysis.elements.get(a); if (el !== undefined) for (const x of analysis.nodes[el]!.atoms) elements.add(x); hasArray = true; @@ -338,9 +377,7 @@ export function planShapeOverlay( !declaresMember(cls, node.expression.name.text) ) { const self = node.expression.expression; - edits.push({ pos: ts.tokenStartOfNode(self), text: `/** @type {${t}} ${MARK} */ (` }); - edits.push({ pos: self.end, text: ")" }); - annotations++; + annotate(ts.tokenStartOfNode(self), `@type {${t}}`, self.end); } ts.forEachChild(node, (c) => { scan(c); @@ -433,7 +470,7 @@ export function planShapeOverlay( const t = typeOf(paramValues(fn, index), false, 0); if (t !== null) tags.push(`@param {${t}} ${p.name.text}`); }); - return tags.length > 0 ? `${tags.join(" ")} ${MARK}` : null; + return tags.length > 0 ? tags.join(" ") : null; }; const visit = (node: ts.Node): void => { @@ -451,10 +488,7 @@ export function planShapeOverlay( ) { if (node.body !== undefined) { const tags = paramTags(node); - if (tags !== null) { - edits.push({ pos: ts.tokenStartOfNode(node), text: `/** ${tags} */ ` }); - annotations++; - } + if (tags !== null) annotate(ts.tokenStartOfNode(node), tags); } // `this.f = …` fields of a constructor's top-level statements. if (ts.isConstructorDeclaration(node) && node.body !== undefined) { @@ -477,10 +511,7 @@ export function planShapeOverlay( if (seen.has(name)) continue; seen.add(name); const t = cls !== undefined ? fieldType(cls, name) : null; - if (t !== null) { - edits.push({ pos: ts.tokenStartOfNode(st), text: `/** @type {${t}} ${MARK} */ ` }); - annotations++; - } + if (t !== null) annotate(ts.tokenStartOfNode(st), `@type {${t}}`); } } } @@ -496,10 +527,7 @@ export function planShapeOverlay( const init = d.initializer; if (ts.isIdentifier(d.name) && init !== undefined && !literalInitializer(init)) { const t = typeOf(analysis.declValues(d), false, 0); - if (t !== null) { - edits.push({ pos: ts.tokenStartOfNode(node), text: `/** @type {${t}} ${MARK} */ ` }); - annotations++; - } + if (t !== null) annotate(ts.tokenStartOfNode(node), `@type {${t}}`); } } ts.forEachChild(node, (c) => { @@ -530,6 +558,7 @@ export function planShapeOverlay( const hoisted = new Map(); for (const branch of dead) { const start = ts.tokenStartOfNode(branch); + if (dropped?.has(start) === true) continue; if (ranges.some(([lo, hi]) => start >= lo && start < hi)) continue; const names = hoistedNames(branch); if (names === null) continue; @@ -554,7 +583,7 @@ export function planShapeOverlay( edits.push({ pos: lo, end: hi, - text: `{ ${vars}throw new Error(${JSON.stringify(message)}); }`, + text: `{ /** ${MARK} #${lo} */ ${vars}throw new Error(${JSON.stringify(message)}); }`, }); deadBranches++; } diff --git a/packages/compiler/src/frontend/shape-inference/registry.test.ts b/packages/compiler/src/frontend/shape-inference/registry.test.ts new file mode 100644 index 000000000..31c07daea --- /dev/null +++ b/packages/compiler/src/frontend/shape-inference/registry.test.ts @@ -0,0 +1,34 @@ +/* loadProgram resets the shape overlay's registry for the lowering world's + * load only. The authored world that preflight can create afterwards + * (load.projectWorld()) is a second load of the same build: it leaves the + * lowering load's overlay registered, so the fallback (fallback.ts) still + * finds the overlay behind a refusal. */ +import { join } from "node:path"; +import { expect, test } from "vitest"; +import { loadProgram } from "../program-node.js"; +import { shapeOverlayActive, shapeOverlayFileNames } from "./index.js"; + +const entry = join( + import.meta.dirname, + "../../../../../tests/fixtures/npm/cases/static-objects/js-main.cjs", +); + +test("the authored world leaves the lowering load's overlay registered", () => { + const saved = process.env["SCRIPTC_SHAPE_OVERLAY"]; + process.env["SCRIPTC_SHAPE_OVERLAY"] = "1"; + try { + const load = loadProgram(entry, { npmStatic: ["static-ns"] }); + try { + expect(shapeOverlayActive()).toBe(true); + const files = shapeOverlayFileNames(); + load.projectWorld(); + expect(shapeOverlayActive()).toBe(true); + expect(shapeOverlayFileNames()).toEqual(files); + } finally { + load.dispose(); + } + } finally { + if (saved === undefined) delete process.env["SCRIPTC_SHAPE_OVERLAY"]; + else process.env["SCRIPTC_SHAPE_OVERLAY"] = saved; + } +}); diff --git a/packages/compiler/src/frontend/ts7/program-host.ts b/packages/compiler/src/frontend/ts7/program-host.ts index 11bb8ef3a..95d65f8f5 100644 --- a/packages/compiler/src/frontend/ts7/program-host.ts +++ b/packages/compiler/src/frontend/ts7/program-host.ts @@ -201,6 +201,11 @@ export class Ts7Host { }); } + /** The text of a registered in-memory file, or undefined. */ + virtualFileText(path: string): string | undefined { + return this.virtualFiles.get(tsgoPath(path)); + } + /** The text this host serves for a file (null when it serves none), * without asking the server for its syntax tree. */ sourceText(fileName: string): string | null { diff --git a/packages/compiler/src/index.ts b/packages/compiler/src/index.ts index 079db8494..ce11c0202 100644 --- a/packages/compiler/src/index.ts +++ b/packages/compiler/src/index.ts @@ -482,10 +482,11 @@ export async function compile( opts: CompileRequestOptions, ): Promise { clearCompileSessionCaches(); - // `shapeOverlay: false` reaches the frontend (and its child process) - // through the environment the overlay reads. + // `shapeOverlay` reaches the frontend (and its child process) through + // the environment the overlay reads. const overlayEnv = process.env["SCRIPTC_SHAPE_OVERLAY"]; - if (opts.shapeOverlay === false) process.env["SCRIPTC_SHAPE_OVERLAY"] = "0"; + if (opts.shapeOverlay !== undefined) + process.env["SCRIPTC_SHAPE_OVERLAY"] = opts.shapeOverlay ? "1" : "0"; const frontendInputs = new FrontendInputTracker(); const warnings: CompileWarningSink = { warnings: [], sourceTexts: null }; // compileTracked starts it for a build with no earlier build of the same invocation. @@ -498,7 +499,7 @@ export async function compile( } finally { setLibDiagnosticsStore(null); frontendProcess?.release(); - if (opts.shapeOverlay === false) { + if (opts.shapeOverlay !== undefined) { if (overlayEnv === undefined) delete process.env["SCRIPTC_SHAPE_OVERLAY"]; else process.env["SCRIPTC_SHAPE_OVERLAY"] = overlayEnv; } @@ -1313,7 +1314,8 @@ function configureLibDiagnostics(root: string | null): void { export async function runFrontendProcess(requestPath: string, resultPath: string): Promise { const request = JSON.parse(await readFile(requestPath, "utf8")) as FrontendProcessRequest; // The process started before the build's options were known. - if (request.opts.shapeOverlay === false) process.env["SCRIPTC_SHAPE_OVERLAY"] = "0"; + if (request.opts.shapeOverlay !== undefined) + process.env["SCRIPTC_SHAPE_OVERLAY"] = request.opts.shapeOverlay ? "1" : "0"; clearCompileSessionCaches(); setProvenanceSources(request.provenance); // The compile that started this process prepared the cache root. diff --git a/scripts/ci-test-plan.mjs b/scripts/ci-test-plan.mjs index 88a77e32e..da9c0a0cd 100644 --- a/scripts/ci-test-plan.mjs +++ b/scripts/ci-test-plan.mjs @@ -7,6 +7,7 @@ export const corpusFiles = [ "tests/harness/differential.test.ts", "tests/harness/llvm-differential.test.ts", "tests/harness/npm.test.ts", + "tests/harness/npm-shape-overlay.test.ts", "tests/harness/server.test.ts", "tests/harness/test262.test.ts", "tests/harness/three.test.ts", diff --git a/tests/harness/npm-shape-overlay.test.ts b/tests/harness/npm-shape-overlay.test.ts new file mode 100644 index 000000000..4db61fbe6 --- /dev/null +++ b/tests/harness/npm-shape-overlay.test.ts @@ -0,0 +1,7 @@ +/* Shape inference's overlay never breaks compilation: every npm case + * compiles with the overlay on and builds and runs exactly as it does in the + * default build (npm-suite.ts's KNOWN). An overlay annotation whose lowering + * is refused is dropped, so its declaration compiles untyped. */ +import { defineNpmSuite } from "./npm-suite.js"; + +defineNpmSuite({ label: "npm native, shape overlay", shapeOverlay: true }); diff --git a/tests/harness/npm-suite.ts b/tests/harness/npm-suite.ts new file mode 100644 index 000000000..6bfbe6465 --- /dev/null +++ b/tests/harness/npm-suite.ts @@ -0,0 +1,282 @@ +/* The npm suite (npm.test.ts runs it as built by default, + * npm-shape-overlay.test.ts with shape inference's overlay on). + * + * npm packages compiled natively: every program under tests/fixtures/npm/cases + * (plus the commander fixtures in npm-cases.ts) builds with the default + * static npm compile and runs against Node. A case matches Node (stdout and + * exit status for every argv list, stderr when it opts in with + * `// @stderr`) unless KNOWN records how it differs: + * + * - build: the compile refuses it with this diagnostic code; + * - run: it compiles, and the program's output names this diagnostic code + * (a refusal deferred to the operation that reached it), for every argv + * list or for the listed `argvs`, while the other lists match Node; + * - divergence: it runs but does not match Node yet. These run as expected + * failures, so a fix that makes one match fails here until KNOWN drops it. + * + * The fixture node_modules are committed test data. Binaries compile the + * package sources at build time and read nothing from node_modules at run + * time. + */ +import { execFile } from "node:child_process"; +import { createHash } from "node:crypto"; +import { globSync, mkdirSync, readFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { promisify } from "node:util"; +import { describe, expect, test } from "vitest"; +import { compile } from "@scriptc/compiler"; +import { npmCases, type NpmCase } from "./npm-cases.js"; +import { shardSelect, shardSuffix } from "./shard.js"; + +const execFileAsync = promisify(execFile); +const repoRoot = join(import.meta.dirname, "../.."); +const fixturesRoot = join(repoRoot, "tests/fixtures"); +const cacheDir = join(repoRoot, "node_modules/.cache/scriptc-tests"); +const sanitize = process.env["SCRIPTC_SAN"] === "1"; + +type Known = { build: string } | { run: string; argvs?: string[][] } | { divergence: string }; + +const KNOWN: Record = { + "any-chaining": { build: "SC2003" }, + "assert-shims": { run: "SC1043" }, + "buffer-shims": { run: "SC2020" }, + "bytes-exit": { build: "SC2004" }, + "cjs-esm-named": { run: "SC1090" }, + "cjs-lexer-aligned": { run: "SC1090" }, + "commander-calc": { run: "SC1090", argvs: [["--help"]] }, + "commander-calc-async": { run: "SC1090", argvs: [["--help"]] }, + "create-require": { run: "SC1090" }, + "dotfolder": { run: "SC1090" }, + "dual-package-kinds": { build: "SC2013" }, + "dynamic-import": { build: "SC2012" }, + "dynamic-import-local": { build: "SC2012" }, + "esbuild-require": { build: "SC2013" }, + "esm-missing-default": { build: "SC1090" }, + "esm-missing-export": { build: "SC1090" }, + "fetch-destructure": { run: "SC1031" }, + "fs-shims": { run: "SC1101" }, + "import-refusal": { build: "SC2020" }, + "island-registry": { build: "SC2011" }, + "island-timers": { + divergence: + "Timeout handles are numbers (typeof) and lose their ref flag once fired or cleared", + }, + "island-web-plumbing": { run: "SC2020" }, + "lazy-traps": { build: "SC2013" }, + "misc-shims": { run: "SC2020" }, + "namespace": { build: "SC1013" }, + "node24-async-events": { run: "SC2020" }, + "node24-commonjs-metadata": { run: "SC2020" }, + "node24-crypto-kdf": { run: "SC2020" }, + "node24-deep-equality": { run: "SC2020" }, + "node24-diagnostics-async": { run: "SC1090" }, + "node24-diagnostics-tracing": { run: "SC1090" }, + "node24-env": { run: "SC1090" }, + "node24-esm-metadata": { run: "SC1090" }, + "node24-event-emitter": { run: "SC2020" }, + "node24-file-url-conversion": { build: "SC1090" }, + "node24-fs-descriptors": { run: "SC2020" }, + "node24-mime": { run: "SC2020" }, + "node24-mime-errors": { run: "SC2020" }, + "node24-no-error-assertions": { run: "SC2020" }, + "node24-parseargs": { run: "SC1090" }, + "node24-promise-timers": { run: "SC2020" }, + "node24-string-decoder": { run: "SC2020" }, + "node24-style-text": { run: "SC2020" }, + "node24-system-errors": { run: "SC1090" }, + "node24-text-encoding": { run: "SC2020" }, + "node24-text-utilities": { run: "SC2020" }, + "node24-timers-unref": { run: "SC2020" }, + "node24-url-factories": { run: "SC2020" }, + "node24-url-mutation": { run: "SC1090" }, + "node24-util-adapters": { run: "Object.create with this value is not supported yet" }, + "node24-util-types": { run: "SC1090" }, + "node24-web-stream-encoding": { run: "SC1090" }, + "node24-web-stream-errors": { run: "SC1090" }, + "node24-web-stream-lifecycle": { run: "SC1090" }, + "node24-web-stream-pipe-abort": { run: "SC2020" }, + "node24-web-stream-pipe-errors": { run: "SC1090" }, + "node24-web-stream-pipes": { run: "SC1090" }, + "node24-web-stream-queues": { run: "SC1090" }, + "node24-web-stream-readable-lifecycle": { run: "SC2020" }, + "node24-web-stream-readers": { run: "SC1090" }, + "node24-web-stream-tee": { run: "SC2020" }, + "node24-web-stream-transforms": { run: "SC1090" }, + "package-imports": { build: "SC2013" }, + "promise-bridge": { build: "SC1090" }, + "self-name": { build: "SC2013" }, + "stream-shims": { run: "SC2020" }, + "util-shims": { run: "SC2020" }, + // The package redeclares the var it exported: Node copies the first value. + "var-export-redeclared": { run: "SC1090" }, + "web-streams": { run: "SC1090" }, + "workspace-copied": { build: "SC2013" }, + "workspace-unclean": { build: "SC2013" }, +}; + +/** Per-case timeouts past the default 120 s: the undici cases compile the + * built-in undici implementation's module graph, the zod 3 cases (zod3, + * zod3-adoption, zod3-boundary, zod3-infer) compile zod 3's 3,700-line + * types module, and the zod 4 cases (zod4-boundary, zod4-record, + * hono-zod-validator, hono-zod-default-hook) + * compile zod 4's core, classic API and locales. A sanitized build of any + * of them takes minutes on a CI runner. */ +const TIMEOUTS: Record = { + "hono-zod-default-hook": 900_000, + "hono-zod-validator": 900_000, + undici5: 600_000, + undici7: 600_000, + "zod4-boundary": 900_000, + "zod4-record": 900_000, + zod3: 600_000, + "zod3-adoption": 600_000, + "zod3-boundary": 600_000, + "zod3-infer": 600_000, +}; + +interface RunResult { + stdout: Buffer; + stderr: Buffer; + exitCode: number; +} + +function programStderr(stderr: Buffer): Buffer { + if (!sanitize) return stderr; + // Linux ASan emits this instrumentation warning at the first fiber swap. + // Latin-1 preserves every other byte, including non-UTF-8 program output. + return Buffer.from( + stderr + .toString("latin1") + .replace( + /^==\d+==WARNING: ASan doesn't fully support makecontext\/swapcontext functions and may produce false positives in some cases!\n/gm, + "", + ), + "latin1", + ); +} + +async function runBinary(cmd: string, args: string[], cwd: string): Promise { + try { + // stdin closes immediately, as in differential.test.ts: a program that + // reads fd 0 to EOF must not hang on an open pipe. + const pending = execFileAsync(cmd, args, { cwd, encoding: "buffer" }); + pending.child.stdin?.end(); + const { stdout, stderr } = await pending; + return { stdout, stderr, exitCode: 0 }; + } catch (err) { + const e = err as { code?: unknown; signal?: unknown; stdout?: Buffer; stderr?: Buffer }; + if (!Buffer.isBuffer(e.stdout) || !Buffer.isBuffer(e.stderr)) throw err; + const exitCode = typeof e.code === "number" ? e.code : -1; + if (exitCode === -1 && typeof e.signal !== "string") throw err; + return { stdout: e.stdout, stderr: e.stderr, exitCode }; + } +} + +type Built = { ok: true; binary: string } | { ok: false; codes: string[]; text: string }; + +/** Compile once per (entry, lane); the cache key hashes the program and the + * fixture packages so edits to either rebuild. */ +async function build(entry: string, shapeOverlay: boolean | undefined): Promise { + const hash = createHash("sha256"); + const inputs = [ + entry, + ...globSync(join(dirname(entry), "*.{ts,mts,cts,js,mjs,cjs,json}")), + ...globSync(join(fixturesRoot, "npm/node_modules/**/*.{js,mjs,cjs,json,d.ts,node}")), + ].sort(); + for (const f of inputs) hash.update(f).update(readFileSync(f)); + const key = hash + .update(sanitize ? "san" : "plain") + .update(shapeOverlay === undefined ? "" : `overlay=${shapeOverlay}`) + .digest("hex") + .slice(0, 16); + const outDir = join(cacheDir, `npm-${key}`); + mkdirSync(outDir, { recursive: true }); + const result = await compile(entry, { + outPath: join(outDir, "program"), + outDir, + sanitize, + ...(shapeOverlay === undefined ? {} : { shapeOverlay }), + }); + if (result.ok) return { ok: true, binary: result.binaryPath }; + return { + ok: false, + codes: result.diagnostics.map((d) => d.code), + text: result.diagnostics.map((d) => `${d.code}: ${d.message}`).join("\n"), + }; +} + +async function expectMatchesNode(c: NpmCase, binary: string): Promise { + const cwd = dirname(c.entry); + for (const argv of c.argvs ?? [[]]) { + const [nodeRes, nativeRes] = await Promise.all([ + runBinary(process.execPath, [c.entry, ...argv], cwd), + runBinary(binary, argv, cwd), + ]); + const label = argv.join(" "); + if (!nodeRes.stdout.equals(nativeRes.stdout)) { + expect(nativeRes.stdout.toString("utf8"), label).toBe(nodeRes.stdout.toString("utf8")); + expect.unreachable("stdout differed at byte level but not after utf8 decode"); + } + expect(nativeRes.exitCode, label).toBe(nodeRes.exitCode); + if (c.compareStderr) expect(programStderr(nativeRes.stderr), label).toEqual(nodeRes.stderr); + } +} + +/** Defines the suite. `shapeOverlay` is passed to every compile (unset: the + * default build). With the overlay on, every case must still build and run + * as KNOWN records for the default build, except that a refused build may + * compile, and then matches Node. */ +export function defineNpmSuite(options: { label: string; shapeOverlay?: boolean }): void { + const { label, shapeOverlay } = options; + const cases = shardSelect(npmCases(fixturesRoot), (c) => c.name); + test("every KNOWN entry names a case", () => { + const names = new Set(npmCases(fixturesRoot).map((c) => c.name)); + expect(Object.keys(KNOWN).filter((name) => !names.has(name))).toEqual([]); + }); + + describe(`${label} (${cases.length} programs${sanitize ? ", sanitized" : ""}${shardSuffix()})`, () => { + for (const c of cases) { + const known = KNOWN[c.name]; + if (known !== undefined && "divergence" in known) { + test.fails(`${c.name} (diverges: ${known.divergence})`, async () => { + const built = await build(c.entry, shapeOverlay); + if (!built.ok) throw new Error(built.text); + await expectMatchesNode(c, built.binary); + }, 120_000); + continue; + } + test( + c.name, + async () => { + const built = await build(c.entry, shapeOverlay); + if (known !== undefined && "build" in known) { + // The overlay may type code the default build refuses: such a + // program must then match Node. + if (shapeOverlay === true && built.ok) { + await expectMatchesNode(c, built.binary); + return; + } + expect(built.ok, "the compile refuses the program").toBe(false); + if (!built.ok) expect(built.codes, built.text).toContain(known.build); + return; + } + if (!built.ok) throw new Error(`npm fixture failed to compile:\n${built.text}`); + if (known !== undefined && "run" in known) { + for (const argv of known.argvs ?? c.argvs ?? [[]]) { + const res = await runBinary(built.binary, argv, dirname(c.entry)); + const output = res.stdout.toString("utf8") + res.stderr.toString("utf8"); + expect(output, `${argv.join(" ")}: the deferred refusal`).toContain(known.run); + } + if (known.argvs === undefined) return; + const refused = new Set(known.argvs.map((argv) => JSON.stringify(argv))); + const argvs = (c.argvs ?? [[]]).filter((argv) => !refused.has(JSON.stringify(argv))); + await expectMatchesNode({ ...c, argvs }, built.binary); + return; + } + await expectMatchesNode(c, built.binary); + }, + TIMEOUTS[c.name] ?? 120_000, + ); + } + }); +} diff --git a/tests/harness/npm.test.ts b/tests/harness/npm.test.ts index 8e133f6e2..dc64d8b86 100644 --- a/tests/harness/npm.test.ts +++ b/tests/harness/npm.test.ts @@ -1,261 +1,4 @@ -/* npm packages compiled natively: every program under tests/fixtures/npm/cases - * (plus the commander fixtures in npm-cases.ts) builds with the default - * static npm compile and runs against Node. A case matches Node (stdout and - * exit status for every argv list, stderr when it opts in with - * `// @stderr`) unless KNOWN records how it differs: - * - * - build: the compile refuses it with this diagnostic code; - * - run: it compiles, and the program's output names this diagnostic code - * (a refusal deferred to the operation that reached it), for every argv - * list or for the listed `argvs`, while the other lists match Node; - * - divergence: it runs but does not match Node yet. These run as expected - * failures, so a fix that makes one match fails here until KNOWN drops it. - * - * The fixture node_modules are committed test data. Binaries compile the - * package sources at build time and read nothing from node_modules at run - * time. - */ -import { execFile } from "node:child_process"; -import { createHash } from "node:crypto"; -import { globSync, mkdirSync, readFileSync } from "node:fs"; -import { dirname, join } from "node:path"; -import { promisify } from "node:util"; -import { describe, expect, test } from "vitest"; -import { compile } from "@scriptc/compiler"; -import { npmCases, type NpmCase } from "./npm-cases.js"; -import { shardSelect, shardSuffix } from "./shard.js"; +/* npm packages compiled natively as built by default (npm-suite.ts). */ +import { defineNpmSuite } from "./npm-suite.js"; -const execFileAsync = promisify(execFile); -const repoRoot = join(import.meta.dirname, "../.."); -const fixturesRoot = join(repoRoot, "tests/fixtures"); -const cacheDir = join(repoRoot, "node_modules/.cache/scriptc-tests"); -const sanitize = process.env["SCRIPTC_SAN"] === "1"; - -type Known = { build: string } | { run: string; argvs?: string[][] } | { divergence: string }; - -const KNOWN: Record = { - "any-chaining": { build: "SC2003" }, - "assert-shims": { run: "SC1043" }, - "buffer-shims": { run: "SC2020" }, - "bytes-exit": { build: "SC2004" }, - "cjs-esm-named": { run: "SC1090" }, - "cjs-lexer-aligned": { run: "SC1090" }, - "commander-calc": { run: "SC1090", argvs: [["--help"]] }, - "commander-calc-async": { run: "SC1090", argvs: [["--help"]] }, - "create-require": { run: "SC1090" }, - "dotfolder": { run: "SC1090" }, - "dual-package-kinds": { build: "SC2013" }, - "dynamic-import": { build: "SC2012" }, - "dynamic-import-local": { build: "SC2012" }, - "esbuild-require": { build: "SC2013" }, - "esm-missing-default": { build: "SC1090" }, - "esm-missing-export": { build: "SC1090" }, - "fetch-destructure": { run: "SC1031" }, - "fs-shims": { run: "SC1101" }, - "import-refusal": { build: "SC2020" }, - "island-registry": { build: "SC2011" }, - "island-timers": { - divergence: - "Timeout handles are numbers (typeof) and lose their ref flag once fired or cleared", - }, - "island-web-plumbing": { run: "SC2020" }, - "lazy-traps": { build: "SC2013" }, - "misc-shims": { run: "SC2020" }, - "namespace": { build: "SC1013" }, - "node24-async-events": { run: "SC2020" }, - "node24-commonjs-metadata": { run: "SC2020" }, - "node24-crypto-kdf": { run: "SC2020" }, - "node24-deep-equality": { run: "SC2020" }, - "node24-diagnostics-async": { run: "SC1090" }, - "node24-diagnostics-tracing": { run: "SC1090" }, - "node24-env": { run: "SC1090" }, - "node24-esm-metadata": { run: "SC1090" }, - "node24-event-emitter": { run: "SC2020" }, - "node24-file-url-conversion": { build: "SC1090" }, - "node24-fs-descriptors": { run: "SC2020" }, - "node24-mime": { run: "SC2020" }, - "node24-mime-errors": { run: "SC2020" }, - "node24-no-error-assertions": { run: "SC2020" }, - "node24-parseargs": { run: "SC1090" }, - "node24-promise-timers": { run: "SC2020" }, - "node24-string-decoder": { run: "SC2020" }, - "node24-style-text": { run: "SC2020" }, - "node24-system-errors": { run: "SC1090" }, - "node24-text-encoding": { run: "SC2020" }, - "node24-text-utilities": { run: "SC2020" }, - "node24-timers-unref": { run: "SC2020" }, - "node24-url-factories": { run: "SC2020" }, - "node24-url-mutation": { run: "SC1090" }, - "node24-util-adapters": { run: "Object.create with this value is not supported yet" }, - "node24-util-types": { run: "SC1090" }, - "node24-web-stream-encoding": { run: "SC1090" }, - "node24-web-stream-errors": { run: "SC1090" }, - "node24-web-stream-lifecycle": { run: "SC1090" }, - "node24-web-stream-pipe-abort": { run: "SC2020" }, - "node24-web-stream-pipe-errors": { run: "SC1090" }, - "node24-web-stream-pipes": { run: "SC1090" }, - "node24-web-stream-queues": { run: "SC1090" }, - "node24-web-stream-readable-lifecycle": { run: "SC2020" }, - "node24-web-stream-readers": { run: "SC1090" }, - "node24-web-stream-tee": { run: "SC2020" }, - "node24-web-stream-transforms": { run: "SC1090" }, - "package-imports": { build: "SC2013" }, - "promise-bridge": { build: "SC1090" }, - "self-name": { build: "SC2013" }, - "stream-shims": { run: "SC2020" }, - "util-shims": { run: "SC2020" }, - // The package redeclares the var it exported: Node copies the first value. - "var-export-redeclared": { run: "SC1090" }, - "web-streams": { run: "SC1090" }, - "workspace-copied": { build: "SC2013" }, - "workspace-unclean": { build: "SC2013" }, -}; - -const cases = shardSelect(npmCases(fixturesRoot), (c) => c.name); - -/** Per-case timeouts past the default 120 s: the undici cases compile the - * built-in undici implementation's module graph, the zod 3 cases (zod3, - * zod3-adoption, zod3-boundary, zod3-infer) compile zod 3's 3,700-line - * types module, and the zod 4 cases (zod4-boundary, zod4-record, - * hono-zod-validator, hono-zod-default-hook) - * compile zod 4's core, classic API and locales. A sanitized build of any - * of them takes minutes on a CI runner. */ -const TIMEOUTS: Record = { - "hono-zod-default-hook": 900_000, - "hono-zod-validator": 900_000, - undici5: 600_000, - undici7: 600_000, - "zod4-boundary": 900_000, - "zod4-record": 900_000, - zod3: 600_000, - "zod3-adoption": 600_000, - "zod3-boundary": 600_000, - "zod3-infer": 600_000, -}; - -interface RunResult { - stdout: Buffer; - stderr: Buffer; - exitCode: number; -} - -function programStderr(stderr: Buffer): Buffer { - if (!sanitize) return stderr; - // Linux ASan emits this instrumentation warning at the first fiber swap. - // Latin-1 preserves every other byte, including non-UTF-8 program output. - return Buffer.from( - stderr - .toString("latin1") - .replace( - /^==\d+==WARNING: ASan doesn't fully support makecontext\/swapcontext functions and may produce false positives in some cases!\n/gm, - "", - ), - "latin1", - ); -} - -async function runBinary(cmd: string, args: string[], cwd: string): Promise { - try { - // stdin closes immediately, as in differential.test.ts: a program that - // reads fd 0 to EOF must not hang on an open pipe. - const pending = execFileAsync(cmd, args, { cwd, encoding: "buffer" }); - pending.child.stdin?.end(); - const { stdout, stderr } = await pending; - return { stdout, stderr, exitCode: 0 }; - } catch (err) { - const e = err as { code?: unknown; signal?: unknown; stdout?: Buffer; stderr?: Buffer }; - if (!Buffer.isBuffer(e.stdout) || !Buffer.isBuffer(e.stderr)) throw err; - const exitCode = typeof e.code === "number" ? e.code : -1; - if (exitCode === -1 && typeof e.signal !== "string") throw err; - return { stdout: e.stdout, stderr: e.stderr, exitCode }; - } -} - -type Built = { ok: true; binary: string } | { ok: false; codes: string[]; text: string }; - -/** Compile once per (entry, lane); the cache key hashes the program and the - * fixture packages so edits to either rebuild. */ -async function build(entry: string): Promise { - const hash = createHash("sha256"); - const inputs = [ - entry, - ...globSync(join(dirname(entry), "*.{ts,mts,cts,js,mjs,cjs,json}")), - ...globSync(join(fixturesRoot, "npm/node_modules/**/*.{js,mjs,cjs,json,d.ts,node}")), - ].sort(); - for (const f of inputs) hash.update(f).update(readFileSync(f)); - const key = hash - .update(sanitize ? "san" : "plain") - .digest("hex") - .slice(0, 16); - const outDir = join(cacheDir, `npm-${key}`); - mkdirSync(outDir, { recursive: true }); - const result = await compile(entry, { outPath: join(outDir, "program"), outDir, sanitize }); - if (result.ok) return { ok: true, binary: result.binaryPath }; - return { - ok: false, - codes: result.diagnostics.map((d) => d.code), - text: result.diagnostics.map((d) => `${d.code}: ${d.message}`).join("\n"), - }; -} - -async function expectMatchesNode(c: NpmCase, binary: string): Promise { - const cwd = dirname(c.entry); - for (const argv of c.argvs ?? [[]]) { - const [nodeRes, nativeRes] = await Promise.all([ - runBinary(process.execPath, [c.entry, ...argv], cwd), - runBinary(binary, argv, cwd), - ]); - const label = argv.join(" "); - if (!nodeRes.stdout.equals(nativeRes.stdout)) { - expect(nativeRes.stdout.toString("utf8"), label).toBe(nodeRes.stdout.toString("utf8")); - expect.unreachable("stdout differed at byte level but not after utf8 decode"); - } - expect(nativeRes.exitCode, label).toBe(nodeRes.exitCode); - if (c.compareStderr) expect(programStderr(nativeRes.stderr), label).toEqual(nodeRes.stderr); - } -} - -test("every KNOWN entry names a case", () => { - const names = new Set(npmCases(fixturesRoot).map((c) => c.name)); - expect(Object.keys(KNOWN).filter((name) => !names.has(name))).toEqual([]); -}); - -describe(`npm native (${cases.length} programs${sanitize ? ", sanitized" : ""}${shardSuffix()})`, () => { - for (const c of cases) { - const known = KNOWN[c.name]; - if (known !== undefined && "divergence" in known) { - test.fails(`${c.name} (diverges: ${known.divergence})`, async () => { - const built = await build(c.entry); - if (!built.ok) throw new Error(built.text); - await expectMatchesNode(c, built.binary); - }, 120_000); - continue; - } - test( - c.name, - async () => { - const built = await build(c.entry); - if (known !== undefined && "build" in known) { - expect(built.ok, "the compile refuses the program").toBe(false); - if (!built.ok) expect(built.codes, built.text).toContain(known.build); - return; - } - if (!built.ok) throw new Error(`npm fixture failed to compile:\n${built.text}`); - if (known !== undefined && "run" in known) { - for (const argv of known.argvs ?? c.argvs ?? [[]]) { - const res = await runBinary(built.binary, argv, dirname(c.entry)); - const output = res.stdout.toString("utf8") + res.stderr.toString("utf8"); - expect(output, `${argv.join(" ")}: the deferred refusal`).toContain(known.run); - } - if (known.argvs === undefined) return; - const refused = new Set(known.argvs.map((argv) => JSON.stringify(argv))); - const argvs = (c.argvs ?? [[]]).filter((argv) => !refused.has(JSON.stringify(argv))); - await expectMatchesNode({ ...c, argvs }, built.binary); - return; - } - await expectMatchesNode(c, built.binary); - }, - TIMEOUTS[c.name] ?? 120_000, - ); - } -}); +defineNpmSuite({ label: "npm native" });