trending/scriptc
1
0
Fork 0
mirror of https://github.com/vercel-labs/scriptc.git synced 2026-11-03 07:54:40 +08:00
Code Issues Packages Projects Releases Wiki Activity

Never let the shape overlay break compilation (#2219)

Browse Source
This commit is contained in:
Chris Tate
2026-10-26 10:19:06 -05:00
committed by GitHub
parent a684543b58 7e6327740b
commit cd4950c25f
15 changed files with 911 additions and 354 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+2
View File
@@ -1,4 +1,6 @@
node_modules/
# A symlinked node_modules is a file to git, which node_modules/ does not match.
/packages/*/node_modules
.pnpm-store/
# test-fixture packages are TEST DATA, committed on purpose
!tests/diagnostics/npm-imports/node_modules/
packages/compiler/src/compile-types.ts
+2 -1
View File
@@ -68,7 +68,8 @@ export interface CompileBaseOptions {
/** Eligible packages compile automatically. Explicit names override automatic selection heuristics. */
npmStatic?: readonly string[] | "auto";
/** Shape inference's typed overlay for opted-in packages' JavaScript.
* False disables it (`--no-shape-overlay`); a package opts out with
* True enables it, false disables it (`--no-shape-overlay`); unset
* follows SCRIPTC_SHAPE_OVERLAY. A package opts out with
* `"scriptc": { "packages": { "<name>": { "shapes": false } } }`. */
shapeOverlay?: boolean;
/** Outbound native FFI manifest. Its signature-only TypeScript bindings
packages/compiler/src/frontend/pipeline.ts
+72 -13
View File
@@ -1,6 +1,11 @@
import { shapeOverlayReport } from "./shape-inference/index.js";
/** Shared frontend orchestration. Hosts supply a program loader so Node
* and native callers use the same npm selection, preflight and lowering. */
import {
beginShapeOverlayBuild,
excludeRefusedOverlay,
shapeOverlayActive,
shapeOverlayReport,
} from "./shape-inference/index.js";
import type { ScrDiagnostic } from "../diagnostics/diagnostic.js";
import type { SrcLoc } from "../ir/ir.js";
import type { NpmStaticStatus } from "../coverage/report.js";
@@ -496,6 +501,10 @@ export function runFrontend(
}
}
/** Rounds of dropping single refused overlay annotations before the
* fallback drops whole files' overlays. */
const OVERLAY_ANNOTATION_ROUNDS = 4;
function loadFrontend(
entryPath: string,
loadProgram: ProgramLoader,
@@ -510,6 +519,7 @@ function loadFrontend(
// separate compiles in a long-lived process. A cache miss must observe
// package.json edits before it can publish a new early-library entry.
clearResolveCaches();
beginShapeOverlayBuild();
const statuses: NpmStaticStatus[] = [];
const npmSites = new Map<string, SrcLoc>();
// Node island detection (report-only): how each package was refused, and
@@ -730,7 +740,9 @@ function loadFrontend(
}
}
const finalLoad = load;
// The lowering may reload once the overlay drops refused annotations.
let finalLoad = load;
const finalNpmStatic = [...effective];
// An unknown key or value in the project's scriptc configuration makes
// the Node island report unavailable (SC8004). The settings configure
// only that report: a build warns about them and goes on.
@@ -790,16 +802,63 @@ function loadFrontend(
return texts;
},
lower: (opts) => {
const lowered = lowerToIr(finalLoad.program, finalLoad.entry, finalLoad.moduleOrder, {
...opts,
frontendServices: finalLoad.services,
startupCrash: finalLoad.startupCrash ?? null,
inlineRequireCycleCalls: finalLoad.inlineRequireCycleCalls,
moduleEdges: finalLoad.moduleEdges,
projectCheckJs: finalLoad.typeWorld?.projectCheckJs ?? false,
externalTypes: finalLoad.externalTypes,
externalTypeSpecifiersByFile: finalLoad.externalTypeSpecifiersByFile,
});
const lowerOnce = (): LowerResult =>
lowerToIr(finalLoad.program, finalLoad.entry, finalLoad.moduleOrder, {
...opts,
frontendServices: finalLoad.services,
startupCrash: finalLoad.startupCrash ?? null,
inlineRequireCycleCalls: finalLoad.inlineRequireCycleCalls,
moduleEdges: finalLoad.moduleEdges,
projectCheckJs: finalLoad.typeWorld?.projectCheckJs ?? false,
externalTypes: finalLoad.externalTypes,
externalTypeSpecifiersByFile: finalLoad.externalTypeSpecifiersByFile,
});
let lowered = lowerOnce();
// Shape inference never breaks compilation (shape-inference/
// fallback.ts): refusals drop the overlay annotations they depend on
// and the program reloads. Rounds that stop converging escalate to
// the overlay of every file involved, then to no overlay at all.
// Refusals deferred to run time (runtime fences) drop only the
// annotations they are blamed on: a fence the overlay did not cause
// stays as it is without the overlay.
let level: "annotations" | "files" | "all" | "fences" = "annotations";
for (let round = 0; shapeOverlayActive(); round++) {
const failed = lowered.diagnostics.length > 0;
const refusals = failed ? lowered.diagnostics : lowered.runtimeFences;
if (refusals.length === 0) break;
let changed: boolean;
if (!failed) {
level = "fences";
changed =
round < OVERLAY_ANNOTATION_ROUNDS &&
excludeRefusedOverlay(finalLoad.program, refusals, level);
} else {
if (level === "fences") level = "annotations";
if (round >= OVERLAY_ANNOTATION_ROUNDS && level === "annotations") level = "files";
else if (round > OVERLAY_ANNOTATION_ROUNDS) level = "all";
changed = excludeRefusedOverlay(finalLoad.program, refusals, level);
while (!changed && level !== "all") {
level = level === "annotations" ? "files" : "all";
changed = excludeRefusedOverlay(finalLoad.program, refusals, level);
}
}
if (process.env["SCRIPTC_DEBUG_SHAPE_FALLBACK"] !== undefined)
process.stderr.write(
`[shape-fallback] round ${round} level ${level}: ${refusals.length} refusals (${refusals
.slice(0, 3)
.map((d) => `${d.loc.file.split("/").pop()}:${d.loc.start} ${d.message.slice(0, 60)}`)
.join("; ")}), changed ${changed}\n`,
);
if (!changed) break;
finalLoad.dispose();
finalLoad = loadProgram(entryPath, { npmStatic: finalNpmStatic, externalTypes });
// The preflight fills the module order; the tsc world reads the
// files as written, so it answers as the first load did.
const reloadPreflight = checkPreflight(finalLoad);
if (reloadPreflight.length > 0)
return { ...lowered, module: null, diagnostics: reloadPreflight };
lowered = lowerOnce();
}
// The declaration-typed boundary's notes (SC6102) join the
// performance warnings; they never fail a build.
const notes = [
@@ -834,6 +893,6 @@ function loadFrontend(
refusals,
};
},
dispose: finalLoad.dispose,
dispose: () => finalLoad.dispose(),
};
}
packages/compiler/src/frontend/program.ts
+45 -8
View File
@@ -1012,10 +1012,9 @@ function loadProgram7(
}
}
// Shape inference's overlay: package files typed from proven facts. The
// tsc gate keeps reading the files as written.
// A load without the overlay must not see an earlier load's overlay
// files (the registry is per process).
resetShapeOverlay();
// tsc gate keeps reading the files as written. loadProgram resets the
// overlay registry before the lowering world's load.
const overlayOriginals = new Map<string, string | null>();
if (overlayer !== undefined && jsx.refusal === null) {
// Rounds: structural rewrites, then the typed overlay of the rebuilt
// program (shape-inference/index.ts).
@@ -1023,15 +1022,23 @@ function loadProgram7(
const overlay = overlayer(program, round);
if (overlay === null || overlay.size === 0) break;
const before = program;
for (const [file, text] of overlay) host.addVirtualFile(file, text);
program = host.createProgram([...programRoots, overridesDtsPath()], options);
// The tsc gate reads the files as written: its diagnostics are
// taken before the overlay's files replace them on the host, and
// its project world (load.projectWorld()) is built over the files
// as written.
if (tscWorld === undefined)
tscWorld = {
program: before,
diagnostics: ts.getPreEmitDiagnostics(before),
projectWorld: null,
declWorldFallback: true,
};
else if (tscWorld.program !== before) before.dispose();
for (const file of overlay.keys())
if (!overlayOriginals.has(file))
overlayOriginals.set(file, before.getSourceFile(file)?.text ?? null);
for (const [file, text] of overlay) host.addVirtualFile(file, text);
program = host.createProgram([...programRoots, overridesDtsPath()], options);
if (tscWorld.program !== before) before.dispose();
if (shapeOverlayDone()) break;
}
}
@@ -1055,7 +1062,10 @@ function loadProgram7(
externalTypeSpecifiersByFile,
...(tscWorld === undefined ? {} : { jsxSources: jsx.sources }),
...(tscWorld === undefined ? {} : { tscWorld }),
projectWorld: () => (projectWorld ??= host.createProgram(programRoots, options)),
projectWorld: () =>
(projectWorld ??= withFilesAsWritten(host, overlayOriginals, () =>
host.createProgram(programRoots, options),
)),
disposeAll: () => {
try {
projectWorld?.dispose();
@@ -1083,6 +1093,27 @@ export interface ProgramLoadOptions {
externalTypes?: readonly (readonly [string, string])[];
}
/** Runs `build` with the overlay's files served as written (their original
* texts), then serves the overlay's texts again. */
function withFilesAsWritten<T>(
host: ts.Ts7Host,
originals: ReadonlyMap<string, string | null>,
build: () => T,
): T {
if (originals.size === 0) return build();
const overlaid = new Map<string, string>();
for (const [file, text] of originals) {
const current = host.virtualFileText(file);
if (current !== undefined) overlaid.set(file, current);
if (text !== null) host.addVirtualFile(file, text);
}
try {
return build();
} finally {
for (const [file, text] of overlaid) host.addVirtualFile(file, text);
}
}
export function loadProgram(
entryPath: string,
services: FrontendServices,
@@ -1296,6 +1327,12 @@ export function loadProgram(
return projection;
};
try {
// The overlay registry is per process and describes the lowering
// world's load: a load without the overlay must not see an earlier
// load's overlay files. The authored world (authored()) is a second load
// of the same build, which preflight can create after the overlay
// rounds, so it leaves the registry as this load set it.
resetShapeOverlay();
const load = loadProgram7(
host,
entryPath,
packages/compiler/src/frontend/shape-inference/analyze.ts
+53 -45
View File
@@ -317,6 +317,14 @@ export function analyzeShapes(program: ts.Program, checker: ts.TypeChecker): Sha
}
class Analyzer {
/** locLabel per node: sink reasons and atom labels name the same sites
* over and over, and each label reads the file's text for its line. */
private readonly locLabels = new Map<Node, string>();
private loc(node: Node): string {
let label = this.locLabels.get(node);
if (label === undefined) this.locLabels.set(node, (label = locLabel(node)));
return label;
}
readonly atoms = new AtomTable();
readonly nodes: ValueNode[] = [];
readonly functions = new Map<string, FunctionInfo>();
@@ -830,7 +838,7 @@ class Analyzer {
if (this.traceTop)
this.nodeLabel.set(
id,
`binding ${(decl as { name?: { text?: string } }).name?.text ?? "?"}@${locLabel(decl)}${this.ctx !== "" ? ` [${this.ctx}]` : ""}`,
`binding ${(decl as { name?: { text?: string } }).name?.text ?? "?"}@${this.loc(decl)}${this.ctx !== "" ? ` [${this.ctx}]` : ""}`,
);
}
return id;
@@ -908,7 +916,7 @@ class Analyzer {
const nameNode = (fn as { name?: Node }).name;
const name = nameNode !== undefined ? (propertyNameText(nameNode) ?? "<computed>") : null;
const owner = ownerClass !== null ? this.className(ownerClass) : null;
const label = `${owner !== null ? `${owner}.` : ""}${name ?? (ts.isConstructorDeclaration(node) ? "constructor" : "<fn>")}@${locLabel(node)}`;
const label = `${owner !== null ? `${owner}.` : ""}${name ?? (ts.isConstructorDeclaration(node) ? "constructor" : "<fn>")}@${this.loc(node)}`;
const info: FunctionInfo = {
node,
ctx,
@@ -952,7 +960,7 @@ class Analyzer {
if (p.dotDotDotToken !== undefined) {
const arr = this.atoms.intern(`rest:${this.nodes.length}`, () => ({
kind: "array",
label: `rest@${locLabel(p)}`,
label: `rest@${this.loc(p)}`,
node: p,
}));
this.add(this.binding(p), arr);
@@ -998,7 +1006,7 @@ class Analyzer {
const p = decl.parent;
if (p !== undefined && ts.isVariableDeclaration(p) && ts.isIdentifier(p.name))
return p.name.text;
return `<class@${locLabel(decl)}>`;
return `<class@${this.loc(decl)}>`;
}
classAtoms(decl: Node): { cls: Atom; inst: Atom } {
@@ -1039,7 +1047,7 @@ class Analyzer {
ctx: "",
root: null,
walked: true,
label: `${label}.${suffix}@${locLabel(decl)}`,
label: `${label}.${suffix}@${this.loc(decl)}`,
params: [],
rest: -1,
thisNode: this.newNode(),
@@ -1187,7 +1195,7 @@ class Analyzer {
if (this.isFactory(info.node)) {
const d = this.atoms.intern(`dict:${this.nodeKey(site)}`, () => ({
kind: "dictionary",
label: `Object.create@${locLabel(site)}`,
label: `Object.create@${this.loc(site)}`,
node: site,
}));
this.add(dst, d);
@@ -1253,7 +1261,7 @@ class Analyzer {
syntheticArray(site: Node, tag: string): Atom {
return this.atoms.intern(`arr:${tag}:${this.nodeKey(site)}`, () => ({
kind: "array",
label: `${tag}[]@${locLabel(site)}`,
label: `${tag}[]@${this.loc(site)}`,
node: site,
}));
}
@@ -1361,7 +1369,7 @@ class Analyzer {
return;
}
if (ts.isThrowStatement(s)) {
this.sink(this.expr(s.expression), `thrown at ${locLabel(s)}`);
this.sink(this.expr(s.expression), `thrown at ${this.loc(s)}`);
return;
}
if (
@@ -1437,7 +1445,7 @@ class Analyzer {
Symbol.iterator member have a computed key; they are G sites. */
} else {
this.add(dst, this.top);
if (a.kind !== "top") this.escape(a, `iterated at ${locLabel(site)}`);
if (a.kind !== "top") this.escape(a, `iterated at ${this.loc(site)}`);
}
});
}
@@ -1453,7 +1461,7 @@ class Analyzer {
if (this.traceTop && !this.nodeLabel.has(id))
this.nodeLabel.set(
id,
`expr ${e.getText().slice(0, 40).replace(/\s+/g, " ")}@${locLabel(e)}`,
`expr ${e.getText().slice(0, 40).replace(/\s+/g, " ")}@${this.loc(e)}`,
);
return id;
}
@@ -1586,7 +1594,7 @@ class Analyzer {
if (ts.isTaggedTemplateExpression(e)) return this.taggedTemplate(e);
if (ts.isYieldExpression(e)) {
if (e.expression !== undefined)
this.sink(this.expr(e.expression), `yielded at ${locLabel(e)}`);
this.sink(this.expr(e.expression), `yielded at ${this.loc(e)}`);
return this.constNode(this.top);
}
if (ts.isMetaProperty(e)) return this.constNode(this.top);
@@ -1841,7 +1849,7 @@ class Analyzer {
const spreads: number[] = [];
const inPackage = isPackageJsFile(e.getSourceFile());
const label = (): string =>
`{${keys.slice(0, 6).join(",")}${keys.length > 6 ? ",…" : ""}${keysKnown ? "" : ",…"}}@${locLabel(e)}`;
`{${keys.slice(0, 6).join(",")}${keys.length > 6 ? ",…" : ""}${keysKnown ? "" : ",…"}}@${this.loc(e)}`;
// keys first, so the label is complete
for (const p of list(e.properties)) {
if (ts.isSpreadAssignment(p)) {
@@ -2053,7 +2061,7 @@ class Analyzer {
arrayLiteral(e: ts.ArrayLiteralExpression): number {
const atom = this.atoms.intern(`arr:${this.nodeKey(e)}`, () => ({
kind: "array",
label: `[]@${locLabel(e)}`,
label: `[]@${this.loc(e)}`,
node: e,
}));
const n = this.newNode();
@@ -2175,7 +2183,7 @@ class Analyzer {
if (ts.isIdentifier(t)) {
const decl = this.resolve(t);
if (decl === undefined || decl.getSourceFile().isDeclarationFile) {
this.sink(value, `global write at ${locLabel(t)}`);
this.sink(value, `global write at ${this.loc(t)}`);
return;
}
this.edge(value, this.binding(decl));
@@ -2421,7 +2429,7 @@ class Analyzer {
switch (a.kind) {
case "top":
this.noteTopWrite(name, site);
this.sink(value, `stored into ⊤.${name} at ${locLabel(site)}`);
this.sink(value, `stored into ⊤.${name} at ${this.loc(site)}`);
return;
case "instance": {
const member = this.lookupMember(a.node!, name, false);
@@ -2448,7 +2456,7 @@ class Analyzer {
case "prim":
return;
default:
this.sink(value, `stored into ${a.label} at ${locLabel(site)}`);
this.sink(value, `stored into ${a.label} at ${this.loc(site)}`);
}
});
}
@@ -2497,7 +2505,7 @@ class Analyzer {
switch (a.kind) {
case "top":
this.noteTopWrite(null, site);
this.sink(value, `stored into ⊤[k] at ${locLabel(site)}`);
this.sink(value, `stored into ⊤[k] at ${this.loc(site)}`);
return;
case "array":
// An index, or a named property under an unknown key.
@@ -2515,7 +2523,7 @@ class Analyzer {
return;
default:
// As a store under a name: a builtin, promise or collection keeps it where it isn't tracked.
this.sink(value, `stored into ${a.label}[k] at ${locLabel(site)}`);
this.sink(value, `stored into ${a.label}[k] at ${this.loc(site)}`);
}
});
}
@@ -2591,7 +2599,7 @@ class Analyzer {
if (inst.kind === "instance") this.callConstructorChain(base, inst, args, spread, e);
});
} else if (base === undefined)
for (const a of args) this.sink(a, `super(...) at ${locLabel(e)}`);
for (const a of args) this.sink(a, `super(...) at ${this.loc(e)}`);
return dst;
}
// CommonJS `require("<spec>")` of an analyzed module: its exports (the
@@ -2686,9 +2694,9 @@ class Analyzer {
case "prim":
return;
default:
for (const x of args) this.sink(x, `argument of ⊤ call at ${locLabel(site)}`);
for (const x of args) this.sink(x, `argument of ⊤ call at ${this.loc(site)}`);
if (recv !== null && a.kind === "top")
this.sink(recv, `receiver of ⊤ call at ${locLabel(site)}`);
this.sink(recv, `receiver of ⊤ call at ${this.loc(site)}`);
this.add(dst, this.top);
}
});
@@ -2731,7 +2739,7 @@ class Analyzer {
this.constructBuiltin(a.path ?? "", args, dst, e);
return;
default:
for (const x of args) this.sink(x, `argument of ⊤ constructor at ${locLabel(e)}`);
for (const x of args) this.sink(x, `argument of ⊤ constructor at ${this.loc(e)}`);
this.add(dst, this.top);
}
});
@@ -2746,7 +2754,7 @@ class Analyzer {
case "WeakSet": {
const c = this.atoms.intern(`coll:${this.nodeKey(site)}`, () => ({
kind: "collection",
label: `${path}@${locLabel(site)}`,
label: `${path}@${this.loc(site)}`,
node: site,
}));
this.add(dst, c);
@@ -2760,7 +2768,7 @@ class Analyzer {
this.add(dst, this.builtin("Error.instance"));
return;
case "Promise": {
const p = this.promiseAtom(site, locLabel(site));
const p = this.promiseAtom(site, this.loc(site));
this.add(this.field(p, "value"), this.top);
this.add(dst, p);
if (args[0] !== undefined) {
@@ -2793,11 +2801,11 @@ class Analyzer {
// The handler's traps run on every operation with the target.
if (args[0] !== undefined)
this.ops.push({ kind: "proxy", recv: args[0], name: null, node: site });
for (const x of args) this.sink(x, `argument of new Proxy at ${locLabel(site)}`);
for (const x of args) this.sink(x, `argument of new Proxy at ${this.loc(site)}`);
this.add(dst, this.top);
return;
default:
for (const x of args) this.sink(x, `argument of new ${path} at ${locLabel(site)}`);
for (const x of args) this.sink(x, `argument of new ${path} at ${this.loc(site)}`);
this.add(dst, this.top);
}
}
@@ -2824,13 +2832,13 @@ class Analyzer {
this.callFunction(this.infoOfAtom(f)!, null, cbArgs, false, ret ?? this.newNode(), site);
this.forceLoop = saved;
} else if (f.kind === "top" || f.kind === "builtin") {
for (const x of cbArgs) this.sink(x, `argument of ⊤ callback at ${locLabel(site)}`);
for (const x of cbArgs) this.sink(x, `argument of ⊤ callback at ${this.loc(site)}`);
if (ret !== null) this.add(ret, this.top);
}
});
};
const escapeAll = (): void => {
for (const x of args) this.sink(x, `argument of ${path} at ${locLabel(site)}`);
for (const x of args) this.sink(x, `argument of ${path} at ${this.loc(site)}`);
this.add(dst, this.top);
};
// Methods of builtin receivers.
@@ -2948,7 +2956,7 @@ class Analyzer {
return;
}
escapeAll();
this.escape(recv, `array.${method} at ${locLabel(site)}`);
this.escape(recv, `array.${method} at ${this.loc(site)}`);
return;
}
if (path.startsWith("string.") || path.startsWith("number.") || path.startsWith("boolean.")) {
@@ -2977,7 +2985,7 @@ class Analyzer {
return;
}
if (path.startsWith("promise.")) {
const out = this.promiseAtom(site, locLabel(site));
const out = this.promiseAtom(site, this.loc(site));
const value = this.field(recv, "value");
const ret = this.newNode();
if (method === "then") {
@@ -3047,7 +3055,7 @@ class Analyzer {
if (info !== undefined && args.length > 1) {
this.activate(info);
for (const p of info.params) this.add(p, this.top);
for (const x of args.slice(1)) this.sink(x, `bound argument at ${locLabel(site)}`);
for (const x of args.slice(1)) this.sink(x, `bound argument at ${this.loc(site)}`);
}
this.add(dst, recv);
return;
@@ -3145,7 +3153,7 @@ class Analyzer {
for (const s of args.slice(1)) this.copyFields(s, t, site);
if (t.kind === "top")
for (const s of args.slice(1))
this.sink(s, `Object.assign into ⊤ at ${locLabel(site)}`);
this.sink(s, `Object.assign into ⊤ at ${this.loc(site)}`);
});
this.edge(arg(0), dst);
return;
@@ -3154,20 +3162,20 @@ class Analyzer {
case "Object.defineProperties":
case "Reflect.defineProperty":
this.ops.push({ kind: "defineProperty", recv: arg(0), name: null, node: site });
this.sink(arg(2), `property descriptor at ${locLabel(site)}`);
this.sink(arg(1), `property descriptor at ${locLabel(site)}`);
this.sink(arg(2), `property descriptor at ${this.loc(site)}`);
this.sink(arg(1), `property descriptor at ${this.loc(site)}`);
this.edge(arg(0), dst);
return;
case "Object.setPrototypeOf":
case "Reflect.setPrototypeOf":
this.ops.push({ kind: "setPrototypeOf", recv: arg(0), name: null, node: site });
this.sink(arg(1), `prototype at ${locLabel(site)}`);
this.sink(arg(1), `prototype at ${this.loc(site)}`);
this.edge(arg(0), dst);
return;
case "Object.create": {
const d = this.atoms.intern(`dict:${this.nodeKey(site)}`, () => ({
kind: "dictionary",
label: `Object.create@${locLabel(site)}`,
label: `Object.create@${this.loc(site)}`,
node: site,
}));
this.add(dst, d);
@@ -3176,7 +3184,7 @@ class Analyzer {
case "Object.fromEntries": {
const d = this.atoms.intern(`dict:${this.nodeKey(site)}`, () => ({
kind: "dictionary",
label: `fromEntries@${locLabel(site)}`,
label: `fromEntries@${this.loc(site)}`,
node: site,
}));
const pairs = this.newNode();
@@ -3225,7 +3233,7 @@ class Analyzer {
site,
);
else if (f.kind === "top")
this.sink(revived, `JSON.parse reviver at ${locLabel(site)}`);
this.sink(revived, `JSON.parse reviver at ${this.loc(site)}`);
});
this.edge(revived, dst);
return;
@@ -3252,7 +3260,7 @@ class Analyzer {
site,
);
else if (f.kind === "top")
this.sink(serialized, `JSON.stringify replacer at ${locLabel(site)}`);
this.sink(serialized, `JSON.stringify replacer at ${this.loc(site)}`);
});
prim("string");
prim("undefined");
@@ -3281,7 +3289,7 @@ class Analyzer {
if (c.kind === "builtin" && c.path === "Proxy")
this.ops.push({ kind: "proxy", recv: listed, name: null, node: site });
});
this.sink(listed, `argument of ${path} at ${locLabel(site)}`);
this.sink(listed, `argument of ${path} at ${this.loc(site)}`);
escapeAll();
return;
}
@@ -3310,23 +3318,23 @@ class Analyzer {
prim("bigint");
return;
case "Promise.resolve": {
const p = this.promiseAtom(site, locLabel(site));
const p = this.promiseAtom(site, this.loc(site));
if (args[0] !== undefined) this.awaitInto(args[0], this.field(p, "value"));
else this.add(this.field(p, "value"), this.prim("undefined"));
this.add(dst, p);
return;
}
case "Promise.reject": {
const p = this.promiseAtom(site, locLabel(site));
const p = this.promiseAtom(site, this.loc(site));
this.add(dst, p);
if (args[0] !== undefined) this.sink(args[0], `rejection at ${locLabel(site)}`);
if (args[0] !== undefined) this.sink(args[0], `rejection at ${this.loc(site)}`);
return;
}
case "Promise.all":
case "Promise.allSettled":
case "Promise.race":
case "Promise.any": {
const p = this.promiseAtom(site, locLabel(site));
const p = this.promiseAtom(site, this.loc(site));
const items = this.newNode();
if (args[0] !== undefined) this.iterateInto(args[0], items, site);
if (path === "Promise.all") {
@@ -3340,7 +3348,7 @@ class Analyzer {
return;
}
case "promise.resolver":
for (const x of args) this.sink(x, `promise resolution at ${locLabel(site)}`);
for (const x of args) this.sink(x, `promise resolution at ${this.loc(site)}`);
return;
}
if (path.startsWith("console.")) {
packages/compiler/src/frontend/shape-inference/fallback.ts
+242
View File
@@ -0,0 +1,242 @@
/* Shape inference never breaks compilation: when the lowering refuses code
* with the overlay on, the overlay annotations the refusal depends on are
* dropped and the program is loaded and lowered again, so those
* declarations compile untyped (as they do without the overlay).
*
* Attribution reads the refused program. A refusal depends on:
* - the annotations of the declarations enclosing it (an annotated
* function's parameter types, a typed local, a receiver cast);
* - the annotations of the declarations its expression names (a method
* whose annotated return type the refused member access reads, a field
* whose annotated type the access sees), followed through the return
* expressions of unannotated functions, whose result types the checker
* infers from what they return.
*
* pipeline.ts escalates when a round does not converge: first the
* overlay of every file involved, then the whole overlay for the build. */
import * as ts from "../ts7/adapter.js";
import type { ScrDiagnostic } from "../../diagnostics/diagnostic.js";
import { OVERLAY_MARK, overlayAnnotationIds } from "./overlay.js";
export interface OverlayAttribution {
/** Annotation ids per overlay file. */
readonly annotations: Map<string, Set<number>>;
/** Overlay files a refusal involves (where it sits, or what it names). */
readonly files: Set<string>;
/** Refusals with no overlay annotation or file to blame. */
readonly unattributed: number;
}
/** How far the attribution follows unannotated functions' return
* expressions. */
const RETURN_DEPTH = 3;
export function attributeOverlayRefusals(
program: ts.Program,
diagnostics: readonly ScrDiagnostic[],
overlayFiles: ReadonlySet<string>,
): OverlayAttribution {
const annotations = new Map<string, Set<number>>();
const files = new Set<string>();
let unattributed = 0;
let checker: ts.TypeChecker | null = null;
// Blamed annotations and files so far (a refusal that adds none is
// unattributed).
let blamed = 0;
const addFile = (file: string): void => {
if (files.has(file)) return;
files.add(file);
blamed++;
};
const add = (file: string, id: number): void => {
let set = annotations.get(file);
if (set === undefined) annotations.set(file, (set = new Set()));
if (!set.has(id)) blamed++;
set.add(id);
addFile(file);
};
// The ids in the trivia before a node (where the overlay writes them).
const marksOf = (node: ts.Node): number[] => {
const sf = node.getSourceFile();
if (!overlayFiles.has(sf.fileName)) return [];
const trivia = sf.text.slice(node.pos, ts.tokenStartOfNode(node));
return trivia.includes(OVERLAY_MARK) ? overlayAnnotationIds(trivia) : [];
};
const blameNode = (node: ts.Node): boolean => {
const ids = marksOf(node);
for (const id of ids) add(node.getSourceFile().fileName, id);
return ids.length > 0;
};
// A declaration's annotation: on the declaration, or on the statement
// holding it (`/** @type */ const x = …`, `/** @type */ this.f = …`).
const visitedDecls = new Set<ts.Node>();
const blameDeclaration = (decl: ts.Node, depth: number): void => {
if (visitedDecls.has(decl)) return;
visitedDecls.add(decl);
const sf = decl.getSourceFile();
if (!overlayFiles.has(sf.fileName)) return;
addFile(sf.fileName);
let marked = false;
for (let n: ts.Node | undefined = decl, i = 0; n !== undefined && i < 4; n = n.parent, i++) {
if (blameNode(n)) marked = true;
if (ts.isStatement(n) || isClassMember(n) || ts.isSourceFile(n)) break;
}
// An unannotated function's result type comes from its returns.
if (!marked && depth < RETURN_DEPTH && ts.isFunctionLike(decl)) {
const body = (decl as { body?: ts.Node }).body;
if (body !== undefined) {
if (!ts.isBlock(body)) blameReferences(body, depth + 1);
else
visitReturns(body, (e) => {
blameReferences(e, depth + 1);
});
}
}
};
// Every declaration an expression's names resolve to.
const blameReferences = (root: ts.Node, depth: number): void => {
checker ??= program.getTypeChecker();
const visit = (n: ts.Node): void => {
if (ts.isIdentifier(n) || ts.isPrivateIdentifier(n)) {
let symbol: ts.Symbol | undefined;
try {
symbol = checker!.getSymbolAtLocation(n);
} catch {
symbol = undefined;
}
for (const d of symbol?.declarations ?? []) blameDeclaration(d, depth);
}
ts.forEachChild(n, (c) => {
visit(c);
return undefined;
});
};
visit(root);
};
// Calls of each function declared in an overlay file, by declaration.
let calls: Map<ts.Node, ts.CallExpression[]> | null = null;
const callsOf = (fn: ts.Node): readonly ts.CallExpression[] => {
if (calls === null) {
calls = new Map();
checker ??= program.getTypeChecker();
for (const file of overlayFiles) {
const sf = program.getSourceFile(file);
if (sf === undefined) continue;
const visit = (n: ts.Node): void => {
if (ts.isCallExpression(n) && ts.isIdentifier(n.expression)) {
let decl: ts.Node | undefined;
try {
decl = checker!.getSymbolAtLocation(n.expression)?.declarations[0];
} catch {
decl = undefined;
}
if (decl !== undefined) {
let list = calls!.get(decl);
if (list === undefined) calls!.set(decl, (list = []));
list.push(n);
}
}
ts.forEachChild(n, (c) => {
visit(c);
return undefined;
});
};
visit(sf);
}
}
return calls.get(fn) ?? [];
};
for (const d of diagnostics) {
const sf = program.getSourceFile(d.loc.file);
if (sf === undefined) {
unattributed++;
continue;
}
const before = blamed;
if (overlayFiles.has(sf.fileName)) addFile(sf.fileName);
const node = innermostNodeAt(sf, d.loc.start, Math.max(d.loc.end, d.loc.start));
// Enclosing annotated declarations, then the names of the refused
// expression, or failing that of the statement around it.
let statement: ts.Node = node;
for (let n: ts.Node | undefined = node; n !== undefined; n = n.parent) {
blameNode(n);
if (statement === node && (ts.isStatement(n) || isClassMember(n))) statement = n;
}
const own = blamed;
blameReferences(node, 0);
if (blamed === own && statement !== node) blameReferences(statement, 0);
// A function without parameter annotations is instantiated per call
// site from its arguments' types: blame what the arguments name.
if (blamed === own) {
const fn = enclosingFunction(node);
if (fn !== undefined)
for (const call of callsOf(fn))
for (const arg of call.arguments.nodes) blameReferences(arg, 1);
}
// Still nothing: the lowering of the enclosing function changed as a
// whole (an instantiation the annotations inside it made possible).
if (blamed === before) {
const fn = enclosingFunctionLike(node);
if (fn !== undefined && overlayFiles.has(fn.getSourceFile().fileName)) {
const visit = (n: ts.Node): void => {
blameNode(n);
ts.forEachChild(n, (c) => {
visit(c);
return undefined;
});
};
visit(fn);
}
}
if (blamed === before) unattributed++;
}
return { annotations, files, unattributed };
}
function enclosingFunction(node: ts.Node): ts.Node | undefined {
for (let n: ts.Node | undefined = node.parent; n !== undefined; n = n.parent)
if (ts.isFunctionDeclaration(n)) return n;
else if (ts.isFunctionLike(n)) return undefined;
return undefined;
}
function enclosingFunctionLike(node: ts.Node): ts.Node | undefined {
for (let n: ts.Node | undefined = node; n !== undefined; n = n.parent)
if (ts.isFunctionLike(n)) return n;
return undefined;
}
function visitReturns(body: ts.Node, f: (e: ts.Expression) => void): void {
const visit = (n: ts.Node): void => {
if (ts.isFunctionLike(n) || ts.isClassLikeDeclaration(n)) return;
if (ts.isReturnStatement(n) && n.expression !== undefined) f(n.expression);
ts.forEachChild(n, (c) => {
visit(c);
return undefined;
});
};
ts.forEachChild(body, (c) => {
visit(c);
return undefined;
});
}
/** The innermost node whose token span covers [start, end). */
function innermostNodeAt(sf: ts.SourceFile, start: number, end: number): ts.Node {
let node: ts.Node = sf;
for (;;) {
let next: ts.Node | undefined;
ts.forEachChild(node, (c) => {
if (next === undefined && ts.tokenStartOfNode(c) <= start && c.end >= end) next = c;
return undefined;
});
if (next === undefined) return node;
node = next;
}
}
function isClassMember(n: ts.Node): boolean {
return n.parent !== undefined && ts.isClassLikeDeclaration(n.parent) && n !== n.parent.name;
}
packages/compiler/src/frontend/shape-inference/index.ts
+109 -4
View File
@@ -7,7 +7,8 @@ import type { ScrDiagnostic } from "../../diagnostics/diagnostic.js";
import { npmStaticPackageOfPath } from "../npm-static.js";
import { isShapelessPackage } from "../package-shims.js";
import { analyzeShapes, isPackageJsFile } from "./analyze.js";
import { OVERLAY_MARK, planShapeOverlay } from "./overlay.js";
import { OVERLAY_MARK, planShapeOverlay, type OverlayExclusions } from "./overlay.js";
import { attributeOverlayRefusals } from "./fallback.js";
import { planStaticObjectRewrites } from "./static-objects.js";
import { isJsSourceFile } from "../program.js";
import { buildShapeReport } from "./report.js";
@@ -58,6 +59,7 @@ export function shapeOverlayAnnotationCount(): number {
}
export function resetShapeOverlay(): void {
staticRewriteFiles = new Set();
staticObjectStats = { classes: 0, functions: 0, directCalls: 0 };
overlayDone = false;
overlayFiles = new Set();
@@ -73,12 +75,90 @@ export function shapeOverlayEnabled(): boolean {
return process.env["SCRIPTC_SHAPE_OVERLAY"] === "1";
}
/* The build's fallback state (fallback.ts): annotations and files whose
* lowering was refused, and whether the build dropped the overlay. It
* outlives the loads of one build. */
let exclusions: { annotations: Map<string, Set<number>>; files: Set<string> } = {
annotations: new Map(),
files: new Set(),
};
let overlayDropped = false;
let droppedAnnotations = 0;
/** Starts a build: no annotation is excluded yet. */
export function beginShapeOverlayBuild(): void {
exclusions = { annotations: new Map(), files: new Set() };
overlayDropped = false;
droppedAnnotations = 0;
}
/** Whether the current load has overlay files or structural rewrites. */
export function shapeOverlayActive(): boolean {
const { classes, functions } = staticObjectStats;
return overlayFiles.size > 0 || classes + functions > 0;
}
/** The overlay's answer to refused lowering diagnostics: it excludes what
* the refusals depend on, at the given level, and says whether a reload
* can change anything.
* - "annotations": the blamed annotations (or, for a blamed file without
* blamed annotations, that file);
* - "fences": like "annotations", for refusals deferred to run time, and
* never a whole file;
* - "files": every overlay file involved;
* - "all": the whole overlay. */
export function excludeRefusedOverlay(
program: ts.Program,
diagnostics: readonly ScrDiagnostic[],
level: "annotations" | "files" | "all" | "fences",
): boolean {
if (overlayDropped) return false;
const files = new Set(overlayFiles);
// Files with structural rewrites only also count as overlay files.
for (const f of staticRewriteFiles) files.add(f);
if (level === "all") {
overlayDropped = true;
droppedAnnotations = overlayAnnotations;
return true;
}
const blame = attributeOverlayRefusals(program, diagnostics, files);
let changed = false;
for (const file of blame.files) {
const ids = blame.annotations.get(file);
// Deferred refusals drop only the annotations they are blamed on.
if (level === "fences" && ids === undefined) continue;
if (level === "files" || ids === undefined) {
if (!exclusions.files.has(file)) {
exclusions.files.add(file);
changed = true;
}
continue;
}
let set = exclusions.annotations.get(file);
if (set === undefined) exclusions.annotations.set(file, (set = new Set()));
for (const id of ids) {
if (set.has(id)) continue;
set.add(id);
droppedAnnotations++;
changed = true;
}
}
return changed;
}
let staticRewriteFiles: ReadonlySet<string> = new Set();
function overlayExclusions(): OverlayExclusions {
return exclusions;
}
/** loadProgram7's overlayer: analyze, plan, and register the overlay files. */
export function shapeOverlayer(
program: ts.Program,
round: number,
): ReadonlyMap<string, string> | null {
if (round === 0) resetShapeOverlay();
if (overlayDropped) return null;
const analysis = analyzeShapes(program, program.getTypeChecker());
if (analysis.budgetExceeded) return null;
// Package JavaScript of packages that keep shapes on, and the program's
@@ -92,7 +172,11 @@ export function shapeOverlayer(
// Round 0: the structural rewrites (static-objects.ts); the program is
// rebuilt from them and analyzed again before the typed overlay.
if (round === 0 && process.env["SCRIPTC_SHAPE_STATIC_OBJECTS"] !== "0") {
const rewrites = planStaticObjectRewrites(analysis, jsFiles);
const rewrites = planStaticObjectRewrites(
analysis,
jsFiles.filter((sf) => !exclusions.files.has(sf.fileName)),
);
staticRewriteFiles = new Set(rewrites.files.keys());
staticObjectStats = {
classes: rewrites.classes,
functions: rewrites.functions,
@@ -107,7 +191,7 @@ export function shapeOverlayer(
}
if (round > 1) return null;
const report = buildShapeReport(analysis, "1");
const overlay = planShapeOverlay(analysis, report.sites, jsFiles);
const overlay = planShapeOverlay(analysis, report.sites, jsFiles, overlayExclusions());
overlayFiles = new Set(overlay.files.keys());
overlayAnnotations = overlay.annotations;
overlayDeadBranches = overlay.deadBranches;
@@ -181,6 +265,26 @@ export function shapeInferredFactCheckCount(): number {
* checks of inferred facts the build inserted. A note, never a failure. */
export function shapeOverlayReport(entryFile: string): ScrDiagnostic[] {
const { classes, functions, directCalls } = staticObjectStats;
// Annotations whose lowering was refused (fallback.ts); they compile
// untyped.
const fallback = overlayDropped
? "the lowering refused code the overlay typed, so the build compiles without it"
: droppedAnnotations + exclusions.files.size > 0
? `it dropped ${droppedAnnotations} annotation${droppedAnnotations === 1 ? "" : "s"}` +
(exclusions.files.size > 0
? ` and the overlay of ${exclusions.files.size} file${exclusions.files.size === 1 ? "" : "s"}`
: "") +
" whose lowering was refused"
: null;
if (overlayDropped)
return [
{
code: "SC6103",
message: `shape inference: ${fallback}`,
loc: { file: entryFile, start: 0, end: 0 },
hint: "--no-shape-overlay compiles without the overlay",
},
];
if (overlayFiles.size === 0 && classes + functions === 0) return [];
const packages = new Set(
[...overlayFiles].map((f) => npmStaticPackageOfPath(f) ?? "the program"),
@@ -197,7 +301,8 @@ export function shapeOverlayReport(entryFile: string): ScrDiagnostic[] {
: "") +
(classes + functions > 0
? `; it compiled ${classes} constructor function${classes === 1 ? "" : "s"} as classes and ${functions} namespace function${functions === 1 ? "" : "s"} as direct functions (${directCalls} direct call${directCalls === 1 ? "" : "s"})`
: ""),
: "") +
(fallback !== null ? `; ${fallback}` : ""),
loc: { file: entryFile, start: 0, end: 0 },
hint: "a failing check reports a scriptc analysis bug; --no-shape-overlay compiles without the overlay",
},
packages/compiler/src/frontend/shape-inference/overlay.ts
+47 -18
View File
@@ -54,10 +54,28 @@ function list<T extends ts.Node>(arr: { readonly nodes: readonly T[] } | undefin
let debugIndex = 0;
/** Annotations a build dropped because their lowering was refused: per
* file, the annotation ids (positions in the overlay's input text); a file
* in `files` gets no overlay at all. */
export interface OverlayExclusions {
readonly annotations: ReadonlyMap<string, ReadonlySet<number>>;
readonly files: ReadonlySet<string>;
}
export const NO_EXCLUSIONS: OverlayExclusions = { annotations: new Map(), files: new Set() };
/** The ids of the overlay annotations in a comment or trivia text. */
export function overlayAnnotationIds(text: string): number[] {
const ids: number[] = [];
for (const m of text.matchAll(/@scriptcShape #(\d+)/g)) ids.push(Number(m[1]));
return ids;
}
export function planShapeOverlay(
analysis: ShapeAnalysis,
verdicts: readonly SiteVerdict[],
packageFiles: readonly ts.SourceFile[],
exclusions: OverlayExclusions = NO_EXCLUSIONS,
): ShapeOverlay {
const pSites = new Map<Atom, SiteVerdict>();
const siteVerdicts = new Set<Atom>(verdicts.map((v) => v.site.atom));
@@ -96,9 +114,20 @@ export function planShapeOverlay(
}
}
for (const sf of packageFiles) {
if (exclusions.files.has(sf.fileName)) continue;
const typedefs = new Map<string, string>();
const typedefText: string[] = [];
const edits: Edit[] = [];
const dropped = exclusions.annotations.get(sf.fileName);
// One annotation: a JSDoc comment at `pos` carrying MARK and the
// annotation's id (its position), so a refused lowering can name it and
// the next load drop it. `close`: the text that closes a type cast.
const annotate = (pos: number, tags: string, close?: number): void => {
if (dropped?.has(pos) === true) return;
edits.push({ pos, text: `/** ${tags} ${MARK} #${pos} */ ${close !== undefined ? "(" : ""}` });
if (close !== undefined) edits.push({ pos: close, text: ")" });
annotations++;
};
const className = (decl: ts.Node): string | null => {
const own = (decl as ts.ClassLikeDeclaration).name;
@@ -111,7 +140,8 @@ export function planShapeOverlay(
own === undefined ||
!ts.isClassDeclaration(decl) ||
decl.parent !== other ||
!overlaid.has(other)
!overlaid.has(other) ||
exclusions.files.has(other.fileName)
)
return null;
const alias = `__shape_class_${own.text}`;
@@ -172,6 +202,15 @@ export function planShapeOverlay(
objects++;
if (analysis.escaped.has(a) || (siteVerdicts.has(a) && !pSites.has(a)))
return inner ? "any" : null;
// Only arrays the program writes as literals: a builtin's array
// (a match result, `new Array(n)`) may have holes, and the
// checker already types it from the library.
if (
a.node === undefined ||
!ts.isArrayLiteralExpression(a.node) ||
a.node.elements.nodes.some((e) => e.kind === ts.SyntaxKind.OmittedExpression)
)
return inner ? "any" : null;
const el = analysis.elements.get(a);
if (el !== undefined) for (const x of analysis.nodes[el]!.atoms) elements.add(x);
hasArray = true;
@@ -338,9 +377,7 @@ export function planShapeOverlay(
!declaresMember(cls, node.expression.name.text)
) {
const self = node.expression.expression;
edits.push({ pos: ts.tokenStartOfNode(self), text: `/** @type {${t}} ${MARK} */ (` });
edits.push({ pos: self.end, text: ")" });
annotations++;
annotate(ts.tokenStartOfNode(self), `@type {${t}}`, self.end);
}
ts.forEachChild(node, (c) => {
scan(c);
@@ -433,7 +470,7 @@ export function planShapeOverlay(
const t = typeOf(paramValues(fn, index), false, 0);
if (t !== null) tags.push(`@param {${t}} ${p.name.text}`);
});
return tags.length > 0 ? `${tags.join(" ")} ${MARK}` : null;
return tags.length > 0 ? tags.join(" ") : null;
};
const visit = (node: ts.Node): void => {
@@ -451,10 +488,7 @@ export function planShapeOverlay(
) {
if (node.body !== undefined) {
const tags = paramTags(node);
if (tags !== null) {
edits.push({ pos: ts.tokenStartOfNode(node), text: `/** ${tags} */ ` });
annotations++;
}
if (tags !== null) annotate(ts.tokenStartOfNode(node), tags);
}
// `this.f = …` fields of a constructor's top-level statements.
if (ts.isConstructorDeclaration(node) && node.body !== undefined) {
@@ -477,10 +511,7 @@ export function planShapeOverlay(
if (seen.has(name)) continue;
seen.add(name);
const t = cls !== undefined ? fieldType(cls, name) : null;
if (t !== null) {
edits.push({ pos: ts.tokenStartOfNode(st), text: `/** @type {${t}} ${MARK} */ ` });
annotations++;
}
if (t !== null) annotate(ts.tokenStartOfNode(st), `@type {${t}}`);
}
}
}
@@ -496,10 +527,7 @@ export function planShapeOverlay(
const init = d.initializer;
if (ts.isIdentifier(d.name) && init !== undefined && !literalInitializer(init)) {
const t = typeOf(analysis.declValues(d), false, 0);
if (t !== null) {
edits.push({ pos: ts.tokenStartOfNode(node), text: `/** @type {${t}} ${MARK} */ ` });
annotations++;
}
if (t !== null) annotate(ts.tokenStartOfNode(node), `@type {${t}}`);
}
}
ts.forEachChild(node, (c) => {
@@ -530,6 +558,7 @@ export function planShapeOverlay(
const hoisted = new Map<number, readonly string[]>();
for (const branch of dead) {
const start = ts.tokenStartOfNode(branch);
if (dropped?.has(start) === true) continue;
if (ranges.some(([lo, hi]) => start >= lo && start < hi)) continue;
const names = hoistedNames(branch);
if (names === null) continue;
@@ -554,7 +583,7 @@ export function planShapeOverlay(
edits.push({
pos: lo,
end: hi,
text: `{ ${vars}throw new Error(${JSON.stringify(message)}); }`,
text: `{ /** ${MARK} #${lo} */ ${vars}throw new Error(${JSON.stringify(message)}); }`,
});
deadBranches++;
}
packages/compiler/src/frontend/shape-inference/registry.test.ts
+34
View File
@@ -0,0 +1,34 @@
/* loadProgram resets the shape overlay's registry for the lowering world's
* load only. The authored world that preflight can create afterwards
* (load.projectWorld()) is a second load of the same build: it leaves the
* lowering load's overlay registered, so the fallback (fallback.ts) still
* finds the overlay behind a refusal. */
import { join } from "node:path";
import { expect, test } from "vitest";
import { loadProgram } from "../program-node.js";
import { shapeOverlayActive, shapeOverlayFileNames } from "./index.js";
const entry = join(
import.meta.dirname,
"../../../../../tests/fixtures/npm/cases/static-objects/js-main.cjs",
);
test("the authored world leaves the lowering load's overlay registered", () => {
const saved = process.env["SCRIPTC_SHAPE_OVERLAY"];
process.env["SCRIPTC_SHAPE_OVERLAY"] = "1";
try {
const load = loadProgram(entry, { npmStatic: ["static-ns"] });
try {
expect(shapeOverlayActive()).toBe(true);
const files = shapeOverlayFileNames();
load.projectWorld();
expect(shapeOverlayActive()).toBe(true);
expect(shapeOverlayFileNames()).toEqual(files);
} finally {
load.dispose();
}
} finally {
if (saved === undefined) delete process.env["SCRIPTC_SHAPE_OVERLAY"];
else process.env["SCRIPTC_SHAPE_OVERLAY"] = saved;
}
});
packages/compiler/src/frontend/ts7/program-host.ts
+5
View File
@@ -201,6 +201,11 @@ export class Ts7Host {
});
}
/** The text of a registered in-memory file, or undefined. */
virtualFileText(path: string): string | undefined {
return this.virtualFiles.get(tsgoPath(path));
}
/** The text this host serves for a file (null when it serves none),
* without asking the server for its syntax tree. */
sourceText(fileName: string): string | null {
packages/compiler/src/index.ts
+7 -5
View File
@@ -482,10 +482,11 @@ export async function compile(
opts: CompileRequestOptions,
): Promise<CompileRequestResult> {
clearCompileSessionCaches();
// `shapeOverlay: false` reaches the frontend (and its child process)
// through the environment the overlay reads.
// `shapeOverlay` reaches the frontend (and its child process) through
// the environment the overlay reads.
const overlayEnv = process.env["SCRIPTC_SHAPE_OVERLAY"];
if (opts.shapeOverlay === false) process.env["SCRIPTC_SHAPE_OVERLAY"] = "0";
if (opts.shapeOverlay !== undefined)
process.env["SCRIPTC_SHAPE_OVERLAY"] = opts.shapeOverlay ? "1" : "0";
const frontendInputs = new FrontendInputTracker();
const warnings: CompileWarningSink = { warnings: [], sourceTexts: null };
// compileTracked starts it for a build with no earlier build of the same invocation.
@@ -498,7 +499,7 @@ export async function compile(
} finally {
setLibDiagnosticsStore(null);
frontendProcess?.release();
if (opts.shapeOverlay === false) {
if (opts.shapeOverlay !== undefined) {
if (overlayEnv === undefined) delete process.env["SCRIPTC_SHAPE_OVERLAY"];
else process.env["SCRIPTC_SHAPE_OVERLAY"] = overlayEnv;
}
@@ -1313,7 +1314,8 @@ function configureLibDiagnostics(root: string | null): void {
export async function runFrontendProcess(requestPath: string, resultPath: string): Promise<void> {
const request = JSON.parse(await readFile(requestPath, "utf8")) as FrontendProcessRequest;
// The process started before the build's options were known.
if (request.opts.shapeOverlay === false) process.env["SCRIPTC_SHAPE_OVERLAY"] = "0";
if (request.opts.shapeOverlay !== undefined)
process.env["SCRIPTC_SHAPE_OVERLAY"] = request.opts.shapeOverlay ? "1" : "0";
clearCompileSessionCaches();
setProvenanceSources(request.provenance);
// The compile that started this process prepared the cache root.
scripts/ci-test-plan.mjs
+1
View File
@@ -7,6 +7,7 @@ export const corpusFiles = [
"tests/harness/differential.test.ts",
"tests/harness/llvm-differential.test.ts",
"tests/harness/npm.test.ts",
"tests/harness/npm-shape-overlay.test.ts",
"tests/harness/server.test.ts",
"tests/harness/test262.test.ts",
"tests/harness/three.test.ts",
tests/harness/npm-shape-overlay.test.ts
+7
View File
@@ -0,0 +1,7 @@
/* Shape inference's overlay never breaks compilation: every npm case
* compiles with the overlay on and builds and runs exactly as it does in the
* default build (npm-suite.ts's KNOWN). An overlay annotation whose lowering
* is refused is dropped, so its declaration compiles untyped. */
import { defineNpmSuite } from "./npm-suite.js";
defineNpmSuite({ label: "npm native, shape overlay", shapeOverlay: true });
tests/harness/npm-suite.ts
+282
View File
@@ -0,0 +1,282 @@
/* The npm suite (npm.test.ts runs it as built by default,
* npm-shape-overlay.test.ts with shape inference's overlay on).
*
* npm packages compiled natively: every program under tests/fixtures/npm/cases
* (plus the commander fixtures in npm-cases.ts) builds with the default
* static npm compile and runs against Node. A case matches Node (stdout and
* exit status for every argv list, stderr when it opts in with
* `// @stderr`) unless KNOWN records how it differs:
*
* - build: the compile refuses it with this diagnostic code;
* - run: it compiles, and the program's output names this diagnostic code
* (a refusal deferred to the operation that reached it), for every argv
* list or for the listed `argvs`, while the other lists match Node;
* - divergence: it runs but does not match Node yet. These run as expected
* failures, so a fix that makes one match fails here until KNOWN drops it.
*
* The fixture node_modules are committed test data. Binaries compile the
* package sources at build time and read nothing from node_modules at run
* time.
*/
import { execFile } from "node:child_process";
import { createHash } from "node:crypto";
import { globSync, mkdirSync, readFileSync } from "node:fs";
import { dirname, join } from "node:path";
import { promisify } from "node:util";
import { describe, expect, test } from "vitest";
import { compile } from "@scriptc/compiler";
import { npmCases, type NpmCase } from "./npm-cases.js";
import { shardSelect, shardSuffix } from "./shard.js";
const execFileAsync = promisify(execFile);
const repoRoot = join(import.meta.dirname, "../..");
const fixturesRoot = join(repoRoot, "tests/fixtures");
const cacheDir = join(repoRoot, "node_modules/.cache/scriptc-tests");
const sanitize = process.env["SCRIPTC_SAN"] === "1";
type Known = { build: string } | { run: string; argvs?: string[][] } | { divergence: string };
const KNOWN: Record<string, Known> = {
"any-chaining": { build: "SC2003" },
"assert-shims": { run: "SC1043" },
"buffer-shims": { run: "SC2020" },
"bytes-exit": { build: "SC2004" },
"cjs-esm-named": { run: "SC1090" },
"cjs-lexer-aligned": { run: "SC1090" },
"commander-calc": { run: "SC1090", argvs: [["--help"]] },
"commander-calc-async": { run: "SC1090", argvs: [["--help"]] },
"create-require": { run: "SC1090" },
"dotfolder": { run: "SC1090" },
"dual-package-kinds": { build: "SC2013" },
"dynamic-import": { build: "SC2012" },
"dynamic-import-local": { build: "SC2012" },
"esbuild-require": { build: "SC2013" },
"esm-missing-default": { build: "SC1090" },
"esm-missing-export": { build: "SC1090" },
"fetch-destructure": { run: "SC1031" },
"fs-shims": { run: "SC1101" },
"import-refusal": { build: "SC2020" },
"island-registry": { build: "SC2011" },
"island-timers": {
divergence:
"Timeout handles are numbers (typeof) and lose their ref flag once fired or cleared",
},
"island-web-plumbing": { run: "SC2020" },
"lazy-traps": { build: "SC2013" },
"misc-shims": { run: "SC2020" },
"namespace": { build: "SC1013" },
"node24-async-events": { run: "SC2020" },
"node24-commonjs-metadata": { run: "SC2020" },
"node24-crypto-kdf": { run: "SC2020" },
"node24-deep-equality": { run: "SC2020" },
"node24-diagnostics-async": { run: "SC1090" },
"node24-diagnostics-tracing": { run: "SC1090" },
"node24-env": { run: "SC1090" },
"node24-esm-metadata": { run: "SC1090" },
"node24-event-emitter": { run: "SC2020" },
"node24-file-url-conversion": { build: "SC1090" },
"node24-fs-descriptors": { run: "SC2020" },
"node24-mime": { run: "SC2020" },
"node24-mime-errors": { run: "SC2020" },
"node24-no-error-assertions": { run: "SC2020" },
"node24-parseargs": { run: "SC1090" },
"node24-promise-timers": { run: "SC2020" },
"node24-string-decoder": { run: "SC2020" },
"node24-style-text": { run: "SC2020" },
"node24-system-errors": { run: "SC1090" },
"node24-text-encoding": { run: "SC2020" },
"node24-text-utilities": { run: "SC2020" },
"node24-timers-unref": { run: "SC2020" },
"node24-url-factories": { run: "SC2020" },
"node24-url-mutation": { run: "SC1090" },
"node24-util-adapters": { run: "Object.create with this value is not supported yet" },
"node24-util-types": { run: "SC1090" },
"node24-web-stream-encoding": { run: "SC1090" },
"node24-web-stream-errors": { run: "SC1090" },
"node24-web-stream-lifecycle": { run: "SC1090" },
"node24-web-stream-pipe-abort": { run: "SC2020" },
"node24-web-stream-pipe-errors": { run: "SC1090" },
"node24-web-stream-pipes": { run: "SC1090" },
"node24-web-stream-queues": { run: "SC1090" },
"node24-web-stream-readable-lifecycle": { run: "SC2020" },
"node24-web-stream-readers": { run: "SC1090" },
"node24-web-stream-tee": { run: "SC2020" },
"node24-web-stream-transforms": { run: "SC1090" },
"package-imports": { build: "SC2013" },
"promise-bridge": { build: "SC1090" },
"self-name": { build: "SC2013" },
"stream-shims": { run: "SC2020" },
"util-shims": { run: "SC2020" },
// The package redeclares the var it exported: Node copies the first value.
"var-export-redeclared": { run: "SC1090" },
"web-streams": { run: "SC1090" },
"workspace-copied": { build: "SC2013" },
"workspace-unclean": { build: "SC2013" },
};
/** Per-case timeouts past the default 120 s: the undici cases compile the
* built-in undici implementation's module graph, the zod 3 cases (zod3,
* zod3-adoption, zod3-boundary, zod3-infer) compile zod 3's 3,700-line
* types module, and the zod 4 cases (zod4-boundary, zod4-record,
* hono-zod-validator, hono-zod-default-hook)
* compile zod 4's core, classic API and locales. A sanitized build of any
* of them takes minutes on a CI runner. */
const TIMEOUTS: Record<string, number> = {
"hono-zod-default-hook": 900_000,
"hono-zod-validator": 900_000,
undici5: 600_000,
undici7: 600_000,
"zod4-boundary": 900_000,
"zod4-record": 900_000,
zod3: 600_000,
"zod3-adoption": 600_000,
"zod3-boundary": 600_000,
"zod3-infer": 600_000,
};
interface RunResult {
stdout: Buffer;
stderr: Buffer;
exitCode: number;
}
function programStderr(stderr: Buffer): Buffer {
if (!sanitize) return stderr;
// Linux ASan emits this instrumentation warning at the first fiber swap.
// Latin-1 preserves every other byte, including non-UTF-8 program output.
return Buffer.from(
stderr
.toString("latin1")
.replace(
/^==\d+==WARNING: ASan doesn't fully support makecontext\/swapcontext functions and may produce false positives in some cases!\n/gm,
"",
),
"latin1",
);
}
async function runBinary(cmd: string, args: string[], cwd: string): Promise<RunResult> {
try {
// stdin closes immediately, as in differential.test.ts: a program that
// reads fd 0 to EOF must not hang on an open pipe.
const pending = execFileAsync(cmd, args, { cwd, encoding: "buffer" });
pending.child.stdin?.end();
const { stdout, stderr } = await pending;
return { stdout, stderr, exitCode: 0 };
} catch (err) {
const e = err as { code?: unknown; signal?: unknown; stdout?: Buffer; stderr?: Buffer };
if (!Buffer.isBuffer(e.stdout) || !Buffer.isBuffer(e.stderr)) throw err;
const exitCode = typeof e.code === "number" ? e.code : -1;
if (exitCode === -1 && typeof e.signal !== "string") throw err;
return { stdout: e.stdout, stderr: e.stderr, exitCode };
}
}
type Built = { ok: true; binary: string } | { ok: false; codes: string[]; text: string };
/** Compile once per (entry, lane); the cache key hashes the program and the
* fixture packages so edits to either rebuild. */
async function build(entry: string, shapeOverlay: boolean | undefined): Promise<Built> {
const hash = createHash("sha256");
const inputs = [
entry,
...globSync(join(dirname(entry), "*.{ts,mts,cts,js,mjs,cjs,json}")),
...globSync(join(fixturesRoot, "npm/node_modules/**/*.{js,mjs,cjs,json,d.ts,node}")),
].sort();
for (const f of inputs) hash.update(f).update(readFileSync(f));
const key = hash
.update(sanitize ? "san" : "plain")
.update(shapeOverlay === undefined ? "" : `overlay=${shapeOverlay}`)
.digest("hex")
.slice(0, 16);
const outDir = join(cacheDir, `npm-${key}`);
mkdirSync(outDir, { recursive: true });
const result = await compile(entry, {
outPath: join(outDir, "program"),
outDir,
sanitize,
...(shapeOverlay === undefined ? {} : { shapeOverlay }),
});
if (result.ok) return { ok: true, binary: result.binaryPath };
return {
ok: false,
codes: result.diagnostics.map((d) => d.code),
text: result.diagnostics.map((d) => `${d.code}: ${d.message}`).join("\n"),
};
}
async function expectMatchesNode(c: NpmCase, binary: string): Promise<void> {
const cwd = dirname(c.entry);
for (const argv of c.argvs ?? [[]]) {
const [nodeRes, nativeRes] = await Promise.all([
runBinary(process.execPath, [c.entry, ...argv], cwd),
runBinary(binary, argv, cwd),
]);
const label = argv.join(" ");
if (!nodeRes.stdout.equals(nativeRes.stdout)) {
expect(nativeRes.stdout.toString("utf8"), label).toBe(nodeRes.stdout.toString("utf8"));
expect.unreachable("stdout differed at byte level but not after utf8 decode");
}
expect(nativeRes.exitCode, label).toBe(nodeRes.exitCode);
if (c.compareStderr) expect(programStderr(nativeRes.stderr), label).toEqual(nodeRes.stderr);
}
}
/** Defines the suite. `shapeOverlay` is passed to every compile (unset: the
* default build). With the overlay on, every case must still build and run
* as KNOWN records for the default build, except that a refused build may
* compile, and then matches Node. */
export function defineNpmSuite(options: { label: string; shapeOverlay?: boolean }): void {
const { label, shapeOverlay } = options;
const cases = shardSelect(npmCases(fixturesRoot), (c) => c.name);
test("every KNOWN entry names a case", () => {
const names = new Set(npmCases(fixturesRoot).map((c) => c.name));
expect(Object.keys(KNOWN).filter((name) => !names.has(name))).toEqual([]);
});
describe(`${label} (${cases.length} programs${sanitize ? ", sanitized" : ""}${shardSuffix()})`, () => {
for (const c of cases) {
const known = KNOWN[c.name];
if (known !== undefined && "divergence" in known) {
test.fails(`${c.name} (diverges: ${known.divergence})`, async () => {
const built = await build(c.entry, shapeOverlay);
if (!built.ok) throw new Error(built.text);
await expectMatchesNode(c, built.binary);
}, 120_000);
continue;
}
test(
c.name,
async () => {
const built = await build(c.entry, shapeOverlay);
if (known !== undefined && "build" in known) {
// The overlay may type code the default build refuses: such a
// program must then match Node.
if (shapeOverlay === true && built.ok) {
await expectMatchesNode(c, built.binary);
return;
}
expect(built.ok, "the compile refuses the program").toBe(false);
if (!built.ok) expect(built.codes, built.text).toContain(known.build);
return;
}
if (!built.ok) throw new Error(`npm fixture failed to compile:\n${built.text}`);
if (known !== undefined && "run" in known) {
for (const argv of known.argvs ?? c.argvs ?? [[]]) {
const res = await runBinary(built.binary, argv, dirname(c.entry));
const output = res.stdout.toString("utf8") + res.stderr.toString("utf8");
expect(output, `${argv.join(" ")}: the deferred refusal`).toContain(known.run);
}
if (known.argvs === undefined) return;
const refused = new Set(known.argvs.map((argv) => JSON.stringify(argv)));
const argvs = (c.argvs ?? [[]]).filter((argv) => !refused.has(JSON.stringify(argv)));
await expectMatchesNode({ ...c, argvs }, built.binary);
return;
}
await expectMatchesNode(c, built.binary);
},
TIMEOUTS[c.name] ?? 120_000,
);
}
});
}
tests/harness/npm.test.ts
+3 -260
View File
@@ -1,261 +1,4 @@
/* npm packages compiled natively: every program under tests/fixtures/npm/cases
* (plus the commander fixtures in npm-cases.ts) builds with the default
* static npm compile and runs against Node. A case matches Node (stdout and
* exit status for every argv list, stderr when it opts in with
* `// @stderr`) unless KNOWN records how it differs:
*
* - build: the compile refuses it with this diagnostic code;
* - run: it compiles, and the program's output names this diagnostic code
* (a refusal deferred to the operation that reached it), for every argv
* list or for the listed `argvs`, while the other lists match Node;
* - divergence: it runs but does not match Node yet. These run as expected
* failures, so a fix that makes one match fails here until KNOWN drops it.
*
* The fixture node_modules are committed test data. Binaries compile the
* package sources at build time and read nothing from node_modules at run
* time.
*/
import { execFile } from "node:child_process";
import { createHash } from "node:crypto";
import { globSync, mkdirSync, readFileSync } from "node:fs";
import { dirname, join } from "node:path";
import { promisify } from "node:util";
import { describe, expect, test } from "vitest";
import { compile } from "@scriptc/compiler";
import { npmCases, type NpmCase } from "./npm-cases.js";
import { shardSelect, shardSuffix } from "./shard.js";
/* npm packages compiled natively as built by default (npm-suite.ts). */
import { defineNpmSuite } from "./npm-suite.js";
const execFileAsync = promisify(execFile);
const repoRoot = join(import.meta.dirname, "../..");
const fixturesRoot = join(repoRoot, "tests/fixtures");
const cacheDir = join(repoRoot, "node_modules/.cache/scriptc-tests");
const sanitize = process.env["SCRIPTC_SAN"] === "1";
type Known = { build: string } | { run: string; argvs?: string[][] } | { divergence: string };
const KNOWN: Record<string, Known> = {
"any-chaining": { build: "SC2003" },
"assert-shims": { run: "SC1043" },
"buffer-shims": { run: "SC2020" },
"bytes-exit": { build: "SC2004" },
"cjs-esm-named": { run: "SC1090" },
"cjs-lexer-aligned": { run: "SC1090" },
"commander-calc": { run: "SC1090", argvs: [["--help"]] },
"commander-calc-async": { run: "SC1090", argvs: [["--help"]] },
"create-require": { run: "SC1090" },
"dotfolder": { run: "SC1090" },
"dual-package-kinds": { build: "SC2013" },
"dynamic-import": { build: "SC2012" },
"dynamic-import-local": { build: "SC2012" },
"esbuild-require": { build: "SC2013" },
"esm-missing-default": { build: "SC1090" },
"esm-missing-export": { build: "SC1090" },
"fetch-destructure": { run: "SC1031" },
"fs-shims": { run: "SC1101" },
"import-refusal": { build: "SC2020" },
"island-registry": { build: "SC2011" },
"island-timers": {
divergence:
"Timeout handles are numbers (typeof) and lose their ref flag once fired or cleared",
},
"island-web-plumbing": { run: "SC2020" },
"lazy-traps": { build: "SC2013" },
"misc-shims": { run: "SC2020" },
"namespace": { build: "SC1013" },
"node24-async-events": { run: "SC2020" },
"node24-commonjs-metadata": { run: "SC2020" },
"node24-crypto-kdf": { run: "SC2020" },
"node24-deep-equality": { run: "SC2020" },
"node24-diagnostics-async": { run: "SC1090" },
"node24-diagnostics-tracing": { run: "SC1090" },
"node24-env": { run: "SC1090" },
"node24-esm-metadata": { run: "SC1090" },
"node24-event-emitter": { run: "SC2020" },
"node24-file-url-conversion": { build: "SC1090" },
"node24-fs-descriptors": { run: "SC2020" },
"node24-mime": { run: "SC2020" },
"node24-mime-errors": { run: "SC2020" },
"node24-no-error-assertions": { run: "SC2020" },
"node24-parseargs": { run: "SC1090" },
"node24-promise-timers": { run: "SC2020" },
"node24-string-decoder": { run: "SC2020" },
"node24-style-text": { run: "SC2020" },
"node24-system-errors": { run: "SC1090" },
"node24-text-encoding": { run: "SC2020" },
"node24-text-utilities": { run: "SC2020" },
"node24-timers-unref": { run: "SC2020" },
"node24-url-factories": { run: "SC2020" },
"node24-url-mutation": { run: "SC1090" },
"node24-util-adapters": { run: "Object.create with this value is not supported yet" },
"node24-util-types": { run: "SC1090" },
"node24-web-stream-encoding": { run: "SC1090" },
"node24-web-stream-errors": { run: "SC1090" },
"node24-web-stream-lifecycle": { run: "SC1090" },
"node24-web-stream-pipe-abort": { run: "SC2020" },
"node24-web-stream-pipe-errors": { run: "SC1090" },
"node24-web-stream-pipes": { run: "SC1090" },
"node24-web-stream-queues": { run: "SC1090" },
"node24-web-stream-readable-lifecycle": { run: "SC2020" },
"node24-web-stream-readers": { run: "SC1090" },
"node24-web-stream-tee": { run: "SC2020" },
"node24-web-stream-transforms": { run: "SC1090" },
"package-imports": { build: "SC2013" },
"promise-bridge": { build: "SC1090" },
"self-name": { build: "SC2013" },
"stream-shims": { run: "SC2020" },
"util-shims": { run: "SC2020" },
// The package redeclares the var it exported: Node copies the first value.
"var-export-redeclared": { run: "SC1090" },
"web-streams": { run: "SC1090" },
"workspace-copied": { build: "SC2013" },
"workspace-unclean": { build: "SC2013" },
};
const cases = shardSelect(npmCases(fixturesRoot), (c) => c.name);
/** Per-case timeouts past the default 120 s: the undici cases compile the
* built-in undici implementation's module graph, the zod 3 cases (zod3,
* zod3-adoption, zod3-boundary, zod3-infer) compile zod 3's 3,700-line
* types module, and the zod 4 cases (zod4-boundary, zod4-record,
* hono-zod-validator, hono-zod-default-hook)
* compile zod 4's core, classic API and locales. A sanitized build of any
* of them takes minutes on a CI runner. */
const TIMEOUTS: Record<string, number> = {
"hono-zod-default-hook": 900_000,
"hono-zod-validator": 900_000,
undici5: 600_000,
undici7: 600_000,
"zod4-boundary": 900_000,
"zod4-record": 900_000,
zod3: 600_000,
"zod3-adoption": 600_000,
"zod3-boundary": 600_000,
"zod3-infer": 600_000,
};
interface RunResult {
stdout: Buffer;
stderr: Buffer;
exitCode: number;
}
function programStderr(stderr: Buffer): Buffer {
if (!sanitize) return stderr;
// Linux ASan emits this instrumentation warning at the first fiber swap.
// Latin-1 preserves every other byte, including non-UTF-8 program output.
return Buffer.from(
stderr
.toString("latin1")
.replace(
/^==\d+==WARNING: ASan doesn't fully support makecontext\/swapcontext functions and may produce false positives in some cases!\n/gm,
"",
),
"latin1",
);
}
async function runBinary(cmd: string, args: string[], cwd: string): Promise<RunResult> {
try {
// stdin closes immediately, as in differential.test.ts: a program that
// reads fd 0 to EOF must not hang on an open pipe.
const pending = execFileAsync(cmd, args, { cwd, encoding: "buffer" });
pending.child.stdin?.end();
const { stdout, stderr } = await pending;
return { stdout, stderr, exitCode: 0 };
} catch (err) {
const e = err as { code?: unknown; signal?: unknown; stdout?: Buffer; stderr?: Buffer };
if (!Buffer.isBuffer(e.stdout) || !Buffer.isBuffer(e.stderr)) throw err;
const exitCode = typeof e.code === "number" ? e.code : -1;
if (exitCode === -1 && typeof e.signal !== "string") throw err;
return { stdout: e.stdout, stderr: e.stderr, exitCode };
}
}
type Built = { ok: true; binary: string } | { ok: false; codes: string[]; text: string };
/** Compile once per (entry, lane); the cache key hashes the program and the
* fixture packages so edits to either rebuild. */
async function build(entry: string): Promise<Built> {
const hash = createHash("sha256");
const inputs = [
entry,
...globSync(join(dirname(entry), "*.{ts,mts,cts,js,mjs,cjs,json}")),
...globSync(join(fixturesRoot, "npm/node_modules/**/*.{js,mjs,cjs,json,d.ts,node}")),
].sort();
for (const f of inputs) hash.update(f).update(readFileSync(f));
const key = hash
.update(sanitize ? "san" : "plain")
.digest("hex")
.slice(0, 16);
const outDir = join(cacheDir, `npm-${key}`);
mkdirSync(outDir, { recursive: true });
const result = await compile(entry, { outPath: join(outDir, "program"), outDir, sanitize });
if (result.ok) return { ok: true, binary: result.binaryPath };
return {
ok: false,
codes: result.diagnostics.map((d) => d.code),
text: result.diagnostics.map((d) => `${d.code}: ${d.message}`).join("\n"),
};
}
async function expectMatchesNode(c: NpmCase, binary: string): Promise<void> {
const cwd = dirname(c.entry);
for (const argv of c.argvs ?? [[]]) {
const [nodeRes, nativeRes] = await Promise.all([
runBinary(process.execPath, [c.entry, ...argv], cwd),
runBinary(binary, argv, cwd),
]);
const label = argv.join(" ");
if (!nodeRes.stdout.equals(nativeRes.stdout)) {
expect(nativeRes.stdout.toString("utf8"), label).toBe(nodeRes.stdout.toString("utf8"));
expect.unreachable("stdout differed at byte level but not after utf8 decode");
}
expect(nativeRes.exitCode, label).toBe(nodeRes.exitCode);
if (c.compareStderr) expect(programStderr(nativeRes.stderr), label).toEqual(nodeRes.stderr);
}
}
test("every KNOWN entry names a case", () => {
const names = new Set(npmCases(fixturesRoot).map((c) => c.name));
expect(Object.keys(KNOWN).filter((name) => !names.has(name))).toEqual([]);
});
describe(`npm native (${cases.length} programs${sanitize ? ", sanitized" : ""}${shardSuffix()})`, () => {
for (const c of cases) {
const known = KNOWN[c.name];
if (known !== undefined && "divergence" in known) {
test.fails(`${c.name} (diverges: ${known.divergence})`, async () => {
const built = await build(c.entry);
if (!built.ok) throw new Error(built.text);
await expectMatchesNode(c, built.binary);
}, 120_000);
continue;
}
test(
c.name,
async () => {
const built = await build(c.entry);
if (known !== undefined && "build" in known) {
expect(built.ok, "the compile refuses the program").toBe(false);
if (!built.ok) expect(built.codes, built.text).toContain(known.build);
return;
}
if (!built.ok) throw new Error(`npm fixture failed to compile:\n${built.text}`);
if (known !== undefined && "run" in known) {
for (const argv of known.argvs ?? c.argvs ?? [[]]) {
const res = await runBinary(built.binary, argv, dirname(c.entry));
const output = res.stdout.toString("utf8") + res.stderr.toString("utf8");
expect(output, `${argv.join(" ")}: the deferred refusal`).toContain(known.run);
}
if (known.argvs === undefined) return;
const refused = new Set(known.argvs.map((argv) => JSON.stringify(argv)));
const argvs = (c.argvs ?? [[]]).filter((argv) => !refused.has(JSON.stringify(argv)));
await expectMatchesNode({ ...c, argvs }, built.binary);
return;
}
await expectMatchesNode(c, built.binary);
},
TIMEOUTS[c.name] ?? 120_000,
);
}
});
defineNpmSuite({ label: "npm native" });