Lower to the same IR whatever order reachable bodies lower in (#1734)

This commit is contained in:
Chris Tate
2026-10-21 21:12:08 -05:00
committed by GitHub
12 changed files with 829 additions and 30 deletions
+1
View File
@@ -20,6 +20,7 @@
"test:test262": "tsx tests/test262/run.ts",
"test:test262:fetch": "node tests/test262/fetch.mjs",
"test:cache-identity": "node tests/harness/cache-identity.mjs",
"test:lowering-order": "node scripts/lowering-order.mjs --fail-on content",
"test:sandbox": "node scripts/sandbox-test.mjs",
"test:sandbox:image": "node scripts/sandbox-image.mjs",
"test:watch": "vitest",
@@ -5,6 +5,7 @@ import {
DYN,
type IrExpr,
type IrFunction,
type IrStmt,
type IrType,
type SrcLoc,
} from "../../ir/ir.js";
@@ -59,28 +60,48 @@ export class CaughtValueDispatch {
loc,
};
lowerer.liftedFns.push(this.helper);
lowerer.shareHelper(this.helper);
changed = true;
}
const helper = this.helper;
let added = false;
for (const className of this.thrownClasses) {
if (this.convertedClasses.has(className)) continue;
this.convertedClasses.add(className);
added = true;
}
if (added) {
// A subclass is tested before its ancestors, and classes of one depth
// by name, so the tests do not depend on the order the census met them.
const depth = (className: string): number => {
let count = 0;
for (let info = lowerer.classes.get(className)?.base; info; info = info.base) count++;
return count;
};
const ordered = [...this.convertedClasses].sort(
(left, right) => depth(right) - depth(left) || (left < right ? -1 : left > right ? 1 : 0),
);
const loc = helper.loc;
const value = varRef("caught", CAUGHT, loc);
const type: IrType = { kind: "object", className };
helper.body.unshift({
kind: "if",
cond: { kind: "caughtTest", value, test: "instanceof", className, type: BOOL, loc },
then: [
{
kind: "return",
value: lowerer.coerceToExpected({ kind: "caughtNarrow", value, type, loc }, DYN),
helper.body = [
...ordered.map((className): IrStmt => {
const type: IrType = { kind: "object", className };
return {
kind: "if",
cond: { kind: "caughtTest", value, test: "instanceof", className, type: BOOL, loc },
then: [
{
kind: "return",
value: lowerer.coerceToExpected({ kind: "caughtNarrow", value, type, loc }, DYN),
loc,
},
],
else_: null,
loc,
},
],
else_: null,
loc,
});
};
}),
helper.body[helper.body.length - 1]!,
];
changed = true;
}
for (const fn of consumers) {
@@ -33,6 +33,7 @@ function context() {
classes: new Map(),
liftedFns,
liftedFnNamed: (name: string | undefined) => liftedFns.find((fn) => fn.name === name),
shareHelper: () => {},
subclassLinks: 0,
constructedClassCount: () => -1,
diags: [],
@@ -281,6 +281,7 @@ export function classInstanceOf(
classValue?: IrExpr,
): IrExpr {
const name = `%dyn.class.${classValue ? "valueInstanceof" : "instanceof"}:${info.def.name}`;
lowerer.shareHelper(name);
const params = [
{ localId: "value", name: "value", type: DYN },
...(classValue ? [{ localId: "class", name: "class", type: classValue.type }] : []),
@@ -406,8 +407,16 @@ function withTypedReceiver(value: IrExpr, receiver: IrExpr): IrExpr {
});
}
/** Re-adds the entries of `set` after the first `from` sorted. */
function sortAdded(set: Set<string>, from: number): void {
const added = [...set].slice(from).sort();
for (const name of added) set.delete(name);
for (const name of added) set.add(name);
}
export function classPropertiesHelper(lowerer: Lowerer, loc: SrcLoc, readOnly = false): IrFunction {
const name = readOnly ? READ_PROPERTIES : "%dyn.class.properties";
lowerer.shareHelper(name);
const existing = lowerer.liftedFnNamed(name);
if (existing) return existing;
const helper: IrFunction = {
@@ -447,6 +456,8 @@ export function classForInHasKey(
loc: SrcLoc,
): IrExpr {
const name = "%dyn.class.forInHasKey";
lowerer.shareHelper(name);
lowerer.shareHelper("%dyn.class.forInBagHas");
if (!lowerer.liftedFns.some((fn) => fn.name === name)) {
// The read-only table: a recheck never creates an instance's table, which
// is undefined while the instance has no own dynamic properties.
@@ -825,6 +836,8 @@ export class ClassDynamicDispatch {
else if (type.kind === "func") discover(type.ret);
else if (type.kind === "promise") discover(type.inner);
};
const boxedBefore = this.boxed.size;
const boxedConstructorsBefore = this.boxedConstructors.size;
for (const fn of functions)
everyStmtList(fn.body, {
stmt: (statement) => {
@@ -919,6 +932,10 @@ export class ClassDynamicDispatch {
return true;
},
});
// The classes this pass found join the tables by name, not in the order
// the scan met them.
sortAdded(this.boxed, boxedBefore);
sortAdded(this.boxedConstructors, boxedConstructorsBefore);
changed = this.caughtDispatch.dynamicPayloads(lowerer, caughtSites, this.boxed) || changed;
if (this.boxed.size === 0 && rewrite.size === 0) return changed;
const converting = this.convertingHints(lowerer);
@@ -4264,6 +4281,8 @@ export class ClassDynamicDispatch {
* prototype-backed bag is produced for this read only. A published object
* cannot take the store, and Node's reads never write. */
finalize(lowerer?: Lowerer): void {
// Each table serves every site that reaches it.
if (lowerer) for (const fn of this.generated) lowerer.shareHelper(fn);
for (const { helper, initialize, orEmpty } of this.readPropertyBags) {
const absent = helper.body[0];
if (isEmptyBag(initialize.value) && !orEmpty) helper.body.shift();
@@ -122,6 +122,7 @@ export function classPrototypeData(
const globalId = `%g.${name}`;
const readyId = `${globalId}.ready`;
info.def.prototypeDataHelper = name;
lowerer.shareHelper(name);
const className = info.def.jsName ?? info.def.name;
lowerer.globalsList.push({
id: globalId,
@@ -24,6 +24,7 @@ export function errorPropertyRead(
): IrExpr {
const name = `%error.${property}.read`,
loc = receiver.loc;
lowerer.shareHelper(name);
if (!lowerer.liftedFns.some((fn) => fn.name === name))
lowerer.liftedFns.push({
name,
@@ -50,6 +51,7 @@ export function errorPropertyWrite(
): IrStmt {
const name = `%error.${property}.write`,
loc = receiver.loc;
lowerer.shareHelper(name);
if (!lowerer.liftedFns.some((fn) => fn.name === name))
lowerer.liftedFns.push({
name,
@@ -392,6 +394,7 @@ export function errorToStringMethod(
receiver: IrExpr,
): Extract<IrExpr, { kind: "call" }> {
const loc = receiver.loc;
lowerer.shareHelper(helperName);
if (!lowerer.liftedFns.some((fn) => fn.name === helperName)) {
lowerer.liftedFns.push({
name: helperName,
@@ -415,6 +418,7 @@ export function errorToStringCall(lowerer: Lowerer, receiver: IrExpr): IrExpr {
const loc = receiver.loc;
const method = errorToStringMethod(lowerer, receiver);
const name = "%error.toString.virtual";
lowerer.shareHelper(name);
if (!lowerer.liftedFns.some((fn) => fn.name === name)) {
const value = varRef("error.0", errorType, loc);
lowerer.liftedFns.push({
@@ -68,8 +68,10 @@ export class GeneratorDynamicDispatch {
this.properties.set(method, { fn, types: new Set() });
this.generated.add(fn);
lowerer.liftedFns.push(fn);
lowerer.shareHelper(fn);
// The runtime's own helper functions answer the helper methods.
if (HELPER_METHODS.has(method)) return fn;
lowerer.shareHelper(`%dyn.generator.method.${method}`);
lowerer.liftedFns.push({
name: `%dyn.generator.method.${method}`,
params: [{ localId: "arg", name: "arg", type: DYN }],
@@ -209,6 +211,7 @@ export class GeneratorDynamicDispatch {
this.dispatches.set(key, dispatch);
this.generated.add(helper);
lowerer.liftedFns.push(helper);
lowerer.shareHelper(helper);
changed = true;
}
return {
@@ -2128,8 +2128,12 @@ function ensureEmitSpec(lowerer: Lowerer, info: ClassInfo, event: string, tuple:
params: tuple.map((t) => ({ type: t, mode: "required" as const })),
ret: BOOL,
});
if (info.def.methods) info.def.methods.push(mName);
else info.def.methods = [mName];
// Specializations sit in the method list sorted by event name, so the
// class table does not depend on which body asked for one first.
const methods = (info.def.methods ??= []);
const after = methods.findIndex((m) => m.startsWith("emit:") && m > mName);
if (after < 0) methods.push(mName);
else methods.splice(after, 0, mName);
}
const ordinal = lowerer.emitSpecQueue.filter((q) => q.info === info).length;
lowerer.emitSpecQueue.push({ info, event, tuple, ordinal });
@@ -1048,7 +1048,8 @@ export function moduleArtifacts(
}
const reachable = lowerer.reachableForArtifacts ?? lowerer.reachable;
return {
classes: [...lowerer.classes.values()]
classes: lowerer
.classesInTableOrder()
.map((c) => c.def)
.filter((def) => classNames.has(def.name))
.map((def): IrClassDef => {
@@ -959,6 +959,33 @@ function directExternalTypeSpecifiersByFile(
return out;
}
/** Debugging aid: SCRIPTC_LOWER_SHUFFLE_SEED=<n> lowers each wave of
* reachable bodies in an order drawn from that seed instead of discovery
* order. Output assembly is ordered independently of it, so any IR
* difference between seeds is state one body's lowering leaves for another
* (scripts/lowering-order.mjs). */
function lowerShuffle(): ((wave: string[]) => void) | null {
const raw = process.env["SCRIPTC_LOWER_SHUFFLE_SEED"];
if (raw === undefined || raw === "") return null;
// mulberry32
let state = Number.parseInt(raw, 10) >>> 0;
const next = (): number => {
state = (state + 0x6d2b79f5) >>> 0;
let t = state;
t = Math.imul(t ^ (t >>> 15), t | 1);
t ^= t + Math.imul(t ^ (t >>> 7), t | 61);
return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
};
return (wave) => {
for (let index = wave.length - 1; index > 0; index--) {
const other = Math.floor(next() * (index + 1));
const swap = wave[index]!;
wave[index] = wave[other]!;
wave[other] = swap;
}
};
}
/** Build lowering runs as a reachability worklist over the ts.Program:
*
* 1. REACHABLE EMIT — a worklist computes the set of reachable bodies.
@@ -1841,6 +1868,13 @@ function monomorphizedImplicit(
return true;
}
/** `fns` sorted by name; functions of one name keep their order. */
function byName(fns: readonly IrFunction[]): IrFunction[] {
return [...fns].sort((left, right) =>
left.name < right.name ? -1 : left.name > right.name ? 1 : 0,
);
}
export class Lowerer {
readonly frontendServices: FrontendServices | undefined;
readonly checker: ts.TypeChecker;
@@ -2027,6 +2061,40 @@ export class Lowerer {
this.genericClassInstances.splice(from, tail.length, ...tail);
}
/** The classes in table order. Generic instances are added as bodies
* demand them, so they keep the slots they hold and fill them in the
* order of their settled demand ranks. */
classesInTableOrder(): ClassInfo[] {
const all = [...this.classes.values()];
const slots = all.flatMap((info, index) =>
this.genericClassDemandPriority.has(info) ? [index] : [],
);
const ranked = slots
.map((index) => all[index]!)
.sort((left, right) => {
const a = this.genericClassDemandPriority.get(left)!.rank;
const b = this.genericClassDemandPriority.get(right)!.rank;
return a[0]! - b[0]! || a[1]! - b[1]!;
});
slots.forEach((slot, index) => (all[slot] = ranked[index]!));
return all;
}
/** The globals in table order. A class's prototype data globals are
* added when a body first reads that prototype, so they keep the slots
* they hold and fill them in id order. */
private globalsInTableOrder(): IrGlobal[] {
const all = [...this.globalsList];
const prototypeData = (global: IrGlobal): boolean =>
global.id.startsWith("%g.%prototype.data.");
const slots = all.flatMap((global, index) => (prototypeData(global) ? [index] : []));
const sorted = slots
.map((index) => all[index]!)
.sort((left, right) => (left.id < right.id ? -1 : left.id > right.id ? 1 : 0));
slots.forEach((slot, index) => (all[slot] = sorted[index]!));
return all;
}
private settleGenericDemandPriorities(): void {
const functionQueue: GenericInstance[] = [];
const classQueue: ClassInfo[] = [];
@@ -3187,6 +3255,60 @@ export class Lowerer {
constructedClassCount(): number {
return this.constructedClasses?.size ?? -1;
}
/** Helpers that every site asking for them shares (shareHelper). */
private readonly sharedHelpers = new Set<IrFunction>();
private readonly sharedHelperNames = new Set<string>();
/** Marks a helper, or the lifted helpers of a name, as shared by every
* site that asks for it. A shared helper is built for whichever site asks
* first, so once lowering finishes its locations are replaced by one that
* names no site. */
shareHelper(helper: IrFunction | string): void {
if (typeof helper === "string") this.sharedHelperNames.add(helper);
else this.sharedHelpers.add(helper);
}
/** Gives each shared helper the location of the entry file's start: the
* helpers marked by shareHelper, and those interned by key in the helper
* tables. */
private settleSharedHelperLocations(): void {
const loc: SrcLoc = { file: this.entry.fileName, start: 0, end: 0 };
const interned = new Set<string>([...this.sharedHelperNames, ...this.classMethodValueHelpers]);
for (const table of [
this.equalityHelpers,
this.arrHofHelpers,
this.unionCallHelpers,
this.valueHelpers,
this.assertHelpers,
this.inspectHelpers,
this.iterDrainHelpers,
this.mapHofHelpers,
this.setHofHelpers,
this.spHofHelpers,
this.primitiveCtorFns,
this.builtinCallableValueFns,
this.regionRunHelpers,
this.internTableHelpers,
this.coercions.retags,
this.coercions.narrows,
])
for (const name of table.values()) interned.add(name);
for (const fn of [...this.liftedFns, ...this.implicitFns])
if (interned.has(fn.name)) this.sharedHelpers.add(fn);
const seen = new Set<object>();
const visit = (node: unknown): void => {
if (typeof node !== "object" || node === null || seen.has(node)) return;
seen.add(node);
if (Array.isArray(node)) {
for (const item of node) visit(item);
return;
}
const record = node as Record<string, unknown>;
for (const key of Object.keys(record)) {
if (key === "loc") record[key] = loc;
else if (key !== "type") visit(record[key]);
}
};
for (const fn of this.sharedHelpers) visit(fn);
}
/** The first lifted function named `name`, the one
* `liftedFns.find((fn) => fn.name === name)` returns. */
liftedFnNamed(name: string | undefined): IrFunction | undefined {
@@ -7279,6 +7401,7 @@ export class Lowerer {
// Lambdas lifted while lowering any of the above (plus synthetic
// array-HOF loop functions, which ride the same list), and the
// implicit-any instances lowered eagerly at their first call sites.
this.settleSharedHelperLocations();
functions.push(...this.liftedFns);
functions.push(...this.implicitFns);
functions.push(...runtimeClassConstructs(this, functions));
@@ -7412,7 +7535,7 @@ export class Lowerer {
classes: artifacts.classes,
records: artifacts.records,
unions: artifacts.unions,
globals: this.globalsList,
globals: this.globalsInTableOrder(),
entry: ENTRY_NAME,
...(this.usesWorkers ? { workers: true } : {}),
...(this.usesLifetimeRegions ? { lifetimeRegions: true as const } : {}),
@@ -7859,6 +7982,15 @@ export class Lowerer {
// exactly like generic instances.
while (specLowered < this.emitSpecQueue.length) {
const waveEnd = this.emitSpecQueue.length;
// A wave's specializations lower by class and event, not in the
// order bodies asked for them.
const wave = this.emitSpecQueue.slice(specLowered, waveEnd);
const specKey = ({ info, event }: EmitSpecRequest): string =>
`${info.def.name}\0${event}`;
wave.sort((left, right) =>
specKey(left) < specKey(right) ? -1 : specKey(left) > specKey(right) ? 1 : 0,
);
this.emitSpecQueue.splice(specLowered, wave.length, ...wave);
this.checker.prefetchRoots(
this.emitSpecQueue
.slice(specLowered, waveEnd)
@@ -7908,9 +8040,11 @@ export class Lowerer {
// seed the worklist beside the init bodies. Unknown names are inert
// (the export-map resolution reports them as SC4002 later).
for (const root of extraRoots ?? []) this.onEdge?.(root);
const shuffle = lowerShuffle();
const drainUnits = (): void => {
while (queue.length > 0) {
const wave = queue.splice(0);
shuffle?.(wave);
this.checker.prefetchRoots(wave.flatMap((name) => units.get(name)!.roots));
for (const name of wave) {
// A body-level poison outside the per-statement catches (a fenced
@@ -7940,21 +8074,28 @@ export class Lowerer {
// queue above is the only portion whose discovery order differed from
// historical emit order.
const classDispatch = new ClassDynamicDispatch(extraRoots !== undefined);
// Units in declaration order, so the passes below meet functions, and the
// classes they collect, in an order that does not depend on lowering order.
const unitsInOrder = (): IrFunction[] =>
[...loweredUnits]
.sort(([left], [right]) => units.get(left)!.order - units.get(right)!.order)
.map(([, fn]) => fn);
for (;;) {
drainInstances();
const declared = unitsInOrder();
const initializerChanged = this.deferredModuleInitializers.process(this, [
...loweredUnits.values(),
...declared,
...initFunctions,
...instanceFunctions,
...this.liftedFns,
...this.implicitFns,
...byName(this.liftedFns),
...byName(this.implicitFns),
]);
const dispatchChanged = classDispatch.process(this, [
...loweredUnits.values(),
...declared,
...initFunctions,
...instanceFunctions,
...this.liftedFns,
...this.implicitFns,
...byName(this.liftedFns),
...byName(this.implicitFns),
]);
if (queue.length === 0 && !dispatchChanged && !initializerChanged) break;
drainUnits();
@@ -7962,9 +8103,7 @@ export class Lowerer {
this.restoreGenericClassInstanceOrder(clsInstLowered);
}
classDispatch.finalize(this);
const orderedUnits = [...loweredUnits]
.sort(([left], [right]) => units.get(left)!.order - units.get(right)!.order)
.map(([, fn]) => fn);
const orderedUnits = unitsInOrder();
this.settleGenericDemandPriorities();
this.shapes.settleDeclaredOrderPriorities();
for (const finalize of this.shapeOrderMetadataFinalizers) finalize();
@@ -7977,13 +8116,16 @@ export class Lowerer {
]);
observeRecordLiteralRebuilds(null);
for (const finalize of this.shapeOrderHelperFinalizers) finalize();
this.settleSharedHelperLocations();
// Lifted and implicit functions are created in lowering order; they
// are assembled by name.
const functions = [
...orderedUnits,
...initFunctions,
this.buildMain(),
...instanceFunctions,
...this.liftedFns,
...this.implicitFns,
...byName(this.liftedFns),
...byName(this.implicitFns),
];
functions.push(...runtimeClassConstructs(this, functions));
if (this.regexGroupResults) for (const fn of functions) foldRegexSpanReads(this, fn);
@@ -27,8 +27,9 @@ test("util.format %f lowers a boolean argument once", async () => {
const helpers = mod.functions
.map((f) => f.name)
.filter((name) => name.startsWith("%arr."))
.map((name) => name.replace(/\.\d+$/, ""));
expect(helpers).toEqual(["%arr.some", "%arr.every"]);
.map((name) => name.replace(/\.\d+$/, ""))
.sort();
expect(helpers).toEqual(["%arr.every", "%arr.some"]);
} finally {
await rm(dir, { recursive: true, force: true });
}
+601
View File
@@ -0,0 +1,601 @@
#!/usr/bin/env node
/* Finds lowering state whose result depends on the order reachable bodies
* lower in.
*
* Each program is lowered to IR in discovery order and with every seed of
* SCRIPTC_LOWER_SHUFFLE_SEED (lowerer.ts), and the IR is compared function
* by function. Output assembly does not depend on the order, so every
* difference is state one body's lowering leaves for another. The
* discovery-order run also audits the lowerer's Map and Set fields: a key
* that two bodies touch, at least one of them writing it, makes the field
* order-sensitive for both bodies. A function whose IR differs between
* orders is attributed to the order-sensitive fields of the body that
* produced it.
*
* node scripts/lowering-order.mjs [--stride 7] [--offset 0] [--limit N]
* [--seeds 1,2,3] [--jobs 8] [--entry <file.ts>]... [--out <dir>]
* [--fail-on names|content]
*
* Needs a built compiler (pnpm build). Writes <out>/results.json and prints
* a ranking of fields by the programs whose IR they change. With --fail-on,
* it exits with status 1 when some program's IR differs between orders:
* in any way (names), or beyond names and ids (content). */
import { spawn } from "node:child_process";
import { createHash } from "node:crypto";
import { mkdirSync, readFileSync, readdirSync, writeFileSync } from "node:fs";
import { cpus, tmpdir } from "node:os";
import { basename, join, resolve } from "node:path";
import { fileURLToPath, pathToFileURL } from "node:url";
import { parseArgs } from "node:util";
const root = fileURLToPath(new URL("..", import.meta.url));
const dist = join(root, "packages/compiler/dist");
if (process.argv[2] === "--child") {
const [, , , entry, irPath, seed, auditPath, snapshotPath] = process.argv;
if (seed !== "-") process.env["SCRIPTC_LOWER_SHUFFLE_SEED"] = seed;
const module = (path) => import(pathToFileURL(join(dist, path)).href);
const audit =
auditPath === "-"
? null
: installAudit(
await module("frontend/lowering/lowerer.js"),
await module("frontend/lowering/class-dynamic-dispatch.js"),
await module("frontend/lowering/deferred-module-initializers.js"),
);
const snapshots = installSnapshot(await module("frontend/lowering/lowerer.js"));
const { compile } = await import(pathToFileURL(join(dist, "index.js")).href);
const outDir = join(tmpdir(), `scriptc-lowering-order-${process.pid}`);
const result = await compile(entry, {
outDir,
outPath: irPath,
backend: "llvm",
outputKind: "ir",
});
if (audit !== null) writeFileSync(auditPath, JSON.stringify(audit.report()));
writeFileSync(snapshotPath, JSON.stringify(snapshots.at(-1) ?? {}));
process.exit(result.ok ? 0 : 3);
}
/** Records the contents of the lowerer's Map and Set fields once reachable
* emit finishes, under keys that do not depend on the process: syntax
* nodes by file and span, symbols by name and declaration, IR values by id
* or name. Values are digested to a bounded depth. */
function installSnapshot({ Lowerer }) {
const snapshots = [];
const nodeKey = (node) => {
try {
return `node:${node.getSourceFile().fileName}:${node.kind}:${node.pos}:${node.end}`;
} catch {
return `node:?:${node.kind}:${node.pos}:${node.end}`;
}
};
const isNode = (value) =>
typeof value.kind === "number" &&
typeof value.pos === "number" &&
typeof value.end === "number" &&
typeof value.getSourceFile === "function";
const isSymbol = (value) =>
typeof value.escapedName === "string" && Array.isArray(value.declarationHandles);
const symbolKey = (symbol) => {
let declaration = "";
try {
const first = symbol.declarations[0];
if (first !== undefined) declaration = nodeKey(first);
} catch {}
return `symbol:${symbol.escapedName}:${declaration}`;
};
const key = (value) => {
if (value === null || (typeof value !== "object" && typeof value !== "function"))
return `${typeof value}:${String(value)}`;
if (isNode(value)) return nodeKey(value);
if (isSymbol(value)) return symbolKey(value);
for (const field of ["name", "id", "localId"])
if (typeof value[field] === "string" || typeof value[field] === "number")
return `${value.constructor?.name}:${field}:${value[field]}`;
if (value.def && typeof value.def.name === "string")
return `${value.constructor?.name}:def:${value.def.name}`;
return null;
};
const digest = (value, depth) => {
if (value === null || value === undefined || typeof value !== "object")
return typeof value === "function" ? "fn" : (JSON.stringify(value) ?? "undefined");
const stable = key(value);
if (depth === 0) return stable ?? `<${value.constructor?.name}>`;
if (isNode(value) || isSymbol(value)) return stable;
if (Array.isArray(value)) return `[${value.map((item) => digest(item, depth - 1)).join(",")}]`;
if (value instanceof Map)
return `Map{${[...value]
.map(([k, v]) => `${key(k) ?? "?"}=>${digest(v, depth - 1)}`)
.sort()
.join(",")}}`;
if (value instanceof Set)
return `Set{${[...value]
.map((item) => key(item) ?? digest(item, depth - 1))
.sort()
.join(",")}}`;
const fields = Object.keys(value).sort();
return `${value.constructor?.name ?? "Object"}{${fields.map((field) => `${field}:${digest(value[field], depth - 1)}`).join(",")}}`;
};
const emitReachable = Lowerer.prototype.emitReachable;
Lowerer.prototype.emitReachable = function (...args) {
const result = emitReachable.apply(this, args);
const snapshot = {};
for (const [field, value] of Object.entries(this)) {
if (!(value instanceof Map) && !(value instanceof Set)) continue;
const entries = {};
const unkeyed = [];
for (const item of value instanceof Map ? value : [...value].map((v) => [v, true])) {
const [k, v] = item;
const stable = key(k);
const text = digest(v, 3);
if (stable === null) unkeyed.push(text);
else entries[stable] = text;
}
snapshot[field] = { entries, unkeyed: unkeyed.sort() };
}
snapshots.push(snapshot);
return result;
};
return snapshots;
}
/** Wraps the lowering methods that produce bodies and the lowerer's Map and
* Set fields; records which bodies touch which keys. */
function installAudit({ Lowerer }, { ClassDynamicDispatch }, { DeferredModuleInitializers }) {
const proto = Lowerer.prototype;
const tokens = [];
const stack = [];
const fields = new Map();
const objectIds = new Map();
const keyId = (key) => {
if (key === null || (typeof key !== "object" && typeof key !== "function"))
return `${typeof key}:${String(key)}`;
let id = objectIds.get(key);
if (id === undefined) objectIds.set(key, (id = objectIds.size));
return `object:${id}`;
};
const current = () => (stack.length === 0 ? null : stack[stack.length - 1]);
const touch = (field, key, write) => {
const token = current();
if (token === null) return;
let keys = fields.get(field);
if (keys === undefined) fields.set(field, (keys = new Map()));
const id = key === ITERATE ? "*" : keyId(key);
let entry = keys.get(id);
if (entry === undefined) keys.set(id, (entry = { readers: new Set(), writers: new Set() }));
(write ? entry.writers : entry.readers).add(token.id);
};
const ITERATE = Symbol("iterate");
const instrument = (field, value) => {
const read = (key) => touch(field, key, false);
const write = (key) => {
touch(field, key, true);
// A write changes what every iteration of the field observes.
touch(field, ITERATE, true);
};
const iterate = () => touch(field, ITERATE, false);
const base = value instanceof Map ? Map.prototype : Set.prototype;
for (const name of ["get", "has"]) {
if (!(name in base)) continue;
const original = base[name];
value[name] = function (key) {
read(key);
return original.call(this, key);
};
}
for (const name of ["set", "add", "delete"]) {
if (!(name in base)) continue;
const original = base[name];
value[name] = function (key, ...rest) {
write(key);
return original.call(this, key, ...rest);
};
}
for (const name of ["forEach", "keys", "values", "entries", Symbol.iterator]) {
const original = base[name];
value[name] = function (...args) {
iterate();
return original.apply(this, args);
};
}
const size = Object.getOwnPropertyDescriptor(base, "size").get;
Object.defineProperty(value, "size", {
get() {
iterate();
return size.call(this);
},
});
};
const emitReachable = proto.emitReachable;
proto.emitReachable = function (...args) {
for (const [field, value] of Object.entries(this))
if (value instanceof Map || value instanceof Set) instrument(field, value);
return emitReachable.apply(this, args);
};
const produced = (result, into) => {
if (result === null || result === undefined) return;
if (Array.isArray(result)) for (const fn of result) produced(fn, into);
else if (typeof result === "object" && typeof result.name === "string") into.add(result.name);
};
// Passes over the lowered bodies after the worklist: the functions they
// create belong to the pass.
const passes = new Map();
const wrapPass = (owner, label, method) => {
const original = owner.prototype[method];
owner.prototype[method] = function (lowerer, ...rest) {
const implicit = lowerer.implicitFns?.length ?? 0;
const lifted = lowerer.liftedFns?.length ?? 0;
try {
return original.call(this, lowerer, ...rest);
} finally {
let names = passes.get(label);
if (names === undefined) passes.set(label, (names = new Set()));
for (const fn of (lowerer.implicitFns ?? []).slice(implicit)) names.add(fn.name);
for (const fn of (lowerer.liftedFns ?? []).slice(lifted)) names.add(fn.name);
}
};
};
wrapPass(ClassDynamicDispatch, "ClassDynamicDispatch.process", "process");
wrapPass(ClassDynamicDispatch, "ClassDynamicDispatch.finalize", "finalize");
wrapPass(DeferredModuleInitializers, "DeferredModuleInitializers.process", "process");
for (const name of [
"lowerFunction",
"lowerClassCtor",
"lowerClassMethodMember",
"lowerClassMembers",
"lowerFileInit",
"lowerGenericInstance",
"throwingSetterFn",
]) {
const original = proto[name];
if (typeof original !== "function") continue;
proto[name] = function (...args) {
const token = { id: tokens.length, names: new Set() };
tokens.push(token);
const implicit = this.implicitFns?.length ?? 0;
const lifted = this.liftedFns?.length ?? 0;
stack.push(token);
try {
const result = original.apply(this, args);
produced(result, token.names);
return result;
} finally {
stack.pop();
for (const fn of (this.implicitFns ?? []).slice(implicit)) token.names.add(fn.name);
for (const fn of (this.liftedFns ?? []).slice(lifted)) token.names.add(fn.name);
}
};
}
return {
report() {
const out = {};
for (const [field, keys] of fields) {
const involved = new Set();
let sensitive = 0;
for (const { readers, writers } of keys.values()) {
const all = new Set([...readers, ...writers]);
if (writers.size === 0 || all.size < 2) continue;
sensitive++;
for (const id of all) for (const fn of tokens[id].names) involved.add(fn);
}
if (sensitive > 0) out[field] = { sensitiveKeys: sensitive, functions: [...involved] };
}
for (const [label, names] of passes)
if (names.size > 0) out[`pass ${label}`] = { sensitiveKeys: 0, functions: [...names] };
return out;
},
};
}
const { values } = parseArgs({
options: {
stride: { type: "string", default: "7" },
offset: { type: "string", default: "0" },
limit: { type: "string", default: "100000" },
seeds: { type: "string", default: "1,2,3" },
jobs: { type: "string", default: String(Math.max(1, Math.floor(cpus().length / 2))) },
entry: { type: "string", multiple: true, default: [] },
out: { type: "string", default: join(tmpdir(), "scriptc-lowering-order") },
"no-audit": { type: "boolean", default: false },
"fail-on": { type: "string" },
},
});
if (values["fail-on"] !== undefined && !["names", "content"].includes(values["fail-on"]))
throw new Error(`--fail-on takes names or content, not ${values["fail-on"]}`);
const out = resolve(values.out);
mkdirSync(out, { recursive: true });
const seeds = values.seeds.split(",").filter(Boolean);
const corpus = join(root, "tests/corpus");
const programs = [
...readdirSync(corpus)
.filter((name) => /^\d+-.*\.ts$/.test(name))
.filter((name) => Number.parseInt(name, 10) % Number(values.stride) === Number(values.offset))
.sort((a, b) => Number.parseInt(a, 10) - Number.parseInt(b, 10))
.slice(0, Number(values.limit))
.map((name) => join(corpus, name)),
...values.entry.map((entry) => resolve(entry)),
];
const child = (entry, irPath, seed, auditPath, snapshotPath) =>
new Promise((done) => {
const proc = spawn(
process.execPath,
[
"--max-old-space-size=8192",
fileURLToPath(import.meta.url),
"--child",
entry,
irPath,
seed,
auditPath,
snapshotPath,
],
{ stdio: ["ignore", "ignore", "pipe"], env: { ...process.env, SCRIPTC_NO_CACHE: "1" } },
);
let stderr = "";
proc.stderr.on("data", (chunk) => (stderr += chunk));
proc.on("close", (code) => done({ ok: code === 0, stderr: stderr.slice(-2000) }));
});
const loadIr = (path) => JSON.parse(readFileSync(path, "utf8"));
const firstDiff = (x, y, path = "") => {
if (JSON.stringify(x) === JSON.stringify(y)) return null;
if (typeof x !== "object" || typeof y !== "object" || x === null || y === null) return path;
for (const key of new Set([...Object.keys(x), ...Object.keys(y)])) {
const found = firstDiff(x[key], y[key], `${path}/${key}`);
if (found !== null) return found;
}
return path;
};
const differingFunctions = (a, b) => {
const left = new Map(a.functions.map((fn) => [fn.name, fn]));
const right = new Map(b.functions.map((fn) => [fn.name, fn]));
const names = new Set();
const paths = {};
for (const [name, fn] of left) {
const other = right.get(name);
if (other === undefined) names.add(name);
else if (JSON.stringify(other) !== JSON.stringify(fn)) {
names.add(name);
paths[name] = firstDiff(fn, other).replace(/\/\d+/g, "/#");
}
}
for (const name of right.keys()) if (!left.has(name)) names.add(name);
const topLevel = [];
for (const key of new Set([...Object.keys(a), ...Object.keys(b)]))
if (key !== "functions" && JSON.stringify(a[key]) !== JSON.stringify(b[key]))
topLevel.push(key);
if (a.functions.map((fn) => fn.name).join("\n") !== b.functions.map((fn) => fn.name).join("\n"))
topLevel.push("function order");
return { functions: [...names], topLevel, paths };
};
/** Keys whose strings name functions, locals, globals, shapes, unions,
* classes and labels. Lowering numbers these in creation order; every other
* string (literal values, field names, library functions) is compared as
* is. A `name` names a function, record or union when it is one of the
* module's numbered definitions or starts with `%`. */
const ID_KEYS = new Set([
"base",
"callee",
"captures",
"catchLocalId",
"className",
"fnName",
"fromId",
"globalId",
"id",
"int32Locals",
"int32Params",
"label",
"labels",
"localId",
"retag",
"shapeId",
"targets",
"unionId",
]);
/** The module with every numbered function, record and union renamed by a
* hash of its own definition (ids with their digits stripped), other
* numbered ids by first use within their function or table entry, source
* locations dropped, and tables sorted. Orders whose canonical modules
* match differ only in names and ids numbered in creation order. */
const canonicalModule = (mod) => {
const hash = (text) => createHash("sha256").update(text).digest("hex").slice(0, 12);
const m = JSON.parse(JSON.stringify(mod, (k, v) => (k === "loc" ? undefined : v)));
const definitions = new Set([
...(m.functions ?? []).map((fn) => fn.name),
...["records", "unions"].flatMap((key) => (m[key] ?? []).map((item) => item.id)),
]);
const isId = (key, value) =>
ID_KEYS.has(key) || (key === "name" && (definitions.has(value) || value.startsWith("%")));
/** `v` with the strings under id keys passed through `id`. */
const mapIds = (v, id, key) => {
if (typeof v === "string") return isId(key, v) ? id(v) : v;
if (Array.isArray(v)) return v.map((x) => mapIds(x, id, key));
if (v && typeof v === "object")
return Object.fromEntries(Object.keys(v).map((k) => [k, mapIds(v[k], id, k)]));
return v;
};
const strip = (v) => JSON.stringify(mapIds(v, (id) => id.replace(/\d+/g, "#")));
const rename = new Map();
for (const fn of m.functions ?? []) {
if (!/\d/.test(fn.name)) continue;
const { name, ...rest } = fn;
rename.set(name, `${name.replace(/\d+/g, "#")}~${hash(strip(rest))}`);
}
for (const key of ["records", "unions"])
for (const item of m[key] ?? []) {
const { id, ...rest } = item;
if (typeof id === "string" && /\d/.test(id))
rename.set(id, `${id.replace(/\d+/g, "#")}~${hash(strip(rest))}`);
}
const canonical = (item) => {
const scope = new Map();
return mapIds(item, (value) => {
const renamed = rename.get(value);
if (renamed !== undefined) return renamed;
if (!/\d/.test(value)) return value;
let id = scope.get(value);
if (id === undefined) scope.set(value, (id = scope.size));
return `${value.replace(/\d+/g, "#")}@${id}`;
});
};
const out = {};
for (const key of Object.keys(m))
out[key] = Array.isArray(m[key])
? m[key].map((item) => JSON.stringify(canonical(item))).sort()
: JSON.stringify(canonical(m[key]));
return out;
};
/** Canonical entries present in one order only, by table. */
const contentDifferences = (a, b) => {
const found = {};
for (const key of new Set([...Object.keys(a), ...Object.keys(b)])) {
if (JSON.stringify(a[key]) === JSON.stringify(b[key])) continue;
if (!Array.isArray(a[key]) || !Array.isArray(b[key])) {
found[key] = ["(differs)"];
continue;
}
const other = new Set(b[key]);
found[key] = a[key]
.filter((item) => !other.has(item))
.map((item) => JSON.parse(item).name ?? JSON.parse(item).id ?? "?");
}
return found;
};
/** Fields whose contents after reachable emit differ between two orders. */
const differingState = (a, b) => {
const fields = [];
for (const field of new Set([...Object.keys(a), ...Object.keys(b)])) {
const x = a[field] ?? { entries: {}, unkeyed: [] };
const y = b[field] ?? { entries: {}, unkeyed: [] };
const keys = new Set([...Object.keys(x.entries), ...Object.keys(y.entries)]);
let differs = [...keys].some((k) => x.entries[k] !== y.entries[k]);
if (!differs) differs = JSON.stringify(x.unkeyed) !== JSON.stringify(y.unkeyed);
if (differs) fields.push(field);
}
return fields;
};
const results = [];
let next = 0;
await Promise.all(
Array.from({ length: Number(values.jobs) }, async () => {
while (next < programs.length) {
const entry = programs[next++];
const name = basename(entry).replace(/\.ts$/, "");
const dir = join(out, name);
mkdirSync(dir, { recursive: true });
const audit = values["no-audit"] ? "-" : join(dir, "audit.json");
const base = await child(
entry,
join(dir, "base.ir.json"),
"-",
audit,
join(dir, "base.state.json"),
);
if (!base.ok) {
results.push({ program: name, status: "does not compile" });
continue;
}
const diffs = {};
const stateDiffs = {};
let baseCanonical = null;
for (const seed of seeds) {
const run = await child(
entry,
join(dir, `seed${seed}.ir.json`),
seed,
"-",
join(dir, `seed${seed}.state.json`),
);
if (!run.ok) {
// A program that lowers in discovery order but not in this one
// differs in content.
diffs[seed] = { failed: run.stderr, content: { lowering: ["fails in this order"] } };
continue;
}
const diff = differingFunctions(
loadIr(join(dir, "base.ir.json")),
loadIr(join(dir, `seed${seed}.ir.json`)),
);
const state = differingState(
loadIr(join(dir, "base.state.json")),
loadIr(join(dir, `seed${seed}.state.json`)),
);
if (state.length > 0) stateDiffs[seed] = state;
if (diff.functions.length > 0 || diff.topLevel.length > 0) {
baseCanonical ??= canonicalModule(loadIr(join(dir, "base.ir.json")));
diff.content = contentDifferences(
baseCanonical,
canonicalModule(loadIr(join(dir, `seed${seed}.ir.json`))),
);
diffs[seed] = diff;
}
}
const auditReport = audit === "-" ? {} : JSON.parse(readFileSync(audit, "utf8"));
const changed = new Set(Object.values(diffs).flatMap((diff) => diff.functions ?? []));
const attribution = {};
for (const fn of changed) {
const fields = Object.entries(auditReport)
.filter(([, entry]) => entry.functions.includes(fn))
.map(([field]) => field);
attribution[fn] = fields;
}
results.push({
program: name,
status:
Object.keys(diffs).length === 0
? "order-independent"
: Object.values(diffs).some((diff) => Object.keys(diff.content ?? {}).length > 0)
? "order-dependent content"
: "order-dependent names",
diffs,
attribution,
sensitiveFields: Object.keys(auditReport),
stateFields: [...new Set(Object.values(stateDiffs).flat())],
});
process.stderr.write(
`${results.length}/${programs.length} ${name}: ${results.at(-1).status}\n`,
);
}
}),
);
writeFileSync(join(out, "results.json"), JSON.stringify(results, null, 2));
const dependent = results.filter((r) => r.status.startsWith("order-dependent"));
const content = results.filter((r) => r.status === "order-dependent content");
const byField = new Map();
let unattributed = 0;
for (const r of dependent) {
const fields = new Set(Object.values(r.attribution).flat());
if (fields.size === 0) unattributed++;
for (const field of fields) byField.set(field, (byField.get(field) ?? 0) + 1);
}
console.log(
`programs ${results.length}: ${dependent.length} order-dependent (${content.length} beyond names and ids), ${results.filter((r) => r.status === "does not compile").length} do not compile`,
);
for (const r of content)
console.log(
` ${r.program}: ${Object.values(r.diffs)
.flatMap((diff) =>
Object.entries(diff.content ?? {}).map(
([table, items]) => `${table}: ${items.slice(0, 4).join(", ")}`,
),
)
.slice(0, 3)
.join("; ")}`,
);
for (const [field, count] of [...byField].sort((a, b) => b[1] - a[1]))
console.log(`${String(count).padStart(5)} ${field}`);
console.log(`${String(unattributed).padStart(5)} (no audited field)`);
const failing =
values["fail-on"] === "names" ? dependent : values["fail-on"] === "content" ? content : [];
if (failing.length > 0) {
console.log(`${failing.length} programs lower to different IR in another order`);
process.exitCode = 1;
}