diff --git a/package.json b/package.json index ff4c84dc2..5cc30d9f8 100644 --- a/package.json +++ b/package.json @@ -20,6 +20,7 @@ "test:test262": "tsx tests/test262/run.ts", "test:test262:fetch": "node tests/test262/fetch.mjs", "test:cache-identity": "node tests/harness/cache-identity.mjs", + "test:lowering-order": "node scripts/lowering-order.mjs --fail-on content", "test:sandbox": "node scripts/sandbox-test.mjs", "test:sandbox:image": "node scripts/sandbox-image.mjs", "test:watch": "vitest", diff --git a/packages/compiler/src/frontend/lowering/caught-value-dispatch.ts b/packages/compiler/src/frontend/lowering/caught-value-dispatch.ts index 645ff8d2c..606d10d56 100644 --- a/packages/compiler/src/frontend/lowering/caught-value-dispatch.ts +++ b/packages/compiler/src/frontend/lowering/caught-value-dispatch.ts @@ -5,6 +5,7 @@ import { DYN, type IrExpr, type IrFunction, + type IrStmt, type IrType, type SrcLoc, } from "../../ir/ir.js"; @@ -59,28 +60,48 @@ export class CaughtValueDispatch { loc, }; lowerer.liftedFns.push(this.helper); + lowerer.shareHelper(this.helper); changed = true; } const helper = this.helper; + let added = false; for (const className of this.thrownClasses) { if (this.convertedClasses.has(className)) continue; this.convertedClasses.add(className); + added = true; + } + if (added) { + // A subclass is tested before its ancestors, and classes of one depth + // by name, so the tests do not depend on the order the census met them. + const depth = (className: string): number => { + let count = 0; + for (let info = lowerer.classes.get(className)?.base; info; info = info.base) count++; + return count; + }; + const ordered = [...this.convertedClasses].sort( + (left, right) => depth(right) - depth(left) || (left < right ? -1 : left > right ? 1 : 0), + ); const loc = helper.loc; const value = varRef("caught", CAUGHT, loc); - const type: IrType = { kind: "object", className }; - helper.body.unshift({ - kind: "if", - cond: { kind: "caughtTest", value, test: "instanceof", className, type: BOOL, loc }, - then: [ - { - kind: "return", - value: lowerer.coerceToExpected({ kind: "caughtNarrow", value, type, loc }, DYN), + helper.body = [ + ...ordered.map((className): IrStmt => { + const type: IrType = { kind: "object", className }; + return { + kind: "if", + cond: { kind: "caughtTest", value, test: "instanceof", className, type: BOOL, loc }, + then: [ + { + kind: "return", + value: lowerer.coerceToExpected({ kind: "caughtNarrow", value, type, loc }, DYN), + loc, + }, + ], + else_: null, loc, - }, - ], - else_: null, - loc, - }); + }; + }), + helper.body[helper.body.length - 1]!, + ]; changed = true; } for (const fn of consumers) { diff --git a/packages/compiler/src/frontend/lowering/class-dynamic-dispatch.test.ts b/packages/compiler/src/frontend/lowering/class-dynamic-dispatch.test.ts index 6923841bd..b717452c2 100644 --- a/packages/compiler/src/frontend/lowering/class-dynamic-dispatch.test.ts +++ b/packages/compiler/src/frontend/lowering/class-dynamic-dispatch.test.ts @@ -33,6 +33,7 @@ function context() { classes: new Map(), liftedFns, liftedFnNamed: (name: string | undefined) => liftedFns.find((fn) => fn.name === name), + shareHelper: () => {}, subclassLinks: 0, constructedClassCount: () => -1, diags: [], diff --git a/packages/compiler/src/frontend/lowering/class-dynamic-dispatch.ts b/packages/compiler/src/frontend/lowering/class-dynamic-dispatch.ts index 9dd7681a6..a41751f01 100644 --- a/packages/compiler/src/frontend/lowering/class-dynamic-dispatch.ts +++ b/packages/compiler/src/frontend/lowering/class-dynamic-dispatch.ts @@ -281,6 +281,7 @@ export function classInstanceOf( classValue?: IrExpr, ): IrExpr { const name = `%dyn.class.${classValue ? "valueInstanceof" : "instanceof"}:${info.def.name}`; + lowerer.shareHelper(name); const params = [ { localId: "value", name: "value", type: DYN }, ...(classValue ? [{ localId: "class", name: "class", type: classValue.type }] : []), @@ -406,8 +407,16 @@ function withTypedReceiver(value: IrExpr, receiver: IrExpr): IrExpr { }); } +/** Re-adds the entries of `set` after the first `from` sorted. */ +function sortAdded(set: Set, from: number): void { + const added = [...set].slice(from).sort(); + for (const name of added) set.delete(name); + for (const name of added) set.add(name); +} + export function classPropertiesHelper(lowerer: Lowerer, loc: SrcLoc, readOnly = false): IrFunction { const name = readOnly ? READ_PROPERTIES : "%dyn.class.properties"; + lowerer.shareHelper(name); const existing = lowerer.liftedFnNamed(name); if (existing) return existing; const helper: IrFunction = { @@ -447,6 +456,8 @@ export function classForInHasKey( loc: SrcLoc, ): IrExpr { const name = "%dyn.class.forInHasKey"; + lowerer.shareHelper(name); + lowerer.shareHelper("%dyn.class.forInBagHas"); if (!lowerer.liftedFns.some((fn) => fn.name === name)) { // The read-only table: a recheck never creates an instance's table, which // is undefined while the instance has no own dynamic properties. @@ -825,6 +836,8 @@ export class ClassDynamicDispatch { else if (type.kind === "func") discover(type.ret); else if (type.kind === "promise") discover(type.inner); }; + const boxedBefore = this.boxed.size; + const boxedConstructorsBefore = this.boxedConstructors.size; for (const fn of functions) everyStmtList(fn.body, { stmt: (statement) => { @@ -919,6 +932,10 @@ export class ClassDynamicDispatch { return true; }, }); + // The classes this pass found join the tables by name, not in the order + // the scan met them. + sortAdded(this.boxed, boxedBefore); + sortAdded(this.boxedConstructors, boxedConstructorsBefore); changed = this.caughtDispatch.dynamicPayloads(lowerer, caughtSites, this.boxed) || changed; if (this.boxed.size === 0 && rewrite.size === 0) return changed; const converting = this.convertingHints(lowerer); @@ -4264,6 +4281,8 @@ export class ClassDynamicDispatch { * prototype-backed bag is produced for this read only. A published object * cannot take the store, and Node's reads never write. */ finalize(lowerer?: Lowerer): void { + // Each table serves every site that reaches it. + if (lowerer) for (const fn of this.generated) lowerer.shareHelper(fn); for (const { helper, initialize, orEmpty } of this.readPropertyBags) { const absent = helper.body[0]; if (isEmptyBag(initialize.value) && !orEmpty) helper.body.shift(); diff --git a/packages/compiler/src/frontend/lowering/class-prototypes.ts b/packages/compiler/src/frontend/lowering/class-prototypes.ts index e65c679c6..4cbd8e5cf 100644 --- a/packages/compiler/src/frontend/lowering/class-prototypes.ts +++ b/packages/compiler/src/frontend/lowering/class-prototypes.ts @@ -122,6 +122,7 @@ export function classPrototypeData( const globalId = `%g.${name}`; const readyId = `${globalId}.ready`; info.def.prototypeDataHelper = name; + lowerer.shareHelper(name); const className = info.def.jsName ?? info.def.name; lowerer.globalsList.push({ id: globalId, diff --git a/packages/compiler/src/frontend/lowering/error-methods.ts b/packages/compiler/src/frontend/lowering/error-methods.ts index 979cf4814..b545f4147 100644 --- a/packages/compiler/src/frontend/lowering/error-methods.ts +++ b/packages/compiler/src/frontend/lowering/error-methods.ts @@ -24,6 +24,7 @@ export function errorPropertyRead( ): IrExpr { const name = `%error.${property}.read`, loc = receiver.loc; + lowerer.shareHelper(name); if (!lowerer.liftedFns.some((fn) => fn.name === name)) lowerer.liftedFns.push({ name, @@ -50,6 +51,7 @@ export function errorPropertyWrite( ): IrStmt { const name = `%error.${property}.write`, loc = receiver.loc; + lowerer.shareHelper(name); if (!lowerer.liftedFns.some((fn) => fn.name === name)) lowerer.liftedFns.push({ name, @@ -392,6 +394,7 @@ export function errorToStringMethod( receiver: IrExpr, ): Extract { const loc = receiver.loc; + lowerer.shareHelper(helperName); if (!lowerer.liftedFns.some((fn) => fn.name === helperName)) { lowerer.liftedFns.push({ name: helperName, @@ -415,6 +418,7 @@ export function errorToStringCall(lowerer: Lowerer, receiver: IrExpr): IrExpr { const loc = receiver.loc; const method = errorToStringMethod(lowerer, receiver); const name = "%error.toString.virtual"; + lowerer.shareHelper(name); if (!lowerer.liftedFns.some((fn) => fn.name === name)) { const value = varRef("error.0", errorType, loc); lowerer.liftedFns.push({ diff --git a/packages/compiler/src/frontend/lowering/generator-dynamic-dispatch.ts b/packages/compiler/src/frontend/lowering/generator-dynamic-dispatch.ts index ca6a5ecdc..3f43e5e05 100644 --- a/packages/compiler/src/frontend/lowering/generator-dynamic-dispatch.ts +++ b/packages/compiler/src/frontend/lowering/generator-dynamic-dispatch.ts @@ -68,8 +68,10 @@ export class GeneratorDynamicDispatch { this.properties.set(method, { fn, types: new Set() }); this.generated.add(fn); lowerer.liftedFns.push(fn); + lowerer.shareHelper(fn); // The runtime's own helper functions answer the helper methods. if (HELPER_METHODS.has(method)) return fn; + lowerer.shareHelper(`%dyn.generator.method.${method}`); lowerer.liftedFns.push({ name: `%dyn.generator.method.${method}`, params: [{ localId: "arg", name: "arg", type: DYN }], @@ -209,6 +211,7 @@ export class GeneratorDynamicDispatch { this.dispatches.set(key, dispatch); this.generated.add(helper); lowerer.liftedFns.push(helper); + lowerer.shareHelper(helper); changed = true; } return { diff --git a/packages/compiler/src/frontend/lowering/lower-event-emitter.ts b/packages/compiler/src/frontend/lowering/lower-event-emitter.ts index d84f84f4e..f472672b2 100644 --- a/packages/compiler/src/frontend/lowering/lower-event-emitter.ts +++ b/packages/compiler/src/frontend/lowering/lower-event-emitter.ts @@ -2128,8 +2128,12 @@ function ensureEmitSpec(lowerer: Lowerer, info: ClassInfo, event: string, tuple: params: tuple.map((t) => ({ type: t, mode: "required" as const })), ret: BOOL, }); - if (info.def.methods) info.def.methods.push(mName); - else info.def.methods = [mName]; + // Specializations sit in the method list sorted by event name, so the + // class table does not depend on which body asked for one first. + const methods = (info.def.methods ??= []); + const after = methods.findIndex((m) => m.startsWith("emit:") && m > mName); + if (after < 0) methods.push(mName); + else methods.splice(after, 0, mName); } const ordinal = lowerer.emitSpecQueue.filter((q) => q.info === info).length; lowerer.emitSpecQueue.push({ info, event, tuple, ordinal }); diff --git a/packages/compiler/src/frontend/lowering/lower-modules.ts b/packages/compiler/src/frontend/lowering/lower-modules.ts index 6ca21ddaa..d1f276d44 100644 --- a/packages/compiler/src/frontend/lowering/lower-modules.ts +++ b/packages/compiler/src/frontend/lowering/lower-modules.ts @@ -1048,7 +1048,8 @@ export function moduleArtifacts( } const reachable = lowerer.reachableForArtifacts ?? lowerer.reachable; return { - classes: [...lowerer.classes.values()] + classes: lowerer + .classesInTableOrder() .map((c) => c.def) .filter((def) => classNames.has(def.name)) .map((def): IrClassDef => { diff --git a/packages/compiler/src/frontend/lowering/lowerer.ts b/packages/compiler/src/frontend/lowering/lowerer.ts index 383143c97..29f807336 100644 --- a/packages/compiler/src/frontend/lowering/lowerer.ts +++ b/packages/compiler/src/frontend/lowering/lowerer.ts @@ -959,6 +959,33 @@ function directExternalTypeSpecifiersByFile( return out; } +/** Debugging aid: SCRIPTC_LOWER_SHUFFLE_SEED= lowers each wave of + * reachable bodies in an order drawn from that seed instead of discovery + * order. Output assembly is ordered independently of it, so any IR + * difference between seeds is state one body's lowering leaves for another + * (scripts/lowering-order.mjs). */ +function lowerShuffle(): ((wave: string[]) => void) | null { + const raw = process.env["SCRIPTC_LOWER_SHUFFLE_SEED"]; + if (raw === undefined || raw === "") return null; + // mulberry32 + let state = Number.parseInt(raw, 10) >>> 0; + const next = (): number => { + state = (state + 0x6d2b79f5) >>> 0; + let t = state; + t = Math.imul(t ^ (t >>> 15), t | 1); + t ^= t + Math.imul(t ^ (t >>> 7), t | 61); + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; + }; + return (wave) => { + for (let index = wave.length - 1; index > 0; index--) { + const other = Math.floor(next() * (index + 1)); + const swap = wave[index]!; + wave[index] = wave[other]!; + wave[other] = swap; + } + }; +} + /** Build lowering runs as a reachability worklist over the ts.Program: * * 1. REACHABLE EMIT — a worklist computes the set of reachable bodies. @@ -1841,6 +1868,13 @@ function monomorphizedImplicit( return true; } +/** `fns` sorted by name; functions of one name keep their order. */ +function byName(fns: readonly IrFunction[]): IrFunction[] { + return [...fns].sort((left, right) => + left.name < right.name ? -1 : left.name > right.name ? 1 : 0, + ); +} + export class Lowerer { readonly frontendServices: FrontendServices | undefined; readonly checker: ts.TypeChecker; @@ -2027,6 +2061,40 @@ export class Lowerer { this.genericClassInstances.splice(from, tail.length, ...tail); } + /** The classes in table order. Generic instances are added as bodies + * demand them, so they keep the slots they hold and fill them in the + * order of their settled demand ranks. */ + classesInTableOrder(): ClassInfo[] { + const all = [...this.classes.values()]; + const slots = all.flatMap((info, index) => + this.genericClassDemandPriority.has(info) ? [index] : [], + ); + const ranked = slots + .map((index) => all[index]!) + .sort((left, right) => { + const a = this.genericClassDemandPriority.get(left)!.rank; + const b = this.genericClassDemandPriority.get(right)!.rank; + return a[0]! - b[0]! || a[1]! - b[1]!; + }); + slots.forEach((slot, index) => (all[slot] = ranked[index]!)); + return all; + } + + /** The globals in table order. A class's prototype data globals are + * added when a body first reads that prototype, so they keep the slots + * they hold and fill them in id order. */ + private globalsInTableOrder(): IrGlobal[] { + const all = [...this.globalsList]; + const prototypeData = (global: IrGlobal): boolean => + global.id.startsWith("%g.%prototype.data."); + const slots = all.flatMap((global, index) => (prototypeData(global) ? [index] : [])); + const sorted = slots + .map((index) => all[index]!) + .sort((left, right) => (left.id < right.id ? -1 : left.id > right.id ? 1 : 0)); + slots.forEach((slot, index) => (all[slot] = sorted[index]!)); + return all; + } + private settleGenericDemandPriorities(): void { const functionQueue: GenericInstance[] = []; const classQueue: ClassInfo[] = []; @@ -3187,6 +3255,60 @@ export class Lowerer { constructedClassCount(): number { return this.constructedClasses?.size ?? -1; } + /** Helpers that every site asking for them shares (shareHelper). */ + private readonly sharedHelpers = new Set(); + private readonly sharedHelperNames = new Set(); + /** Marks a helper, or the lifted helpers of a name, as shared by every + * site that asks for it. A shared helper is built for whichever site asks + * first, so once lowering finishes its locations are replaced by one that + * names no site. */ + shareHelper(helper: IrFunction | string): void { + if (typeof helper === "string") this.sharedHelperNames.add(helper); + else this.sharedHelpers.add(helper); + } + /** Gives each shared helper the location of the entry file's start: the + * helpers marked by shareHelper, and those interned by key in the helper + * tables. */ + private settleSharedHelperLocations(): void { + const loc: SrcLoc = { file: this.entry.fileName, start: 0, end: 0 }; + const interned = new Set([...this.sharedHelperNames, ...this.classMethodValueHelpers]); + for (const table of [ + this.equalityHelpers, + this.arrHofHelpers, + this.unionCallHelpers, + this.valueHelpers, + this.assertHelpers, + this.inspectHelpers, + this.iterDrainHelpers, + this.mapHofHelpers, + this.setHofHelpers, + this.spHofHelpers, + this.primitiveCtorFns, + this.builtinCallableValueFns, + this.regionRunHelpers, + this.internTableHelpers, + this.coercions.retags, + this.coercions.narrows, + ]) + for (const name of table.values()) interned.add(name); + for (const fn of [...this.liftedFns, ...this.implicitFns]) + if (interned.has(fn.name)) this.sharedHelpers.add(fn); + const seen = new Set(); + const visit = (node: unknown): void => { + if (typeof node !== "object" || node === null || seen.has(node)) return; + seen.add(node); + if (Array.isArray(node)) { + for (const item of node) visit(item); + return; + } + const record = node as Record; + for (const key of Object.keys(record)) { + if (key === "loc") record[key] = loc; + else if (key !== "type") visit(record[key]); + } + }; + for (const fn of this.sharedHelpers) visit(fn); + } /** The first lifted function named `name`, the one * `liftedFns.find((fn) => fn.name === name)` returns. */ liftedFnNamed(name: string | undefined): IrFunction | undefined { @@ -7279,6 +7401,7 @@ export class Lowerer { // Lambdas lifted while lowering any of the above (plus synthetic // array-HOF loop functions, which ride the same list), and the // implicit-any instances lowered eagerly at their first call sites. + this.settleSharedHelperLocations(); functions.push(...this.liftedFns); functions.push(...this.implicitFns); functions.push(...runtimeClassConstructs(this, functions)); @@ -7412,7 +7535,7 @@ export class Lowerer { classes: artifacts.classes, records: artifacts.records, unions: artifacts.unions, - globals: this.globalsList, + globals: this.globalsInTableOrder(), entry: ENTRY_NAME, ...(this.usesWorkers ? { workers: true } : {}), ...(this.usesLifetimeRegions ? { lifetimeRegions: true as const } : {}), @@ -7859,6 +7982,15 @@ export class Lowerer { // exactly like generic instances. while (specLowered < this.emitSpecQueue.length) { const waveEnd = this.emitSpecQueue.length; + // A wave's specializations lower by class and event, not in the + // order bodies asked for them. + const wave = this.emitSpecQueue.slice(specLowered, waveEnd); + const specKey = ({ info, event }: EmitSpecRequest): string => + `${info.def.name}\0${event}`; + wave.sort((left, right) => + specKey(left) < specKey(right) ? -1 : specKey(left) > specKey(right) ? 1 : 0, + ); + this.emitSpecQueue.splice(specLowered, wave.length, ...wave); this.checker.prefetchRoots( this.emitSpecQueue .slice(specLowered, waveEnd) @@ -7908,9 +8040,11 @@ export class Lowerer { // seed the worklist beside the init bodies. Unknown names are inert // (the export-map resolution reports them as SC4002 later). for (const root of extraRoots ?? []) this.onEdge?.(root); + const shuffle = lowerShuffle(); const drainUnits = (): void => { while (queue.length > 0) { const wave = queue.splice(0); + shuffle?.(wave); this.checker.prefetchRoots(wave.flatMap((name) => units.get(name)!.roots)); for (const name of wave) { // A body-level poison outside the per-statement catches (a fenced @@ -7940,21 +8074,28 @@ export class Lowerer { // queue above is the only portion whose discovery order differed from // historical emit order. const classDispatch = new ClassDynamicDispatch(extraRoots !== undefined); + // Units in declaration order, so the passes below meet functions, and the + // classes they collect, in an order that does not depend on lowering order. + const unitsInOrder = (): IrFunction[] => + [...loweredUnits] + .sort(([left], [right]) => units.get(left)!.order - units.get(right)!.order) + .map(([, fn]) => fn); for (;;) { drainInstances(); + const declared = unitsInOrder(); const initializerChanged = this.deferredModuleInitializers.process(this, [ - ...loweredUnits.values(), + ...declared, ...initFunctions, ...instanceFunctions, - ...this.liftedFns, - ...this.implicitFns, + ...byName(this.liftedFns), + ...byName(this.implicitFns), ]); const dispatchChanged = classDispatch.process(this, [ - ...loweredUnits.values(), + ...declared, ...initFunctions, ...instanceFunctions, - ...this.liftedFns, - ...this.implicitFns, + ...byName(this.liftedFns), + ...byName(this.implicitFns), ]); if (queue.length === 0 && !dispatchChanged && !initializerChanged) break; drainUnits(); @@ -7962,9 +8103,7 @@ export class Lowerer { this.restoreGenericClassInstanceOrder(clsInstLowered); } classDispatch.finalize(this); - const orderedUnits = [...loweredUnits] - .sort(([left], [right]) => units.get(left)!.order - units.get(right)!.order) - .map(([, fn]) => fn); + const orderedUnits = unitsInOrder(); this.settleGenericDemandPriorities(); this.shapes.settleDeclaredOrderPriorities(); for (const finalize of this.shapeOrderMetadataFinalizers) finalize(); @@ -7977,13 +8116,16 @@ export class Lowerer { ]); observeRecordLiteralRebuilds(null); for (const finalize of this.shapeOrderHelperFinalizers) finalize(); + this.settleSharedHelperLocations(); + // Lifted and implicit functions are created in lowering order; they + // are assembled by name. const functions = [ ...orderedUnits, ...initFunctions, this.buildMain(), ...instanceFunctions, - ...this.liftedFns, - ...this.implicitFns, + ...byName(this.liftedFns), + ...byName(this.implicitFns), ]; functions.push(...runtimeClassConstructs(this, functions)); if (this.regexGroupResults) for (const fn of functions) foldRegexSpanReads(this, fn); diff --git a/packages/compiler/test/format-float-arguments.test.ts b/packages/compiler/test/format-float-arguments.test.ts index 8aa08469d..e59e023ca 100644 --- a/packages/compiler/test/format-float-arguments.test.ts +++ b/packages/compiler/test/format-float-arguments.test.ts @@ -27,8 +27,9 @@ test("util.format %f lowers a boolean argument once", async () => { const helpers = mod.functions .map((f) => f.name) .filter((name) => name.startsWith("%arr.")) - .map((name) => name.replace(/\.\d+$/, "")); - expect(helpers).toEqual(["%arr.some", "%arr.every"]); + .map((name) => name.replace(/\.\d+$/, "")) + .sort(); + expect(helpers).toEqual(["%arr.every", "%arr.some"]); } finally { await rm(dir, { recursive: true, force: true }); } diff --git a/scripts/lowering-order.mjs b/scripts/lowering-order.mjs new file mode 100644 index 000000000..81611e6a4 --- /dev/null +++ b/scripts/lowering-order.mjs @@ -0,0 +1,601 @@ +#!/usr/bin/env node +/* Finds lowering state whose result depends on the order reachable bodies + * lower in. + * + * Each program is lowered to IR in discovery order and with every seed of + * SCRIPTC_LOWER_SHUFFLE_SEED (lowerer.ts), and the IR is compared function + * by function. Output assembly does not depend on the order, so every + * difference is state one body's lowering leaves for another. The + * discovery-order run also audits the lowerer's Map and Set fields: a key + * that two bodies touch, at least one of them writing it, makes the field + * order-sensitive for both bodies. A function whose IR differs between + * orders is attributed to the order-sensitive fields of the body that + * produced it. + * + * node scripts/lowering-order.mjs [--stride 7] [--offset 0] [--limit N] + * [--seeds 1,2,3] [--jobs 8] [--entry ]... [--out ] + * [--fail-on names|content] + * + * Needs a built compiler (pnpm build). Writes /results.json and prints + * a ranking of fields by the programs whose IR they change. With --fail-on, + * it exits with status 1 when some program's IR differs between orders: + * in any way (names), or beyond names and ids (content). */ +import { spawn } from "node:child_process"; +import { createHash } from "node:crypto"; +import { mkdirSync, readFileSync, readdirSync, writeFileSync } from "node:fs"; +import { cpus, tmpdir } from "node:os"; +import { basename, join, resolve } from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import { parseArgs } from "node:util"; + +const root = fileURLToPath(new URL("..", import.meta.url)); +const dist = join(root, "packages/compiler/dist"); + +if (process.argv[2] === "--child") { + const [, , , entry, irPath, seed, auditPath, snapshotPath] = process.argv; + if (seed !== "-") process.env["SCRIPTC_LOWER_SHUFFLE_SEED"] = seed; + const module = (path) => import(pathToFileURL(join(dist, path)).href); + const audit = + auditPath === "-" + ? null + : installAudit( + await module("frontend/lowering/lowerer.js"), + await module("frontend/lowering/class-dynamic-dispatch.js"), + await module("frontend/lowering/deferred-module-initializers.js"), + ); + const snapshots = installSnapshot(await module("frontend/lowering/lowerer.js")); + const { compile } = await import(pathToFileURL(join(dist, "index.js")).href); + const outDir = join(tmpdir(), `scriptc-lowering-order-${process.pid}`); + const result = await compile(entry, { + outDir, + outPath: irPath, + backend: "llvm", + outputKind: "ir", + }); + if (audit !== null) writeFileSync(auditPath, JSON.stringify(audit.report())); + writeFileSync(snapshotPath, JSON.stringify(snapshots.at(-1) ?? {})); + process.exit(result.ok ? 0 : 3); +} + +/** Records the contents of the lowerer's Map and Set fields once reachable + * emit finishes, under keys that do not depend on the process: syntax + * nodes by file and span, symbols by name and declaration, IR values by id + * or name. Values are digested to a bounded depth. */ +function installSnapshot({ Lowerer }) { + const snapshots = []; + const nodeKey = (node) => { + try { + return `node:${node.getSourceFile().fileName}:${node.kind}:${node.pos}:${node.end}`; + } catch { + return `node:?:${node.kind}:${node.pos}:${node.end}`; + } + }; + const isNode = (value) => + typeof value.kind === "number" && + typeof value.pos === "number" && + typeof value.end === "number" && + typeof value.getSourceFile === "function"; + const isSymbol = (value) => + typeof value.escapedName === "string" && Array.isArray(value.declarationHandles); + const symbolKey = (symbol) => { + let declaration = ""; + try { + const first = symbol.declarations[0]; + if (first !== undefined) declaration = nodeKey(first); + } catch {} + return `symbol:${symbol.escapedName}:${declaration}`; + }; + const key = (value) => { + if (value === null || (typeof value !== "object" && typeof value !== "function")) + return `${typeof value}:${String(value)}`; + if (isNode(value)) return nodeKey(value); + if (isSymbol(value)) return symbolKey(value); + for (const field of ["name", "id", "localId"]) + if (typeof value[field] === "string" || typeof value[field] === "number") + return `${value.constructor?.name}:${field}:${value[field]}`; + if (value.def && typeof value.def.name === "string") + return `${value.constructor?.name}:def:${value.def.name}`; + return null; + }; + const digest = (value, depth) => { + if (value === null || value === undefined || typeof value !== "object") + return typeof value === "function" ? "fn" : (JSON.stringify(value) ?? "undefined"); + const stable = key(value); + if (depth === 0) return stable ?? `<${value.constructor?.name}>`; + if (isNode(value) || isSymbol(value)) return stable; + if (Array.isArray(value)) return `[${value.map((item) => digest(item, depth - 1)).join(",")}]`; + if (value instanceof Map) + return `Map{${[...value] + .map(([k, v]) => `${key(k) ?? "?"}=>${digest(v, depth - 1)}`) + .sort() + .join(",")}}`; + if (value instanceof Set) + return `Set{${[...value] + .map((item) => key(item) ?? digest(item, depth - 1)) + .sort() + .join(",")}}`; + const fields = Object.keys(value).sort(); + return `${value.constructor?.name ?? "Object"}{${fields.map((field) => `${field}:${digest(value[field], depth - 1)}`).join(",")}}`; + }; + const emitReachable = Lowerer.prototype.emitReachable; + Lowerer.prototype.emitReachable = function (...args) { + const result = emitReachable.apply(this, args); + const snapshot = {}; + for (const [field, value] of Object.entries(this)) { + if (!(value instanceof Map) && !(value instanceof Set)) continue; + const entries = {}; + const unkeyed = []; + for (const item of value instanceof Map ? value : [...value].map((v) => [v, true])) { + const [k, v] = item; + const stable = key(k); + const text = digest(v, 3); + if (stable === null) unkeyed.push(text); + else entries[stable] = text; + } + snapshot[field] = { entries, unkeyed: unkeyed.sort() }; + } + snapshots.push(snapshot); + return result; + }; + return snapshots; +} + +/** Wraps the lowering methods that produce bodies and the lowerer's Map and + * Set fields; records which bodies touch which keys. */ +function installAudit({ Lowerer }, { ClassDynamicDispatch }, { DeferredModuleInitializers }) { + const proto = Lowerer.prototype; + const tokens = []; + const stack = []; + const fields = new Map(); + const objectIds = new Map(); + const keyId = (key) => { + if (key === null || (typeof key !== "object" && typeof key !== "function")) + return `${typeof key}:${String(key)}`; + let id = objectIds.get(key); + if (id === undefined) objectIds.set(key, (id = objectIds.size)); + return `object:${id}`; + }; + const current = () => (stack.length === 0 ? null : stack[stack.length - 1]); + const touch = (field, key, write) => { + const token = current(); + if (token === null) return; + let keys = fields.get(field); + if (keys === undefined) fields.set(field, (keys = new Map())); + const id = key === ITERATE ? "*" : keyId(key); + let entry = keys.get(id); + if (entry === undefined) keys.set(id, (entry = { readers: new Set(), writers: new Set() })); + (write ? entry.writers : entry.readers).add(token.id); + }; + const ITERATE = Symbol("iterate"); + const instrument = (field, value) => { + const read = (key) => touch(field, key, false); + const write = (key) => { + touch(field, key, true); + // A write changes what every iteration of the field observes. + touch(field, ITERATE, true); + }; + const iterate = () => touch(field, ITERATE, false); + const base = value instanceof Map ? Map.prototype : Set.prototype; + for (const name of ["get", "has"]) { + if (!(name in base)) continue; + const original = base[name]; + value[name] = function (key) { + read(key); + return original.call(this, key); + }; + } + for (const name of ["set", "add", "delete"]) { + if (!(name in base)) continue; + const original = base[name]; + value[name] = function (key, ...rest) { + write(key); + return original.call(this, key, ...rest); + }; + } + for (const name of ["forEach", "keys", "values", "entries", Symbol.iterator]) { + const original = base[name]; + value[name] = function (...args) { + iterate(); + return original.apply(this, args); + }; + } + const size = Object.getOwnPropertyDescriptor(base, "size").get; + Object.defineProperty(value, "size", { + get() { + iterate(); + return size.call(this); + }, + }); + }; + const emitReachable = proto.emitReachable; + proto.emitReachable = function (...args) { + for (const [field, value] of Object.entries(this)) + if (value instanceof Map || value instanceof Set) instrument(field, value); + return emitReachable.apply(this, args); + }; + const produced = (result, into) => { + if (result === null || result === undefined) return; + if (Array.isArray(result)) for (const fn of result) produced(fn, into); + else if (typeof result === "object" && typeof result.name === "string") into.add(result.name); + }; + // Passes over the lowered bodies after the worklist: the functions they + // create belong to the pass. + const passes = new Map(); + const wrapPass = (owner, label, method) => { + const original = owner.prototype[method]; + owner.prototype[method] = function (lowerer, ...rest) { + const implicit = lowerer.implicitFns?.length ?? 0; + const lifted = lowerer.liftedFns?.length ?? 0; + try { + return original.call(this, lowerer, ...rest); + } finally { + let names = passes.get(label); + if (names === undefined) passes.set(label, (names = new Set())); + for (const fn of (lowerer.implicitFns ?? []).slice(implicit)) names.add(fn.name); + for (const fn of (lowerer.liftedFns ?? []).slice(lifted)) names.add(fn.name); + } + }; + }; + wrapPass(ClassDynamicDispatch, "ClassDynamicDispatch.process", "process"); + wrapPass(ClassDynamicDispatch, "ClassDynamicDispatch.finalize", "finalize"); + wrapPass(DeferredModuleInitializers, "DeferredModuleInitializers.process", "process"); + for (const name of [ + "lowerFunction", + "lowerClassCtor", + "lowerClassMethodMember", + "lowerClassMembers", + "lowerFileInit", + "lowerGenericInstance", + "throwingSetterFn", + ]) { + const original = proto[name]; + if (typeof original !== "function") continue; + proto[name] = function (...args) { + const token = { id: tokens.length, names: new Set() }; + tokens.push(token); + const implicit = this.implicitFns?.length ?? 0; + const lifted = this.liftedFns?.length ?? 0; + stack.push(token); + try { + const result = original.apply(this, args); + produced(result, token.names); + return result; + } finally { + stack.pop(); + for (const fn of (this.implicitFns ?? []).slice(implicit)) token.names.add(fn.name); + for (const fn of (this.liftedFns ?? []).slice(lifted)) token.names.add(fn.name); + } + }; + } + return { + report() { + const out = {}; + for (const [field, keys] of fields) { + const involved = new Set(); + let sensitive = 0; + for (const { readers, writers } of keys.values()) { + const all = new Set([...readers, ...writers]); + if (writers.size === 0 || all.size < 2) continue; + sensitive++; + for (const id of all) for (const fn of tokens[id].names) involved.add(fn); + } + if (sensitive > 0) out[field] = { sensitiveKeys: sensitive, functions: [...involved] }; + } + for (const [label, names] of passes) + if (names.size > 0) out[`pass ${label}`] = { sensitiveKeys: 0, functions: [...names] }; + return out; + }, + }; +} + +const { values } = parseArgs({ + options: { + stride: { type: "string", default: "7" }, + offset: { type: "string", default: "0" }, + limit: { type: "string", default: "100000" }, + seeds: { type: "string", default: "1,2,3" }, + jobs: { type: "string", default: String(Math.max(1, Math.floor(cpus().length / 2))) }, + entry: { type: "string", multiple: true, default: [] }, + out: { type: "string", default: join(tmpdir(), "scriptc-lowering-order") }, + "no-audit": { type: "boolean", default: false }, + "fail-on": { type: "string" }, + }, +}); +if (values["fail-on"] !== undefined && !["names", "content"].includes(values["fail-on"])) + throw new Error(`--fail-on takes names or content, not ${values["fail-on"]}`); +const out = resolve(values.out); +mkdirSync(out, { recursive: true }); +const seeds = values.seeds.split(",").filter(Boolean); +const corpus = join(root, "tests/corpus"); +const programs = [ + ...readdirSync(corpus) + .filter((name) => /^\d+-.*\.ts$/.test(name)) + .filter((name) => Number.parseInt(name, 10) % Number(values.stride) === Number(values.offset)) + .sort((a, b) => Number.parseInt(a, 10) - Number.parseInt(b, 10)) + .slice(0, Number(values.limit)) + .map((name) => join(corpus, name)), + ...values.entry.map((entry) => resolve(entry)), +]; + +const child = (entry, irPath, seed, auditPath, snapshotPath) => + new Promise((done) => { + const proc = spawn( + process.execPath, + [ + "--max-old-space-size=8192", + fileURLToPath(import.meta.url), + "--child", + entry, + irPath, + seed, + auditPath, + snapshotPath, + ], + { stdio: ["ignore", "ignore", "pipe"], env: { ...process.env, SCRIPTC_NO_CACHE: "1" } }, + ); + let stderr = ""; + proc.stderr.on("data", (chunk) => (stderr += chunk)); + proc.on("close", (code) => done({ ok: code === 0, stderr: stderr.slice(-2000) })); + }); + +const loadIr = (path) => JSON.parse(readFileSync(path, "utf8")); +const firstDiff = (x, y, path = "") => { + if (JSON.stringify(x) === JSON.stringify(y)) return null; + if (typeof x !== "object" || typeof y !== "object" || x === null || y === null) return path; + for (const key of new Set([...Object.keys(x), ...Object.keys(y)])) { + const found = firstDiff(x[key], y[key], `${path}/${key}`); + if (found !== null) return found; + } + return path; +}; +const differingFunctions = (a, b) => { + const left = new Map(a.functions.map((fn) => [fn.name, fn])); + const right = new Map(b.functions.map((fn) => [fn.name, fn])); + const names = new Set(); + const paths = {}; + for (const [name, fn] of left) { + const other = right.get(name); + if (other === undefined) names.add(name); + else if (JSON.stringify(other) !== JSON.stringify(fn)) { + names.add(name); + paths[name] = firstDiff(fn, other).replace(/\/\d+/g, "/#"); + } + } + for (const name of right.keys()) if (!left.has(name)) names.add(name); + const topLevel = []; + for (const key of new Set([...Object.keys(a), ...Object.keys(b)])) + if (key !== "functions" && JSON.stringify(a[key]) !== JSON.stringify(b[key])) + topLevel.push(key); + if (a.functions.map((fn) => fn.name).join("\n") !== b.functions.map((fn) => fn.name).join("\n")) + topLevel.push("function order"); + return { functions: [...names], topLevel, paths }; +}; + +/** Keys whose strings name functions, locals, globals, shapes, unions, + * classes and labels. Lowering numbers these in creation order; every other + * string (literal values, field names, library functions) is compared as + * is. A `name` names a function, record or union when it is one of the + * module's numbered definitions or starts with `%`. */ +const ID_KEYS = new Set([ + "base", + "callee", + "captures", + "catchLocalId", + "className", + "fnName", + "fromId", + "globalId", + "id", + "int32Locals", + "int32Params", + "label", + "labels", + "localId", + "retag", + "shapeId", + "targets", + "unionId", +]); + +/** The module with every numbered function, record and union renamed by a + * hash of its own definition (ids with their digits stripped), other + * numbered ids by first use within their function or table entry, source + * locations dropped, and tables sorted. Orders whose canonical modules + * match differ only in names and ids numbered in creation order. */ +const canonicalModule = (mod) => { + const hash = (text) => createHash("sha256").update(text).digest("hex").slice(0, 12); + const m = JSON.parse(JSON.stringify(mod, (k, v) => (k === "loc" ? undefined : v))); + const definitions = new Set([ + ...(m.functions ?? []).map((fn) => fn.name), + ...["records", "unions"].flatMap((key) => (m[key] ?? []).map((item) => item.id)), + ]); + const isId = (key, value) => + ID_KEYS.has(key) || (key === "name" && (definitions.has(value) || value.startsWith("%"))); + /** `v` with the strings under id keys passed through `id`. */ + const mapIds = (v, id, key) => { + if (typeof v === "string") return isId(key, v) ? id(v) : v; + if (Array.isArray(v)) return v.map((x) => mapIds(x, id, key)); + if (v && typeof v === "object") + return Object.fromEntries(Object.keys(v).map((k) => [k, mapIds(v[k], id, k)])); + return v; + }; + const strip = (v) => JSON.stringify(mapIds(v, (id) => id.replace(/\d+/g, "#"))); + const rename = new Map(); + for (const fn of m.functions ?? []) { + if (!/\d/.test(fn.name)) continue; + const { name, ...rest } = fn; + rename.set(name, `${name.replace(/\d+/g, "#")}~${hash(strip(rest))}`); + } + for (const key of ["records", "unions"]) + for (const item of m[key] ?? []) { + const { id, ...rest } = item; + if (typeof id === "string" && /\d/.test(id)) + rename.set(id, `${id.replace(/\d+/g, "#")}~${hash(strip(rest))}`); + } + const canonical = (item) => { + const scope = new Map(); + return mapIds(item, (value) => { + const renamed = rename.get(value); + if (renamed !== undefined) return renamed; + if (!/\d/.test(value)) return value; + let id = scope.get(value); + if (id === undefined) scope.set(value, (id = scope.size)); + return `${value.replace(/\d+/g, "#")}@${id}`; + }); + }; + const out = {}; + for (const key of Object.keys(m)) + out[key] = Array.isArray(m[key]) + ? m[key].map((item) => JSON.stringify(canonical(item))).sort() + : JSON.stringify(canonical(m[key])); + return out; +}; +/** Canonical entries present in one order only, by table. */ +const contentDifferences = (a, b) => { + const found = {}; + for (const key of new Set([...Object.keys(a), ...Object.keys(b)])) { + if (JSON.stringify(a[key]) === JSON.stringify(b[key])) continue; + if (!Array.isArray(a[key]) || !Array.isArray(b[key])) { + found[key] = ["(differs)"]; + continue; + } + const other = new Set(b[key]); + found[key] = a[key] + .filter((item) => !other.has(item)) + .map((item) => JSON.parse(item).name ?? JSON.parse(item).id ?? "?"); + } + return found; +}; + +/** Fields whose contents after reachable emit differ between two orders. */ +const differingState = (a, b) => { + const fields = []; + for (const field of new Set([...Object.keys(a), ...Object.keys(b)])) { + const x = a[field] ?? { entries: {}, unkeyed: [] }; + const y = b[field] ?? { entries: {}, unkeyed: [] }; + const keys = new Set([...Object.keys(x.entries), ...Object.keys(y.entries)]); + let differs = [...keys].some((k) => x.entries[k] !== y.entries[k]); + if (!differs) differs = JSON.stringify(x.unkeyed) !== JSON.stringify(y.unkeyed); + if (differs) fields.push(field); + } + return fields; +}; + +const results = []; +let next = 0; +await Promise.all( + Array.from({ length: Number(values.jobs) }, async () => { + while (next < programs.length) { + const entry = programs[next++]; + const name = basename(entry).replace(/\.ts$/, ""); + const dir = join(out, name); + mkdirSync(dir, { recursive: true }); + const audit = values["no-audit"] ? "-" : join(dir, "audit.json"); + const base = await child( + entry, + join(dir, "base.ir.json"), + "-", + audit, + join(dir, "base.state.json"), + ); + if (!base.ok) { + results.push({ program: name, status: "does not compile" }); + continue; + } + const diffs = {}; + const stateDiffs = {}; + let baseCanonical = null; + for (const seed of seeds) { + const run = await child( + entry, + join(dir, `seed${seed}.ir.json`), + seed, + "-", + join(dir, `seed${seed}.state.json`), + ); + if (!run.ok) { + // A program that lowers in discovery order but not in this one + // differs in content. + diffs[seed] = { failed: run.stderr, content: { lowering: ["fails in this order"] } }; + continue; + } + const diff = differingFunctions( + loadIr(join(dir, "base.ir.json")), + loadIr(join(dir, `seed${seed}.ir.json`)), + ); + const state = differingState( + loadIr(join(dir, "base.state.json")), + loadIr(join(dir, `seed${seed}.state.json`)), + ); + if (state.length > 0) stateDiffs[seed] = state; + if (diff.functions.length > 0 || diff.topLevel.length > 0) { + baseCanonical ??= canonicalModule(loadIr(join(dir, "base.ir.json"))); + diff.content = contentDifferences( + baseCanonical, + canonicalModule(loadIr(join(dir, `seed${seed}.ir.json`))), + ); + diffs[seed] = diff; + } + } + const auditReport = audit === "-" ? {} : JSON.parse(readFileSync(audit, "utf8")); + const changed = new Set(Object.values(diffs).flatMap((diff) => diff.functions ?? [])); + const attribution = {}; + for (const fn of changed) { + const fields = Object.entries(auditReport) + .filter(([, entry]) => entry.functions.includes(fn)) + .map(([field]) => field); + attribution[fn] = fields; + } + results.push({ + program: name, + status: + Object.keys(diffs).length === 0 + ? "order-independent" + : Object.values(diffs).some((diff) => Object.keys(diff.content ?? {}).length > 0) + ? "order-dependent content" + : "order-dependent names", + diffs, + attribution, + sensitiveFields: Object.keys(auditReport), + stateFields: [...new Set(Object.values(stateDiffs).flat())], + }); + process.stderr.write( + `${results.length}/${programs.length} ${name}: ${results.at(-1).status}\n`, + ); + } + }), +); +writeFileSync(join(out, "results.json"), JSON.stringify(results, null, 2)); + +const dependent = results.filter((r) => r.status.startsWith("order-dependent")); +const content = results.filter((r) => r.status === "order-dependent content"); +const byField = new Map(); +let unattributed = 0; +for (const r of dependent) { + const fields = new Set(Object.values(r.attribution).flat()); + if (fields.size === 0) unattributed++; + for (const field of fields) byField.set(field, (byField.get(field) ?? 0) + 1); +} +console.log( + `programs ${results.length}: ${dependent.length} order-dependent (${content.length} beyond names and ids), ${results.filter((r) => r.status === "does not compile").length} do not compile`, +); +for (const r of content) + console.log( + ` ${r.program}: ${Object.values(r.diffs) + .flatMap((diff) => + Object.entries(diff.content ?? {}).map( + ([table, items]) => `${table}: ${items.slice(0, 4).join(", ")}`, + ), + ) + .slice(0, 3) + .join("; ")}`, + ); +for (const [field, count] of [...byField].sort((a, b) => b[1] - a[1])) + console.log(`${String(count).padStart(5)} ${field}`); +console.log(`${String(unattributed).padStart(5)} (no audited field)`); +const failing = + values["fail-on"] === "names" ? dependent : values["fail-on"] === "content" ? content : []; +if (failing.length > 0) { + console.log(`${failing.length} programs lower to different IR in another order`); + process.exitCode = 1; +}