From f65c5518bf808b3ecea69a8eed2781530b41c09e Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Tue, 27 Oct 2026 06:43:19 -0500 Subject: [PATCH] Record each reachable-emit unit's lowering as plain data, checked by a serial mode A unit's record holds its IR in the compact encoding, the entries it added to the helper, lifted-function and other appendable lowerer tables, and the reachability, generic demand, statement and flag events it made. A unit has no record when it grew a table no record carries, changed a field, replaced an entry it did not add, or emitted a diagnostic. Every lowerer collection and scalar field is classified, and a test fails on one that is not. SCRIPTC_LOWER_JOURNAL=serial records every unit and replaces its results with the ones its record carries. scripts/recorded-lowering.mjs (pnpm test:recorded-lowering) compares corpus programs' LLVM text with and without the mode. SCRIPTC_LOWER_JOURNAL_DUMP= also turns the mode on and appends each unit's record to /records.jsonl. Without either, lowering is unchanged. --- package.json | 1 + .../lowering/deferred-module-initializers.ts | 7 + .../frontend/lowering/key-order-witness.ts | 11 + .../compiler/src/frontend/lowering/lowerer.ts | 686 +++++++++++++++++- .../frontend/lowering/unit-journal.test.ts | 282 +++++++ .../src/frontend/lowering/unit-journal.ts | 474 ++++++++++++ packages/compiler/src/frontend/type-mapper.ts | 41 ++ packages/compiler/src/ir/serialize.ts | 11 +- .../compiler/test/recorded-lowering.test.ts | 70 ++ scripts/recorded-lowering.mjs | 145 ++++ 10 files changed, 1718 insertions(+), 10 deletions(-) create mode 100644 packages/compiler/src/frontend/lowering/unit-journal.test.ts create mode 100644 packages/compiler/src/frontend/lowering/unit-journal.ts create mode 100644 packages/compiler/test/recorded-lowering.test.ts create mode 100644 scripts/recorded-lowering.mjs diff --git a/package.json b/package.json index 9b0213b44..e54ee164f 100644 --- a/package.json +++ b/package.json @@ -22,6 +22,7 @@ "test:test262:fetch": "node tests/test262/fetch.mjs", "test:cache-identity": "node tests/harness/cache-identity.mjs", "test:lowering-order": "node scripts/lowering-order.mjs --fail-on names", + "test:recorded-lowering": "node scripts/recorded-lowering.mjs", "test:sandbox": "node scripts/sandbox-test.mjs", "test:sandbox:image": "node scripts/sandbox-image.mjs", "test:watch": "vitest", diff --git a/packages/compiler/src/frontend/lowering/deferred-module-initializers.ts b/packages/compiler/src/frontend/lowering/deferred-module-initializers.ts index 3af054dd6..4d1674127 100644 --- a/packages/compiler/src/frontend/lowering/deferred-module-initializers.ts +++ b/packages/compiler/src/frontend/lowering/deferred-module-initializers.ts @@ -26,6 +26,13 @@ export class DeferredModuleInitializers { /** The last scan of each function (scanOf). */ private readonly scans = new Map(); + /** The sizes of the deferral tables (unit-journal.ts). */ + journalSizes(): number[] { + let initializers = 0; + for (const list of this.pending.values()) initializers += list.length; + return [this.pending.size, initializers, this.registryGlobals.size, this.deferredKeys.size]; + } + defer(lowerer: Lowerer, statement: ts.Statement): IrStmt | null { let key: string; if ( diff --git a/packages/compiler/src/frontend/lowering/key-order-witness.ts b/packages/compiler/src/frontend/lowering/key-order-witness.ts index a5e635e04..8da0f2edf 100644 --- a/packages/compiler/src/frontend/lowering/key-order-witness.ts +++ b/packages/compiler/src/frontend/lowering/key-order-witness.ts @@ -140,6 +140,17 @@ export class KeyOrderNotes { /** Shapes whose table this pass replaced. */ readonly changed = new Set(); + /** The sizes of the notes' tables (unit-journal.ts). */ + journalSizes(): number[] { + return [ + this.literals.size, + this.copies.size, + this.methodResults.size, + this.observed.size, + this.changed.size, + ]; + } + literal(node: IrExpr, segments: KeyOrderSegment[]): IrExpr { this.literals.set(node, segments); return node; diff --git a/packages/compiler/src/frontend/lowering/lowerer.ts b/packages/compiler/src/frontend/lowering/lowerer.ts index 0c63997df..2b40f64ef 100644 --- a/packages/compiler/src/frontend/lowering/lowerer.ts +++ b/packages/compiler/src/frontend/lowering/lowerer.ts @@ -20,6 +20,26 @@ import { } from "./decl-origin.js"; import { isDeclBoundaryFile } from "../decl-projection.js"; import { relieveHeapPressure } from "../../heap-pressure.js"; +import { + checkedTable, + countTable, + fixedScalar, + flagScalar, + functionTable, + irDataTable, + journalDump, + journalMode, + lowerWaveRecorded, + mapTable, + setTable, + waveStats, + writeRecords, + type JournalEvent, + type JournalScalar, + type JournalTable, + type UnitEvents, + type WaveHost, +} from "./unit-journal.js"; import { moduleNamespaceObject, moduleNamespaceValue } from "./module-namespace-elements.js"; import { keyedHelperName, moduleTagNumber, portableRelativePath } from "./stable-names.js"; import { positionTag, StructuralPositions } from "./structural-positions.js"; @@ -2213,6 +2233,7 @@ export class Lowerer { } noteGenericInstanceDemand(inst: GenericInstance): void { + this.unitEvents?.record("functionDemand", inst.name); const owner = this.genericDemandOwner; owner?.functionDemands.push(inst); const ref = this.genericDemandPriority.get(inst) ?? { rank: [4, Number.MAX_SAFE_INTEGER] }; @@ -2228,6 +2249,7 @@ export class Lowerer { } noteGenericClassInstanceDemand(info: ClassInfo): void { + this.unitEvents?.record("classDemand", info.def.name); const owner = this.genericDemandOwner; owner?.classDemands.push(info); const ref = this.genericClassDemandPriority.get(info) ?? { rank: [4, Number.MAX_SAFE_INTEGER] }; @@ -3766,6 +3788,9 @@ export class Lowerer { * promise operand (returnedPromiseAwait). */ private returnAwaitedPromise = false; private readonly virtualEdges = new Map>(); + /** The events of the unit being recorded (unit-journal.ts); null + * otherwise. */ + unitEvents: UnitEvents | null = null; classCanBeConstructed(info: ClassInfo): boolean { return ( @@ -3903,6 +3928,517 @@ export class Lowerer { } return name === undefined ? undefined : this.liftedFnsByName.get(name); } + + /** The tables lowering a unit can grow (unit-journal.ts): the + * appendable tables whose entries a unit's record carries, then the + * tables whose growth leaves the unit without a record. + * Lowerer.unjournaled lists the collections that are in neither. */ + journalTables(): JournalTable[] { + const sizes = (name: string, of: () => readonly number[]): JournalTable => + checkedTable(name, () => { + let hash = 0; + for (const size of of()) hash = (Math.imul(hash, 31) + size) | 0; + return hash; + }); + return [ + functionTable("implicitFns", this.implicitFns, (fn) => + this.implicitFns.some((other) => other.name === fn), + ), + mapTable("equalityHelpers", this.equalityHelpers), + mapTable("arrHofHelpers", this.arrHofHelpers), + irDataTable("arrHofInfo", this.arrHofInfo), + mapTable("unionCallHelpers", this.unionCallHelpers), + mapTable("indexedUnionNumericTwins", this.indexedUnionNumericTwins), + setTable("coercions.checkedNarrows", this.coercions.checkedNarrows), + irDataTable("coercions.checkedNarrowArms", this.coercions.checkedNarrowArms), + mapTable("coercions.retags", this.coercions.retags), + mapTable("coercions.narrows", this.coercions.narrows), + mapTable("coercions.widthNarrows", this.coercions.widthNarrows), + setTable("coercions.widenedReturns", this.coercions.widenedReturns), + mapTable("valueHelpers", this.valueHelpers), + countTable("keyedNameKeys", this.keyedNameKeys), + mapTable("assertHelpers", this.assertHelpers), + mapTable("inspectHelpers", this.inspectHelpers), + setTable("classMethodValueHelpers", this.classMethodValueHelpers), + setTable("freshClosureAdapters", this.freshClosureAdapters), + mapTable("iterDrainHelpers", this.iterDrainHelpers), + mapTable("mapHofHelpers", this.mapHofHelpers), + mapTable("setHofHelpers", this.setHofHelpers), + mapTable("spHofHelpers", this.spHofHelpers), + mapTable("primitiveCtorFns", this.primitiveCtorFns), + mapTable("builtinCallableValueFns", this.builtinCallableValueFns), + mapTable("builtinCallableValueNames", this.builtinCallableValueNames), + countTable("builtinCallableValueLengths", this.builtinCallableValueLengths), + mapTable("regionRunHelpers", this.regionRunHelpers), + mapTable("internTableHelpers", this.internTableHelpers), + functionTable( + "liftedFns", + this.liftedFns, + (fn) => this.liftedFnNamed(fn) !== undefined, + () => { + this.liftedFnsByName.clear(); + this.liftedFnsIndexed = 0; + }, + ), + countTable("closureNameCounts", this.closureNameCounts), + setTable("sharedHelperNames", this.sharedHelperNames), + setTable("sloppyRecordWrites", this.sloppyRecordWrites), + setTable("recordFieldDeletes", this.recordFieldDeletes), + setTable("freezableArrays", this.freezableArrays), + checkedTable("moduleOrder", () => this.moduleOrder.length), + checkedTable("diags", () => this.diags.length), + checkedTable("fnSigsBySymbol", () => this.fnSigsBySymbol.size), + checkedTable("cjsExportsAssignments", () => this.cjsExportsAssignments.size), + checkedTable("inlineRequireCycleCalls", () => this.inlineRequireCycleCalls.size), + checkedTable("staticCallables", () => this.staticCallables.size), + checkedTable("genericFnsBySymbol", () => this.genericFnsBySymbol.size), + checkedTable("objLitGenericFns", () => this.objLitGenericFns.size), + checkedTable("bindingGenericFns", () => this.bindingGenericFns.size), + checkedTable("bindingIdentifiers", () => this.bindingIdentifiers.size), + checkedTable("ownToStringTargets", () => this.ownToStringTargets.size), + checkedTable("neverReassignedCache", () => this.neverReassignedCache.size), + checkedTable("jsBindingWritesByOwner", () => this.jsBindingWritesByOwner.size), + checkedTable("rebindableFunctions", () => this.rebindableFunctions.size), + checkedTable("trapBindings", () => this.trapBindings.size), + checkedTable("deadBindings", () => this.deadBindings.size), + checkedTable("instantiationQueue", () => this.instantiationQueue.length), + checkedTable("genericFunctionDemandOwner", () => this.genericFunctionDemandOwner.size), + checkedTable("genericClassDemandOwner", () => this.genericClassDemandOwner.size), + checkedTable("erasingLambdas", () => this.erasingLambdas.size), + checkedTable("typedArgumentsLambdas", () => this.typedArgumentsLambdas.size), + checkedTable("tupleAliasSymbols", () => this.tupleAliasSymbols.size), + checkedTable("nullishFuncResults", () => this.nullishFuncResults.size), + checkedTable("localClassInstantiations", () => this.localClassInstantiations.length), + checkedTable("localClassInstanceTypes", () => this.localClassInstanceTypes.size), + checkedTable("localClassValueTypes", () => this.localClassValueTypes.size), + checkedTable("dynParamLambdas", () => this.dynParamLambdas.size), + checkedTable("aliasNarrowTypes", () => this.aliasNarrowTypes.size), + checkedTable("classSymbolKeys", () => this.classSymbolKeys.size), + checkedTable("registeredClassSymbols", () => this.registeredClassSymbols.size), + checkedTable("symbolFieldIdentities", () => this.symbolFieldIdentities.size), + checkedTable("httpClientFnBindings", () => this.httpClientFnBindings.size), + checkedTable("emitterComputedPatterns", () => this.emitterComputedPatterns.length), + checkedTable("unstableSwitchBindings", () => this.unstableSwitchBindings.size), + checkedTable("earlyReadVars", () => this.earlyReadVars.size), + checkedTable("runtimeOptionalGlobals", () => this.runtimeOptionalGlobals.size), + checkedTable("cjsExportsThis", () => this.cjsExportsThis.size), + checkedTable("inlineRequireInitDone", () => this.inlineRequireInitDone.size), + checkedTable( + "runtimeOptionalArithmeticGlobals", + () => this.runtimeOptionalArithmeticGlobals.size, + ), + checkedTable("runtimeOptionalBindingTypes", () => this.runtimeOptionalBindingTypes.size), + checkedTable("stageSourceTypes", () => this.stageSourceTypes.size), + checkedTable( + "runtimeOptionalAssignedSymbols", + () => this.runtimeOptionalAssignedSymbols.size, + ), + checkedTable("runtimeOptionalGenericReturns", () => this.runtimeOptionalGenericReturns.size), + checkedTable( + "runtimeOptionalFunctionReturns", + () => this.runtimeOptionalFunctionReturns.size, + ), + checkedTable("contextualFunctionReturns", () => this.contextualFunctionReturns.size), + checkedTable( + "runtimeOptionalArithmeticTypes", + () => this.runtimeOptionalArithmeticTypes.size, + ), + checkedTable("runtimeOptionalFields", () => this.runtimeOptionalFields.size), + checkedTable("dateJsonBindings", () => this.dateJsonBindings.size), + checkedTable( + "runtimeOptionalRecordsKeptDeclared", + () => this.runtimeOptionalRecordsKeptDeclared.size, + ), + checkedTable("runtimeOptionalPatternTypes", () => this.runtimeOptionalPatternTypes.size), + checkedTable("runtimeOptionalReduceTypes", () => this.runtimeOptionalReduceTypes.size), + checkedTable("runtimeOptionalAssertions", () => this.runtimeOptionalAssertions.size), + checkedTable("presentElementCalls", () => this.presentElementCalls.size), + checkedTable("presentElementLoops", () => this.presentElementLoops.size), + checkedTable("errorFirstLiterals", () => this.errorFirstLiterals.size), + checkedTable("anySymbols", () => this.anySymbols.size), + checkedTable("knownErrors", () => this.knownErrors.size), + checkedTable("shapeOrderMetadataFinalizers", () => this.shapeOrderMetadataFinalizers.length), + checkedTable("receiverTypeOverrides", () => this.receiverTypeOverrides.size), + checkedTable("shapeOrderHelperFinalizers", () => this.shapeOrderHelperFinalizers.length), + checkedTable("emitSpecDone", () => this.emitSpecDone.size), + checkedTable("emitSpecQueue", () => this.emitSpecQueue.length), + checkedTable("structuralClassPeers", () => this.structuralClassPeers.size), + checkedTable("methodLambdasInProgress", () => this.methodLambdasInProgress.size), + checkedTable("classMethodValueSelections", () => this.classMethodValueSelections.size), + checkedTable("stdlibGlobalAliases", () => this.stdlibGlobalAliases.size), + checkedTable("cjsAccessorFns", () => this.cjsAccessorFns.size), + checkedTable("cjsAccessorSnapshots", () => this.cjsAccessorSnapshots.size), + checkedTable("cjsSnapshotReady", () => this.cjsSnapshotReady.size), + checkedTable("matchAllIndexBindings", () => this.matchAllIndexBindings.size), + checkedTable("matchAllDrainIndexes", () => this.matchAllDrainIndexes.size), + checkedTable("numericIterators", () => this.numericIterators.size), + checkedTable("classes", () => this.classes.size), + checkedTable("classBySymbol", () => this.classBySymbol.size), + checkedTable("prototypeMethodAccesses", () => this.prototypeMethodAccesses.size), + checkedTable("replacedSymbolMethods", () => this.replacedSymbolMethods.size), + checkedTable("protocolDispatches", () => this.protocolDispatches.size), + checkedTable("protocolOwnedBindings", () => this.protocolOwnedBindings.size), + checkedTable("receiverFreeCallbackFields", () => this.receiverFreeCallbackFields.size), + checkedTable("virtualJsMethods", () => this.virtualJsMethods.size), + checkedTable("inheritedJsFieldWrites", () => this.inheritedJsFieldWrites.size), + checkedTable("virtualJsMethodArity", () => this.virtualJsMethodArity.size), + checkedTable("checkedVirtualJsMethods", () => this.checkedVirtualJsMethods.size), + checkedTable("unionReturnOverrides", () => this.unionReturnOverrides.size), + checkedTable("checkedVirtualJsAccessors", () => this.checkedVirtualJsAccessors.size), + checkedTable("checkedVirtualJsValueReturns", () => this.checkedVirtualJsValueReturns.size), + checkedTable("globalsBySymbol", () => this.globalsBySymbol.size), + checkedTable("expandoMembers", () => this.expandoMembers.size), + checkedTable("globalsByDeclNode", () => this.globalsByDeclNode.size), + checkedTable("globalsList", () => this.globalsList.length), + checkedTable("npmInitActions", () => this.npmInitActions.size), + checkedTable("jsonInitActions", () => this.jsonInitActions.size), + checkedTable("jsonRequireModules", () => this.jsonRequireModules.size), + checkedTable("jsonRequireCalls", () => this.jsonRequireCalls.size), + checkedTable("dynImports", () => this.dynImports.size), + checkedTable("createRequireImports", () => this.createRequireImports.size), + checkedTable("dynNsBuilders", () => this.dynNsBuilders.size), + checkedTable("checkedCallbackParams", () => this.checkedCallbackParams.size), + checkedTable("valueStreamReceivers", () => this.valueStreamReceivers.size), + checkedTable("stagedArguments", () => this.stagedArguments.size), + checkedTable("moduleNsParamOverrides", () => this.moduleNsParamOverrides.size), + checkedTable("fileTag", () => this.fileTag.size), + checkedTable("forkTargets", () => this.forkTargets.length), + checkedTable("workerTargets", () => this.workerTargets.length), + checkedTable("workerTargetIdByPath", () => this.workerTargetIdByPath.size), + checkedTable("experimentalFiles", () => this.experimentalFiles.size), + checkedTable("internTableNames", () => this.internTableNames.size), + checkedTable("threadSlotNames", () => this.threadSlotNames.size), + checkedTable("threadSlotFamilies", () => this.threadSlotFamilies.size), + checkedTable("threadSlotKeys", () => this.threadSlotKeys.size), + checkedTable("forkTargetIdByPath", () => this.forkTargetIdByPath.size), + checkedTable("nsBlocks", () => this.nsBlocks.size), + checkedTable("initNameOf", () => this.initNameOf.size), + checkedTable("cjsModuleIdOf", () => this.cjsModuleIdOf.size), + checkedTable("cjsModuleFiles", () => this.cjsModuleFiles.length), + checkedTable("moduleGuardOf", () => this.moduleGuardOf.size), + checkedTable("asyncInitFiles", () => this.asyncInitFiles.size), + checkedTable("modulePromiseOf", () => this.modulePromiseOf.size), + checkedTable("asyncCycleRepresentativeOf", () => this.asyncCycleRepresentativeOf.size), + checkedTable("asyncCyclePromiseOf", () => this.asyncCyclePromiseOf.size), + checkedTable("runtimeClassValues", () => this.runtimeClassValues.size), + checkedTable("preparedClassFactories", () => this.preparedClassFactories.size), + checkedTable("preparingClassFactories", () => this.preparingClassFactories.size), + checkedTable("varGlobalEntryInits", () => this.varGlobalEntryInits.size), + checkedTable("cycleInitFlags", () => this.cycleInitFlags.size), + checkedTable("classInitFlags", () => this.classInitFlags.size), + checkedTable("cycleInitDeclarations", () => this.cycleInitDeclarations.size), + checkedTable("uninitializedLexicalGlobals", () => this.uninitializedLexicalGlobals.size), + checkedTable("requireInitFlags", () => this.requireInitFlags.size), + checkedTable("requireEarlyReferences", () => this.requireEarlyReferences.length), + checkedTable("sharedHelpers", () => this.sharedHelpers.size), + checkedTable("dynamicFunctions", () => this.dynamicFunctions.length), + checkedTable("dynamicBodies", () => this.dynamicBodies.length), + checkedTable("deferredDiags", () => this.deferredDiags.size), + checkedTable("deferredClassByName", () => this.deferredClassByName.size), + checkedTable("flushedSymbols", () => this.flushedSymbols.size), + checkedTable("alreadyFlushed", () => this.alreadyFlushed.size), + checkedTable("pendingFlushes", () => this.pendingFlushes.size), + checkedTable("checkedJsFiles", () => this.checkedJsFiles.size), + checkedTable("ffiImports", () => this.ffiImports.length), + checkedTable("ffiImportsByName", () => this.ffiImportsByName.size), + checkedTable("externalTypes", () => this.externalTypes.size), + checkedTable("externalTypeSpecifiersByFile", () => this.externalTypeSpecifiersByFile.size), + checkedTable("blockedBindings", () => this.blockedBindings.size), + checkedTable("runtimeFences", () => this.runtimeFences.length), + checkedTable("provenanceElided", () => this.provenanceElided.length), + checkedTable("statsByFile", () => this.statsByFile.size), + checkedTable( + "reachedBlockedBodyDiagnostics", + () => this.reachedBlockedBodyDiagnostics.length, + ), + checkedTable("exprClasses", () => this.exprClasses.length), + checkedTable("exprClassInfoByName", () => this.exprClassInfoByName.size), + checkedTable("collectingExprClasses", () => this.collectingExprClasses.size), + checkedTable("pendingClassExprInits", () => this.pendingClassExprInits.length), + checkedTable("computedClassBases", () => this.computedClassBases.size), + checkedTable("computedCallableBases", () => this.computedCallableBases.size), + checkedTable("mixinInstanceByCall", () => this.mixinInstanceByCall.size), + checkedTable("mixinCollectingCalls", () => this.mixinCollectingCalls.size), + checkedTable("mixinOrdinals", () => this.mixinOrdinals.size), + checkedTable("mixinInstancesByClassNode", () => this.mixinInstancesByClassNode.size), + checkedTable("genericClassByDecl", () => this.genericClassByDecl.size), + checkedTable("collectingClassDeclarations", () => this.collectingClassDeclarations.size), + checkedTable("genericClassInstances", () => this.genericClassInstances.length), + checkedTable("mainPrelude", () => this.mainPrelude.length), + checkedTable("structuralFnInfos", () => this.structuralFnInfos.size), + checkedTable("preservedLayoutTypes", () => this.preservedLayoutTypes.size), + checkedTable("layoutKeptNodes", () => this.layoutKeptNodes.size), + checkedTable("boundAliasInstances", () => this.boundAliasInstances.size), + checkedTable("castGlobalAliases", () => this.castGlobalAliases.size), + checkedTable("methodAccessSites", () => this.methodAccessSites.size), + checkedTable("checkedJsOverrideMethods", () => this.checkedJsOverrideMethods.size), + checkedTable("untypedLoops", () => this.untypedLoops.size), + checkedTable("classBagInitializers", () => this.classBagInitializers.size), + checkedTable("degradedTypes", () => this.degradedTypes.size), + checkedTable("rejectedGroups", () => this.rejectedGroups.size), + checkedTable("lenientPieces", () => this.lenientPieces.size), + checkedTable("dynamicSlots", () => this.dynamicSlots.length), + checkedTable("retriedSymbols", () => this.retriedSymbols.size), + checkedTable("skippedBodies", () => this.skippedBodies.size), + checkedTable("retriedDeclarations", () => this.retriedDeclarations.length), + checkedTable("loweredOnceDiags", () => this.loweredOnceDiags.size), + checkedTable("staticInitsPassed", () => this.staticInitsPassed.size), + checkedTable("deferredCollectors", () => this.deferredCollectors.size), + checkedTable("retriableDiags", () => this.retriableDiags.size), + checkedTable("failedTypes", () => this.failedTypes.size), + checkedTable("retryDegraded", () => this.retryDegraded.length), + checkedTable("declOrigins", () => this.declOrigins.length), + checkedTable("declOriginIds", () => this.declOriginIds.size), + sizes("shapes", () => this.shapes.journalSizes()), + sizes("unions", () => this.unions.journalSizes()), + sizes("keyOrder", () => this.keyOrder.journalSizes()), + sizes("deferredModuleInitializers", () => + this.deferredModuleInitializers === null + ? [] + : this.deferredModuleInitializers.journalSizes(), + ), + ]; + } + + /** The fields a unit's record need not carry: settings, caches of + * program facts that are computed again when needed, state of the unit + * being lowered (keyed by its own IR, nodes or frames), scratch that + * unwinds before the unit ends, and state that applying the unit's events + * rebuilds. */ + static readonly unjournaled: readonly string[] = [ + // settings + "frontendServices", + "ambient", + "overridesAmbient", + "remainder", + "tierMode", + "generalNodeStreams", + "dynNodeStreams", + "targetPlatform", + "startupCrash", + "projectCheckJs", + "libraryCallbacks", + "ffiBindingSymbols", + "priorRetriedSymbols", + "wholeSlotDegradation", + "inferredCollectionTypes", + "foreignElementKeys", + // caches made when first needed + "nanCodedReads", + "mentionsProcessVersions", + "stringIndexBoundsCache", + "functionLocalSubclasses", + "socketEncodingReachable", + "thenablesMayBeDefined", + "unixListenReachable", + "streamPropMutatedSyms", + "readsExitCode", + "requestInitPropMutatedSyms", + "classProtocols", + "cjsSplitRequireBindings", + "cjsWholeExportSymbols", + "cjsLiveMemberExports", + "cjsNamespaces", + "createRequireBindingTargets", + "createRequireElementTargets", + "computedGlobalReads", + "timeValueDateDecisions", + "timeValueDateWrites", + "dynamicOverloadAnswers", + "argumentCountSlots", + "coercions.widenedReturnTypes", + "packageFileMemo", + "slotOriginMemo", + "recordOriginMemo", + "boundaryCastOrder", + "declBoundaryMemo", + "structuredCloneReadCounts", + // state of the unit being lowered + "assertedValues", + "timeValueDateLocals", + "speculatedClosures", + "fsPathPlans", + "fallibleFsCalls", + "failureTouches", + "partialHits", + // sized by journalTables, indexes over journaled tables, and state that + // applying a unit's events rebuilds + "deferredModuleInitializers", + "liftedFnsIndexed", + "liftedFnsByName", + "reachable", + "reachableForArtifacts", + "constructedClasses", + "virtualEdges", + "countedStatements", + "failedStatements", + "genericDemandRoots", + "genericDemandPriority", + "genericClassDemandPriority", + "unitEvents", + // scratch of the lowering in progress, and memos keyed by nodes + "nullishBindings", + "implicitLocalFns", + "builtinClassAliases", + "runtimeOptionalLocals.active", + "runtimeOptionalLocals.scopes", + "narrowing.facts", + "narrowing.verdicts", + "narrowing.effects", + "narrowing.capturedMutable", + "narrowing.staleBindings", + "narrowing.identifierWrites", + "narrowing.widened", + "narrowing.plainContainers", + "runtimeOptionalStorageLocals", + "runtimeOptionalArithmeticLocals", + "runtimeOptionalRoots", + "coercions.planning", + "coercions.copyRetags", + "assertInstances", + "unionLiteralOwners", + "strippedUndefinedArms", + "addedUndefinedArms", + "stdlibSymbols", + "chainRecvByNode", + "neverValued", + "chainNarrowedType", + "chainHandled", + "splitIdentifierUses", + "typeMemo", + "fnStack", + "positions.nodesByKey", + "positions.paths", + "activeStmtLists", + "constructorReturnPlans", + "mixinFnShapes", + ]; + + /** The fields a unit must leave as they were, and the flags it may set + * (unit-journal.ts). */ + journalScalars(): JournalScalar[] { + const flag = flagScalar; + const fixed = fixedScalar; + return [ + flag( + "handleEventReader", + () => this.handleEventReader, + () => (this.handleEventReader = true), + ), + flag( + "writesOntoChildren", + () => this.writesOntoChildren, + () => (this.writesOntoChildren = true), + ), + flag( + "regexGroupResults", + () => this.regexGroupResults, + () => (this.regexGroupResults = true), + ), + flag( + "requiresHistoricalOrderRelower", + () => this.requiresHistoricalOrderRelower, + () => (this.requiresHistoricalOrderRelower = true), + ), + flag( + "requiresGeneralNodeStreams", + () => this.requiresGeneralNodeStreams, + () => (this.requiresGeneralNodeStreams = true), + ), + flag( + "requiresDynNodeStreams", + () => this.requiresDynNodeStreams, + () => (this.requiresDynNodeStreams = true), + ), + flag( + "errorPrototypeValues", + () => this.errorPrototypeValues, + () => (this.errorPrototypeValues = true), + ), + flag( + "usesWorkers", + () => this.usesWorkers, + () => (this.usesWorkers = true), + ), + flag( + "usesWebAssembly", + () => this.usesWebAssembly, + () => (this.usesWebAssembly = true), + ), + flag( + "usesWasmCustomSections", + () => this.usesWasmCustomSections, + () => (this.usesWasmCustomSections = true), + ), + flag( + "usesLifetimeRegions", + () => this.usesLifetimeRegions, + () => (this.usesLifetimeRegions = true), + ), + flag( + "cjsModuleGraphEnabled", + () => this.cjsModuleGraphEnabled, + () => (this.cjsModuleGraphEnabled = true), + ), + flag( + "globalThisComputedWrite", + () => this.globalThisComputedWrite, + () => (this.globalThisComputedWrite = true), + ), + flag( + "untypedIntegrity", + () => this.untypedIntegrity, + () => (this.untypedIntegrity = true), + ), + flag( + "partialDegradedTypesMapped", + () => this.partialDegradedTypesMapped, + () => (this.partialDegradedTypesMapped = true), + ), + fixed("errorAccessorReflection", () => this.errorAccessorReflection), + fixed("subclassLinks", () => this.subclassLinks), + fixed("errorPathCopies", () => this.errorPathCopies), + fixed("errorMethodDispatchRevision", () => this.errorMethodDispatchRevision), + fixed("errorPropertyDispatchRevision", () => this.errorPropertyDispatchRevision), + fixed("childAdditions", () => this.childAdditions), + fixed("emitterEventTable", () => this.emitterEventTable), + fixed("globalThisWrittenNames", () => this.globalThisWrittenNames), + fixed("importRetryModules", () => this.importRetryModules), + fixed("localSubclassIndex", () => this.localSubclassIndex), + // State of the lowering in progress, which a unit restores. + fixed("genericInstantiationPath", () => this.genericInstantiationPath), + fixed("genericDemandOwner", () => this.genericDemandOwner), + fixed("typeParamBindings", () => this.typeParamBindings), + fixed("staticFnNames", () => this.staticFnNames), + fixed("erasingDepth", () => this.erasingDepth), + fixed("instanceResultClass", () => this.instanceResultClass), + fixed("typeParamTsBindings", () => this.typeParamTsBindings), + fixed("implicitParamTypes", () => this.implicitParamTypes), + fixed("fspGlobHead", () => this.fspGlobHead), + fixed("instantiationContext", () => this.instantiationContext), + fixed("suppressStats", () => this.suppressStats), + fixed("emitSpecCtx", () => this.emitSpecCtx), + fixed("assertCallOptions", () => this.assertCallOptions), + fixed("currentClass", () => this.currentClass), + fixed("closureRoot", () => this.closureRoot), + fixed("superConstructorEdge", () => this.superConstructorEdge), + fixed("returnAwaitedPromise", () => this.returnAwaitedPromise), + fixed("collecting", () => this.collecting), + fixed("diagSink", () => this.diagSink), + fixed("speculativeTypeQueryDepth", () => this.speculativeTypeQueryDepth), + fixed("mixinTypeContext", () => this.mixinTypeContext), + fixed("pendingLabels", () => this.pendingLabels), + fixed("onEdge", () => this.onEdge), + fixed("onExprClassCollected", () => this.onExprClassCollected), + fixed("onLateClassCollected", () => this.onLateClassCollected), + fixed("onDeclarationRegistered", () => this.onDeclarationRegistered), + fixed("liftDynFieldPolicy", () => this.liftDynFieldPolicy), + fixed("coercionOriginNode", () => this.coercionOriginNode), + fixed("coercionOriginUnion", () => this.coercionOriginUnion), + fixed("coercionFieldOrigin", () => this.coercionFieldOrigin), + ]; + } + /** Coverage remainder mode: the reachability gate inverts (see wantBody) * and no module is built. */ readonly remainder: boolean; @@ -4053,6 +4589,12 @@ export class Lowerer { countStatement(node: ts.Node, file: string | undefined): void { if (this.countedStatements.has(node)) return; this.countedStatements.add(node); + this.unitEvents?.record( + "statement", + ts.getSourceFileOfNode(node).fileName, + file ?? null, + node.index, + ); this.stats.statementsTotal++; if (file !== undefined) this.bumpFileStat(file, "total"); } @@ -4062,6 +4604,12 @@ export class Lowerer { countStatementFailed(node: ts.Node, file: string | undefined): void { if (this.failedStatements.has(node)) return; this.failedStatements.add(node); + this.unitEvents?.record( + "failedStatement", + ts.getSourceFileOfNode(node).fileName, + file ?? null, + node.index, + ); this.stats.statementsFailed++; if (file !== undefined) this.bumpFileStat(file, "failed"); } @@ -9500,25 +10048,36 @@ export class Lowerer { const rootEdge = this.onEdge; for (const root of extraRoots ?? []) rootEdge?.(root); const shuffle = lowerShuffle(); + const lowerUnitBody = (name: string): IrFunction | null => { + const unit = units.get(name)!; + const priority = metadataPriority.get(name) ?? [3, expressionMetadataOrder++]; + const owner = demandOwner(priority); + return this.withGenericDemandOwner(owner, () => + this.shapes.withDeclaredOrderPriority(priority, () => + this.withClosureRoot(name, unit.lower), + ), + ); + }; + const recorded = journalMode() === "serial" ? waveStats() : null; + const host = recorded === null ? null : this.waveHost(loweredUnits, lowerUnitBody); + const dump = recorded === null ? null : journalDump(); + const recordLines: string[] = []; const drainUnits = (): void => { while (queue.length > 0) { const wave = queue.splice(0); shuffle?.(wave); this.checker.prefetchRoots(wave.flatMap((name) => units.get(name)!.roots)); + if (host !== null && recorded !== null) { + lowerWaveRecorded(host, wave, recorded, dump === null ? null : recordLines); + continue; + } for (const name of wave) { // A body-level poison outside the per-statement catches (a fenced // constructor/method parameter default lowered by declareParams): // every edge fired before poisoning remains retained; the diagnostic // stays recorded and the member stays omitted. try { - const unit = units.get(name)!; - const priority = metadataPriority.get(name) ?? [3, expressionMetadataOrder++]; - const owner = demandOwner(priority); - const fn = this.withGenericDemandOwner(owner, () => - this.shapes.withDeclaredOrderPriority(priority, () => - this.withClosureRoot(name, unit.lower), - ), - ); + const fn = lowerUnitBody(name); if (fn) loweredUnits.set(name, fn); relieveHeapPressure(); } catch (e) { @@ -9569,6 +10128,11 @@ export class Lowerer { this.restoreGenericClassInstanceOrder(clsInstLowered); } classDispatch.finalize(this); + if (dump !== null) writeRecords(dump, recordLines); + if (recorded !== null && process.env["SCRIPTC_TIMING"] === "1") + process.stderr.write( + `scriptc lowering ${JSON.stringify({ phase: "unit-records", ...recorded })}\n`, + ); const orderedUnits = unitsInOrder(); this.settleGenericDemandPriorities(); this.shapes.settleDeclaredOrderPriorities(); @@ -9613,12 +10177,101 @@ export class Lowerer { return { reachable, result: this.finishModule(functions) }; } + /** The lowerer's side of a recorded wave (unit-journal.ts). + * `lowerUnitBody` lowers a unit as reachable emit does. */ + private waveHost( + loweredUnits: Map, + lowerUnitBody: (name: string) => IrFunction | null, + ): WaveHost { + const scalars = this.journalScalars(); + const flags = new Map(scalars.map((scalar) => [scalar.name, scalar])); + let instances = new Map(); + let indexed = -1; + const instanceNamed = (name: string): GenericInstance | undefined => { + if (indexed !== this.instantiationQueue.length) { + instances = new Map(this.instantiationQueue.map(({ inst }) => [inst.name, inst])); + indexed = this.instantiationQueue.length; + } + return instances.get(name); + }; + const classNamed = (name: string): ClassInfo | undefined => { + const info = this.classes.get(name); + return info !== undefined && info.def.name === name ? info : undefined; + }; + const files = new Map(); + const statementNode = (event: JournalEvent): ts.Node | undefined => { + let sf = files.get(event.name); + if (sf === undefined && !files.has(event.name)) { + sf = this.program.getSourceFile(event.name); + files.set(event.name, sf); + } + if (sf === undefined) return undefined; + try { + return sf.file.node(event.index); + } catch { + return undefined; + } + }; + return { + tables: this.journalTables(), + scalars, + lowerRecorded: (name, events) => { + this.unitEvents = events; + try { + const fn = lowerUnitBody(name); + relieveHeapPressure(); + return { fn, poisoned: false }; + } catch (e) { + if (!(e instanceof PoisonError)) throw e; + return { fn: null, poisoned: true }; + } finally { + this.unitEvents = null; + } + }, + store: (name, fn) => { + if (fn) loweredUnits.set(name, fn); + }, + statementCounts: () => ({ + total: this.stats.statementsTotal, + failed: this.stats.statementsFailed, + other: this.stats.functionsSkipped + this.diags.length, + }), + resolves: (event) => { + switch (event.kind) { + case "virtualEdge": + case "classDemand": + return classNamed(event.name) !== undefined; + case "functionDemand": + return instanceNamed(event.name) !== undefined; + case "statement": + case "failedStatement": + return statementNode(event) !== undefined; + case "flag": + return flags.get(event.name)?.set !== undefined; + default: + return true; + } + }, + }; + } + /** Reachability hook (see emitReachable): fires when lowering resolves a * reference to a lowerable body. */ noteEdge(name: string): void { // Called as a plain function: the instance is not its receiver. const onEdge = this.onEdge; - if (onEdge) onEdge(name); + const events = this.unitEvents; + if (events === null) { + if (onEdge) onEdge(name); + return; + } + events.record(this.superConstructorEdge ? "superConstructorEdge" : "edge", name); + events.depth++; + try { + if (onEdge) onEdge(name); + } finally { + events.depth--; + } } /** A subclass constructor's super() edge: the base constructor runs, but @@ -9639,6 +10292,21 @@ export class Lowerer { * them). */ noteVirtualEdge(info: ClassInfo, method: string): void { if (!this.onEdge) return; + const events = this.unitEvents; + if (events === null) { + this.virtualEdgeOf(info, method); + return; + } + events.record("virtualEdge", info.def.name, method); + events.depth++; + try { + this.virtualEdgeOf(info, method); + } finally { + events.depth--; + } + } + + private virtualEdgeOf(info: ClassInfo, method: string): void { if (this.constructedClasses) { const methods = this.virtualEdges.get(info) ?? new Set(); methods.add(method); diff --git a/packages/compiler/src/frontend/lowering/unit-journal.test.ts b/packages/compiler/src/frontend/lowering/unit-journal.test.ts new file mode 100644 index 000000000..74dc57b1a --- /dev/null +++ b/packages/compiler/src/frontend/lowering/unit-journal.test.ts @@ -0,0 +1,282 @@ +import { expect, test } from "vitest"; +import type { Program, SourceFile } from "../ts7/adapter.js"; +import { Lowerer } from "./lowerer.js"; +import { + checkedTable, + flagScalar, + functionTable, + lowerWaveRecorded, + mapTable, + recordUnit, + setTable, + waveStats, + type WaveHost, +} from "./unit-journal.js"; +import type { IrFunction } from "../../ir/ir.js"; + +const lowerer = (): Lowerer => + new Lowerer( + { getTypeChecker: () => ({}), isSourceFileDefaultLibrary: () => false } as unknown as Program, + { fileName: "/island/entry.ts" } as SourceFile, + [], + ); + +const isCollection = (value: unknown): boolean => + value instanceof Map || value instanceof Set || Array.isArray(value); + +/** The registries whose collections journalSizes covers, with the + * collections each one's journalSizes reads. */ +const sized: Record = { + shapes: [ + "keyedNameKeys", + "byKey", + "byId", + "declaredOrderPriority", + "declaredOrderCandidates", + "derivedDeclaredOrderFinalizers", + "shapes", + "recIds", + "pendingRec", + "extendedNames", + ], + unions: ["keyedNameKeys", "byKey", "byId", "armTags", "unions", "recIds", "pendingRec"], + keyOrder: ["literals", "copies", "methodResults", "observed", "changed"], +}; + +/** Scratch of the registries' own mapping (it unwinds before mapType returns). */ +const scratch = new Set([ + "shapes.inProgress", + "shapes.currentDeclaredOrderPriority", + "unions.inProgress", +]); + +/** Owned objects whose collections are listed one by one. */ +const owned = new Set(["coercions", "narrowing", "runtimeOptionalLocals", "positions"]); + +test("every collection of the lowerer is journaled, checked or left out by name", () => { + const subject = lowerer(); + const named = new Set([ + ...subject.journalTables().map((table) => table.name), + ...Lowerer.unjournaled, + ]); + const missing: string[] = []; + const fields = subject as unknown as Record; + for (const field of Object.keys(fields)) { + const value = fields[field]; + if (isCollection(value)) { + if (!named.has(field)) missing.push(field); + continue; + } + if (value === null || typeof value !== "object") continue; + const inner = value as Record; + const covered = sized[field]; + if (covered === undefined && !owned.has(field)) continue; + for (const key of Object.keys(inner)) { + const path = `${field}.${key}`; + if (!isCollection(inner[key]) || scratch.has(path)) continue; + if (covered !== undefined ? !covered.includes(key) : !named.has(path)) missing.push(path); + } + } + expect(missing).toEqual([]); +}); + +test("every scalar field of the lowerer is journaled or left out by name", () => { + const subject = lowerer(); + const named = new Set([ + ...subject.journalScalars().map((scalar) => scalar.name), + ...Lowerer.unjournaled, + ]); + const fields = subject as unknown as Record; + const missing = Object.keys(fields).filter((field) => { + const value = fields[field]; + if (isCollection(value) || typeof value === "function") return false; + const scalar = value === null || value === undefined || typeof value !== "object"; + return scalar && !named.has(field); + }); + expect(missing).toEqual([]); +}); + +test("every name the lists hold is a field of the lowerer", () => { + const subject = lowerer(); + const fields = subject as unknown as Record; + const names = [ + ...subject.journalTables().map((table) => table.name), + ...subject.journalScalars().map((scalar) => scalar.name), + ...Lowerer.unjournaled, + ]; + const missing = names.filter((name) => { + const [head, key] = name.split("."); + if (!(head! in fields)) return true; + if (key === undefined) return false; + const inner = fields[head!]; + return inner === null || typeof inner !== "object" || !(key in inner); + }); + expect(missing).toEqual([]); +}); + +const fn = (name: string, ret: "f64" | "string" = "f64"): IrFunction => + ({ + name, + params: [], + returnType: { kind: ret }, + locals: [], + body: [], + loc: { file: "a.ts", start: 0, end: 0 }, + }) as unknown as IrFunction; + +test("a map table hands over the entries added after a size", () => { + const map = new Map([["a", "%h.1"]]); + const table = mapTable("helpers", map); + map.set("b", "%h.2"); + map.set("c", "%h.3"); + const entries = table.entries!(1)!; + expect(entries.length).toBe(2); + const parent = new Map([ + ["a", "%h.1"], + ["c", "%h.3"], + ]); + mapTable("helpers", parent).apply!(entries); + expect([...parent]).toEqual([ + ["a", "%h.1"], + ["c", "%h.3"], + ["b", "%h.2"], + ]); + table.truncate!(1); + expect([...map]).toEqual([["a", "%h.1"]]); +}); + +test("a held table notices a replaced entry", () => { + const map = new Map([["a", "%h.1"]]); + const held = mapTable("helpers", map).hold!(); + map.set("b", "%h.2"); + expect(held.same()).toBe(true); + map.set("a", "%h.9"); + expect(held.same()).toBe(false); + const set = new Set(["x"]); + const heldSet = setTable("names", set).hold!(); + set.delete("x"); + set.add("x"); + expect(heldSet.same()).toBe(true); + set.delete("x"); + expect(heldSet.same()).toBe(false); +}); + +test("a function table keeps the function an earlier unit made", () => { + const worker = [fn("%a")]; + const table = functionTable("lifted", worker, () => false); + worker.push(fn("%b"), fn("%c")); + const entries = table.entries!(1)!; + const parent = [fn("%c", "string")]; + const target = functionTable("lifted", parent, (name) => + parent.some((other) => other.name === name), + ); + target.apply!(entries); + expect(parent.map((f) => [f.name, f.returnType.kind])).toEqual([ + ["%c", "string"], + ["%b", "f64"], + ]); +}); + +test("a set table adds what the parent lacks", () => { + const set = new Set(["x"]); + const table = setTable("names", set); + set.add("y"); + const parent = new Set(); + setTable("names", parent).apply!(table.entries!(1)!); + expect([...parent]).toEqual(["y"]); +}); + +/** A wave host over one helper map, one checked list and one flag, whose + * units are lowered by `lower`. */ +function host(lower: (name: string, state: HostState) => IrFunction | null): { + host: WaveHost; + state: HostState; +} { + const state: HostState = { helpers: new Map(), checked: [], flag: false, stored: new Map() }; + return { + state, + host: { + tables: [ + mapTable("helpers", state.helpers), + checkedTable("checked", () => state.checked.length), + ], + scalars: [ + flagScalar( + "flag", + () => state.flag, + () => (state.flag = true), + ), + ], + lowerRecorded: (name, events) => { + events.record("edge", `%callee.${name}`); + return { fn: lower(name, state), poisoned: false }; + }, + store: (name, unitFn) => { + if (unitFn !== null) state.stored.set(name, unitFn); + }, + statementCounts: () => ({ total: 0, failed: 0, other: 0 }), + resolves: () => true, + }, + }; +} + +interface HostState { + helpers: Map; + checked: string[]; + flag: boolean; + stored: Map; +} + +test("a recorded wave keeps what the records carry, in wave order", () => { + const made = new Map(); + const { host: subject, state } = host((name, held) => { + held.helpers.set(`key.${name}`, `%h.${name}`); + if (name === "b") held.flag = true; + if (name === "c") held.checked.push(name); + const unitFn = fn(`%${name}`); + made.set(name, unitFn); + return unitFn; + }); + const stats = waveStats(); + lowerWaveRecorded(subject, ["a", "b", "c"], stats); + expect([...state.helpers.keys()]).toEqual(["key.a", "key.b", "key.c"]); + expect(stats).toMatchObject({ + waves: 1, + units: 3, + recorded: 2, + unrecorded: 1, + reasons: { checked: 1 }, + }); + // Recorded units keep the function decoded from their record; the unit + // without one keeps the function lowering made. + expect(state.stored.get("a")).not.toBe(made.get("a")); + expect(state.stored.get("a")).toEqual(made.get("a")); + expect(state.stored.get("c")).toBe(made.get("c")); +}); + +test("a unit's record lists its events, flags and added entries", () => { + const { host: subject } = host((name, held) => { + held.helpers.set(`key.${name}`, `%h.${name}`); + held.flag = true; + return fn(`%${name}`); + }); + const { record } = recordUnit(subject, "a"); + expect(record).not.toBeNull(); + expect(record!.events.map((event) => [event.kind, event.name])).toEqual([ + ["edge", "%callee.a"], + ["flag", "flag"], + ]); + expect(record!.tables).toEqual({ helpers: [`"key.a"\n"%h.a"`] }); + expect(JSON.parse(JSON.stringify(record))).toEqual(record); +}); + +test("a unit that changes an entry it did not add has no record", () => { + const { host: subject, state } = host((name, held) => { + held.helpers.set("shared", `%h.${name}`); + return null; + }); + state.helpers.set("shared", "%h.first"); + const unit = recordUnit(subject, "a"); + expect(unit.record).toBeNull(); + expect(unit.reasons).toEqual(["helpers:replaced"]); +}); diff --git a/packages/compiler/src/frontend/lowering/unit-journal.ts b/packages/compiler/src/frontend/lowering/unit-journal.ts new file mode 100644 index 000000000..8c8de3001 --- /dev/null +++ b/packages/compiler/src/frontend/lowering/unit-journal.ts @@ -0,0 +1,474 @@ +/* Per-unit records of reachable-emit lowering (Lowerer.emitReachable). + * + * A unit's record is what another process needs to reach the state the + * unit's lowering reached: the unit's IR, the entries it added to the + * tables that collect lowering results (Lowerer.journalTables), and the + * calls it made into reachability, generic demand and statement counting + * (JournalEvent). Records hold only strings, numbers and names, so they can + * be written to a file and read by another process or another build. + * + * A unit has no record when its lowering changed state that no record + * carries: a table that is not appendable grew, a field changed, an entry + * held before the unit was replaced, or a diagnostic was emitted. Such a + * unit lowers again wherever its record would have been applied. + * + * SCRIPTC_LOWER_JOURNAL=serial records every unit while it lowers, and + * replaces the unit's results with the ones its record carries: the + * function decoded from the record and the table entries added again from + * it. The build's output is then the output of applying the records, which + * tests compare with ordinary lowering. SCRIPTC_LOWER_JOURNAL_DUMP= + * turns the serial mode on and appends each unit's record to + * /records.jsonl, one JSON line per unit in lowering order; a unit + * without a record has a line with its name and reasons. */ +import { appendFileSync, mkdirSync } from "node:fs"; +import { join } from "node:path"; +import type { IrFunction } from "../../ir/ir.js"; +import { deserializeFunction, encodeFunction } from "../../ir/serialize.js"; + +/** A function as records hold it, or null when it does not encode. Every + * function a record holds goes through this pair. */ +export function encodeUnitFunction(fn: IrFunction): string | null { + return encodeFunction(fn) ?? null; +} + +/** A function from its record encoding (encodeUnitFunction). */ +export function decodeUnitFunction(text: string): IrFunction { + return deserializeFunction(text); +} + +/** A call a unit made while it lowered, which applying its record makes + * again. */ +export interface JournalEvent { + readonly kind: + | "edge" + | "superConstructorEdge" + | "virtualEdge" + | "functionDemand" + | "classDemand" + | "statement" + | "failedStatement" + | "flag"; + /** The edge's body name, the class of a virtual edge or class demand, the + * generic instance's name, a counted statement's source file, or the + * flag's field. */ + readonly name: string; + /** The virtual edge's method, or the file a statement counts toward. */ + readonly detail: string | null; + /** A counted statement's node index in its source file. */ + readonly index: number; +} + +/** One unit's record. */ +export interface UnitRecord { + readonly name: string; + /** The unit's function in the compact IR encoding, or null when the unit + * lowered to nothing. */ + readonly fn: string | null; + /** True when the unit's lowering ended in a poison: its diagnostics and + * the edges it fired before poisoning stay, and it has no function. */ + readonly poisoned: boolean; + readonly events: readonly JournalEvent[]; + /** The encoded entries the unit added, by table name, for the tables it + * grew. */ + readonly tables: Readonly>; +} + +/** A table a unit's lowering can grow. An appendable table only adds + * entries at the end (Map and Set insertion order, array pushes), so a + * unit's additions are the entries past the size the table had when the + * unit started. A checked table has only `size`: when it grows, the unit + * has no record. */ +export interface JournalTable { + readonly name: string; + size(): number; + /** The encoded entries from position `from` on, or null when one of + * them does not encode. */ + readonly entries?: (from: number) => string[] | null; + /** Adds the encoded entries whose keys this process's table lacks. */ + readonly apply?: (entries: readonly string[]) => void; + /** Removes the entries from position `from` on. */ + readonly truncate?: (from: number) => void; + /** The entries the table holds now. */ + readonly hold?: () => HeldEntries; +} + +/** A table's entries at one point (JournalTable.hold). */ +export interface HeldEntries { + /** Whether the table's first entries are still these keys and values. */ + same(): boolean; +} + +/** A field a unit's lowering must leave as it found it (fixedScalar), or a + * flag it may set, which applying its record sets too (flagScalar). */ +export interface JournalScalar { + readonly name: string; + /** Takes the field's value; the result answers how the field changed. */ + mark(): ScalarMark; + readonly set?: () => void; +} + +/** A field's value when a unit started. */ +export interface ScalarMark { + /** Not at all, set (a flag that went from false to true), or otherwise. */ + change(): "same" | "set" | "changed"; +} + +/** A field a unit's lowering must leave as it found it. */ +export function fixedScalar(name: string, get: () => T): JournalScalar { + return { + name, + mark: () => { + const start = get(); + return { change: () => (get() === start ? "same" : "changed") }; + }, + }; +} + +/** A flag a unit's lowering may set. */ +export function flagScalar(name: string, get: () => boolean, set: () => void): JournalScalar { + return { + name, + mark: () => { + const start = get(); + return { + change: () => { + const now = get(); + return now === start ? "same" : now && !start ? "set" : "changed"; + }, + }; + }, + set, + }; +} + +/** The events of the unit being recorded (Lowerer.unitEvents). */ +export class UnitEvents { + readonly events: JournalEvent[] = []; + /** Calls into reachability nest (an edge can fire virtual edges). Only + * the outermost call is recorded, because making it again makes the + * nested ones. */ + depth = 0; + + record(kind: JournalEvent["kind"], name: string, detail: string | null = null, index = 0): void { + if (this.depth === 0) this.events.push({ kind, name, detail, index }); + } +} + +/** The lowerer's side of a recorded wave (Lowerer.emitReachable). */ +export interface WaveHost { + readonly tables: readonly JournalTable[]; + readonly scalars: readonly JournalScalar[]; + /** Lowers `name` as reachable emit does, with `events` recording: the + * function, or null, and whether the unit poisoned. Throws what lowering + * throws other than a poison. */ + lowerRecorded(name: string, events: UnitEvents): { fn: IrFunction | null; poisoned: boolean }; + /** Keeps a unit's function. */ + store(name: string, fn: IrFunction | null): void; + /** Statement totals, and the count of diagnostics and skipped functions. */ + statementCounts(): { total: number; failed: number; other: number }; + /** Whether `event` names something this process holds. */ + resolves(event: JournalEvent): boolean; +} + +/** Whether reachable emit records its units (SCRIPTC_LOWER_JOURNAL=serial, + * or a SCRIPTC_LOWER_JOURNAL_DUMP directory). */ +export function journalMode(): "serial" | "off" { + return process.env["SCRIPTC_LOWER_JOURNAL"] === "serial" || journalDump() !== null + ? "serial" + : "off"; +} + +/** The directory recorded waves write their records to + * (SCRIPTC_LOWER_JOURNAL_DUMP), or null. */ +export function journalDump(): string | null { + const dir = process.env["SCRIPTC_LOWER_JOURNAL_DUMP"]; + return dir === undefined || dir === "" ? null : dir; +} + +/** Appends `lines` to `dir`/records.jsonl. */ +export function writeRecords(dir: string, lines: readonly string[]): void { + if (lines.length === 0) return; + mkdirSync(dir, { recursive: true }); + appendFileSync(join(dir, "records.jsonl"), `${lines.join("\n")}\n`); +} + +/** What a build's recorded waves did (SCRIPTC_TIMING). */ +export interface WaveStats { + waves: number; + units: number; + /** Units whose record stands for them. */ + recorded: number; + /** Units without a record. */ + unrecorded: number; + /** Per table, field or check, the units it left without a record. */ + reasons: Record; +} + +export function waveStats(): WaveStats { + return { waves: 0, units: 0, recorded: 0, unrecorded: 0, reasons: {} }; +} + +/** What lowering one unit with its events recording produced. */ +export interface RecordedUnit { + /** The unit's record, or null when the unit has none. */ + readonly record: UnitRecord | null; + /** Why the unit has no record. */ + readonly reasons: readonly string[]; + /** The function lowering made. */ + readonly fn: IrFunction | null; + /** Each table's size before the unit. */ + readonly before: readonly number[]; +} + +/** Lowers `name` with its events recording, and makes its record. The unit + * has a record when every table that grew is appendable and holds the + * entries it held before, every field is as it was or a flag that was set, + * statement totals grew by the recorded statements alone, no diagnostic was + * emitted, every event resolves and the function encodes. */ +export function recordUnit(host: WaveHost, name: string): RecordedUnit { + const tables = host.tables; + const before = tables.map((table) => table.size()); + const held = tables.map((table) => (table.hold === undefined ? null : table.hold())); + const marks = host.scalars.map((scalar) => scalar.mark()); + const counts = host.statementCounts(); + const recorder = new UnitEvents(); + const { fn, poisoned } = host.lowerRecorded(name, recorder); + const reasons: string[] = []; + const grown: number[] = []; + tables.forEach((table, index) => { + const check = held[index] ?? null; + if (check !== null && !check.same()) reasons.push(`${table.name}:replaced`); + if (table.size() === before[index]) return; + if (table.entries === undefined) reasons.push(table.name); + else grown.push(index); + }); + const events = [...recorder.events]; + host.scalars.forEach((scalar, index) => { + const change = marks[index]!.change(); + if (change === "set") events.push({ kind: "flag", name: scalar.name, detail: null, index: 0 }); + else if (change === "changed") reasons.push(scalar.name); + }); + let statements = 0; + let failed = 0; + for (const event of events) { + if (event.kind === "statement") statements++; + else if (event.kind === "failedStatement") failed++; + if (!host.resolves(event)) reasons.push(`event:${event.kind}`); + } + const now = host.statementCounts(); + if (now.total - counts.total !== statements || now.failed - counts.failed !== failed) + reasons.push("statements"); + if (now.other !== counts.other) reasons.push("diagnostics"); + const text = fn === null ? null : encodeUnitFunction(fn); + if (fn !== null && text === null) reasons.push("encoding"); + const added: Record = {}; + if (reasons.length === 0) + for (const index of grown) { + const table = tables[index]!; + const entries = table.entries!(before[index]!); + if (entries === null) reasons.push(`${table.name}:encoding`); + else added[table.name] = entries; + } + if (reasons.length > 0) return { record: null, reasons, fn, before }; + return { record: { name, fn: text, poisoned, events, tables: added }, reasons, fn, before }; +} + +/** Lowers `wave` with every unit recorded. A unit with a record keeps the + * results its record carries: the entries it added are taken out of their + * tables and added again from the record, and its function is the one + * decoded from the record. A unit without one keeps what lowering made. + * `lines`, when given, receives each unit's record as a JSON line. */ +export function lowerWaveRecorded( + host: WaveHost, + wave: readonly string[], + stats: WaveStats, + lines: string[] | null = null, +): void { + stats.waves++; + const tables = host.tables; + for (const name of wave) { + const unit = recordUnit(host, name); + stats.units++; + const record = unit.record; + if (record === null) { + lines?.push(JSON.stringify({ name, reasons: unit.reasons })); + stats.unrecorded++; + for (const reason of unit.reasons) stats.reasons[reason] = (stats.reasons[reason] ?? 0) + 1; + host.store(name, unit.fn); + continue; + } + lines?.push(JSON.stringify(record)); + stats.recorded++; + tables.forEach((table, index) => { + const entries = record.tables[table.name]; + if (entries === undefined) return; + table.truncate!(unit.before[index]!); + table.apply!(entries); + }); + host.store(name, record.fn === null ? null : decodeUnitFunction(record.fn)); + } +} + +/** An appendable table over a Map from strings to strings, such as helper + * names by key. */ +export function mapTable(name: string, map: Map): JournalTable { + return codecTable( + name, + map, + (value) => JSON.stringify(value), + (text) => JSON.parse(text) as string, + ); +} + +/** An appendable table over a Map from strings to numbers. */ +export function countTable(name: string, map: Map): JournalTable { + return codecTable( + name, + map, + (value) => String(value), + (text) => Number(text), + ); +} + +/** An appendable table over a Map from strings to IR data, encoded as + * JSON. JSON writes a number that is not finite as null, so a value whose + * encoding holds a null does not encode. */ +export function irDataTable(name: string, map: Map): JournalTable { + return codecTable( + name, + map, + (value) => { + const text = JSON.stringify(value); + return /[:,[]null\b/.test(text) ? null : text; + }, + (text) => JSON.parse(text) as V, + ); +} + +/** An appendable table over a Map from strings to values that `encode` + * writes, or answers null for, and `decode` reads back. An entry is its key + * as a JSON string, a line break and its value: a JSON string holds no line + * break. A value that does not encode leaves its unit without a record. */ +export function codecTable( + name: string, + map: Map, + encode: (value: V) => string | null, + decode: (text: string) => V, +): JournalTable { + return { + name, + size: () => map.size, + entries: (from) => { + const out: string[] = []; + let index = 0; + for (const [key, value] of map) { + if (index++ < from) continue; + const text = encode(value); + if (text === null) return null; + out.push(`${JSON.stringify(key)}\n${text}`); + } + return out; + }, + apply: (entries) => { + for (const entry of entries) { + const split = entry.indexOf("\n"); + const key = JSON.parse(entry.slice(0, split)) as string; + if (!map.has(key)) map.set(key, decode(entry.slice(split + 1))); + } + }, + truncate: (from) => { + const keys = [...map.keys()].slice(from); + for (const key of keys) map.delete(key); + }, + hold: () => { + const keys = [...map.keys()]; + const values = [...map.values()]; + return { + same: () => { + let index = 0; + for (const [key, value] of map) { + if (index >= keys.length) break; + if (key !== keys[index] || value !== values[index]) return false; + index++; + } + return index === keys.length; + }, + }; + }, + }; +} + +/** An appendable table over a Set of strings. */ +export function setTable(name: string, set: Set): JournalTable { + return { + name, + size: () => set.size, + entries: (from) => [...set].slice(from), + apply: (entries) => { + for (const entry of entries) set.add(entry); + }, + truncate: (from) => { + const entries = [...set].slice(from); + for (const entry of entries) set.delete(entry); + }, + hold: () => { + const entries = [...set]; + return { + same: () => { + let index = 0; + for (const entry of set) { + if (index >= entries.length) break; + if (entry !== entries[index]) return false; + index++; + } + return index === entries.length; + }, + }; + }, + }; +} + +/** An appendable list of functions this process holds one of per name. A + * function whose name the list already holds is the same function, made by + * an earlier unit. `changed` runs after the list changes, for indexes over + * it. */ +export function functionTable( + name: string, + functions: IrFunction[], + has: (name: string) => boolean, + changed: () => void = () => {}, +): JournalTable { + return { + name, + size: () => functions.length, + entries: (from) => { + const out: string[] = []; + for (const fn of functions.slice(from)) { + const text = encodeUnitFunction(fn); + if (text === null) return null; + out.push(text); + } + return out; + }, + apply: (entries) => { + for (const entry of entries) { + const fn = decodeUnitFunction(entry); + if (!has(fn.name)) functions.push(fn); + } + changed(); + }, + truncate: (from) => { + while (functions.length > from) functions.pop(); + changed(); + }, + hold: () => { + const held = [...functions]; + return { same: () => held.every((fn, index) => functions[index] === fn) }; + }, + }; +} + +/** A table whose growth leaves the unit without a record. */ +export function checkedTable(name: string, size: () => number): JournalTable { + return { name, size }; +} diff --git a/packages/compiler/src/frontend/type-mapper.ts b/packages/compiler/src/frontend/type-mapper.ts index 40ca10b38..1d505f08f 100644 --- a/packages/compiler/src/frontend/type-mapper.ts +++ b/packages/compiler/src/frontend/type-mapper.ts @@ -324,6 +324,30 @@ export class ShapeRegistry { private currentDeclaredOrderPriority: DeclaredOrderPriority = [0, 0]; /** All interned shapes in first-seen order. */ readonly shapes: IrRecordShape[] = []; + + /** The sizes of the registry's tables, which only grow while shapes are + * interned (unit-journal.ts). */ + journalSizes(): number[] { + let candidates = 0; + for (const list of this.declaredOrderCandidates.values()) candidates += list.length; + let extended = 0; + for (const names of this.extendedNames.values()) extended += names.length; + return [ + this.revision, + this.shapes.length, + this.byKey.size, + this.byId.size, + this.keyedNameKeys.size, + this.declaredOrderPriority.size, + this.declaredOrderCandidates.size, + candidates, + this.derivedDeclaredOrderFinalizers.length, + this.recIds.size, + this.pendingRec.size, + this.extendedNames.size, + extended, + ]; + } /** ts.Types currently being mapped — a BACK-REFERENCE to one of these is * the recursive knot (`interface TreeNode { children: TreeNode[] }`): * mapType answers a NAMED RECURSIVE SHAPE (recursiveRef) whose fields @@ -675,6 +699,23 @@ export class UnionRegistry { private readonly armTags = new Map>(); /** All interned unions in first-seen order. */ readonly unions: IrUnionDef[] = []; + + /** The sizes of the registry's tables (ShapeRegistry.journalSizes). */ + journalSizes(): number[] { + let tags = 0; + for (const byArm of this.armTags.values()) tags += byArm.size; + return [ + this.revision, + this.unions.length, + this.byKey.size, + this.byId.size, + this.keyedNameKeys.size, + this.armTags.size, + tags, + this.recIds.size, + this.pendingRec.size, + ]; + } /** ts.Types currently being mapped — a back-reference to one is the * recursive knot passing through a union (`type Tree = Leaf | Branch` * whose Branch arm carries `Tree[]`, the optional field `a?: A` of a diff --git a/packages/compiler/src/ir/serialize.ts b/packages/compiler/src/ir/serialize.ts index 314f258d6..185472b89 100644 --- a/packages/compiler/src/ir/serialize.ts +++ b/packages/compiler/src/ir/serialize.ts @@ -42,7 +42,8 @@ function encodeNumber(_key: string, value: unknown): unknown { return value; } -function encodeFunction(fn: IrFunction): string | undefined { +/** One function in the compact encoding (deserializeFunction reads it). */ +export function encodeFunction(fn: IrFunction): string | undefined { return ordinaryNumbers(fn) ? JSON.stringify(fn) : JSON.stringify(fn, encodeNumber); } @@ -127,6 +128,14 @@ function checkedVersion(mod: IrModule): IrModule { return mod; } +/** A function from its compact encoding (encodeFunction). */ +export function deserializeFunction(json: string): IrFunction { + // Only an encoding that holds a number sentinel needs the reviver. + return ( + json.includes('"$nonfinite"') ? JSON.parse(json, decodeNumber) : JSON.parse(json) + ) as IrFunction; +} + export function deserializeModule(json: string): IrModule { return checkedVersion(JSON.parse(json, decodeNumber) as IrModule); } diff --git a/packages/compiler/test/recorded-lowering.test.ts b/packages/compiler/test/recorded-lowering.test.ts new file mode 100644 index 000000000..a40f33964 --- /dev/null +++ b/packages/compiler/test/recorded-lowering.test.ts @@ -0,0 +1,70 @@ +import { mkdtemp, readFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, expect, test } from "vitest"; +import { compile } from "../src/index.js"; + +// Programs whose units add the entries unit records carry (lifted closures +// and helpers, interned helper names, statement counts, generic demands, +// virtual edges) and grow tables that leave a unit without a record (new +// records and unions, generic instances, key-order notes). +const PROGRAMS = [ + "1003-json-parse-unions.ts", + "1200-regex-test-basics.ts", + "1304-errors-uncaught.ts", + "1573-tdz-scalar-forward-capture.ts", + "1631-inspect-arrays.ts", + "4900-console-instances.ts", + "5047-generator-delegation-iterables-typed.ts", + "5390-interface-loop-element-lifetimes.ts", + "6496-key-order-built-and-parsed.ts", + "6916-index-record-spread-overrides.ts", + "6951-unknown-cast-class-methods.ts", +]; + +async function lower(program: string, dir: string, label: string): Promise { + const entry = join(import.meta.dirname, "../../../tests/corpus", program); + const outPath = join(dir, `${program}.${label}.ir.json`); + const result = await compile(entry, { outDir: dir, outPath, outputKind: "ir" }); + if (!result.ok) + throw new Error(result.diagnostics.map((d) => `${d.code}: ${d.message}`).join("\n")); + return readFile(outPath, "utf8"); +} + +afterEach(() => { + delete process.env["SCRIPTC_LOWER_JOURNAL"]; +}); + +test("units that keep what their records carry give the IR lowering gives", async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-recorded-lowering-")); + try { + for (const program of PROGRAMS) { + delete process.env["SCRIPTC_LOWER_JOURNAL"]; + const plain = await lower(program, dir, "plain"); + process.env["SCRIPTC_LOWER_JOURNAL"] = "serial"; + const recorded = await lower(program, dir, "recorded"); + expect(recorded, program).toBe(plain); + } + } finally { + await rm(dir, { recursive: true, force: true }); + } +}, 300_000); + +test("two builds of a program write the same records", async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-recorded-lowering-dump-")); + try { + const dumps: string[] = []; + for (const label of ["first", "second"]) { + process.env["SCRIPTC_LOWER_JOURNAL_DUMP"] = join(dir, label); + await lower("1573-tdz-scalar-forward-capture.ts", dir, label); + dumps.push(await readFile(join(dir, label, "records.jsonl"), "utf8")); + } + expect(dumps[0]!.length).toBeGreaterThan(0); + expect(dumps[1]).toBe(dumps[0]); + for (const line of dumps[0]!.trimEnd().split("\n")) + expect(typeof (JSON.parse(line) as { name: unknown }).name).toBe("string"); + } finally { + delete process.env["SCRIPTC_LOWER_JOURNAL_DUMP"]; + await rm(dir, { recursive: true, force: true }); + } +}, 300_000); diff --git a/scripts/recorded-lowering.mjs b/scripts/recorded-lowering.mjs new file mode 100644 index 000000000..ec9a528da --- /dev/null +++ b/scripts/recorded-lowering.mjs @@ -0,0 +1,145 @@ +#!/usr/bin/env node +/* Checks that unit records stand for the units they record. + * + * Each corpus program is compiled to LLVM text twice: as usual, and with + * SCRIPTC_LOWER_JOURNAL=serial, where every reachable-emit unit that has a + * record keeps the function and table entries its record carries + * (frontend/lowering/unit-journal.ts). The two texts must be the same. + * + * node scripts/recorded-lowering.mjs [--stride 1] [--offset 0] [--limit N] + * [--jobs 8] [--entry ]... [--out ] + * + * --stride and --offset sample the corpus entries in name order. Needs a + * built compiler (pnpm build). Writes /results.json, prints the programs + * whose texts differ and the units each mode recorded, and exits with status + * 1 when a program's texts differ or only one mode builds it. */ +import { spawn } from "node:child_process"; +import { globSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { cpus, tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import { parseArgs } from "node:util"; + +const root = fileURLToPath(new URL("..", import.meta.url)); +const dist = join(root, "packages/compiler/dist"); + +if (process.argv[2] === "--child") { + const [, , , entry, outPath] = process.argv; + const { compile } = await import(pathToFileURL(join(dist, "index.js")).href); + const outDir = join(tmpdir(), `scriptc-recorded-lowering-${process.pid}`); + try { + const result = await compile(entry, { outDir, outPath, outputKind: "llvm" }); + if (!result.ok) + process.stderr.write( + `${result.diagnostics.map((d) => `${d.code}: ${d.message}`).join("\n")}\n`, + ); + process.exitCode = result.ok ? 0 : 3; + } finally { + rmSync(outDir, { recursive: true, force: true }); + } +} else { + await main(); +} + +async function main() { + const { values } = parseArgs({ + options: { + stride: { type: "string", default: "1" }, + offset: { type: "string", default: "0" }, + limit: { type: "string", default: "100000" }, + jobs: { type: "string", default: String(Math.max(1, Math.floor(cpus().length / 2))) }, + entry: { type: "string", multiple: true, default: [] }, + out: { type: "string", default: join(tmpdir(), "scriptc-recorded-lowering") }, + }, + }); + const out = resolve(values.out); + mkdirSync(out, { recursive: true }); + const corpus = join(root, "tests/corpus"); + const sampled = + values.entry.length > 0 + ? values.entry.map((entry) => resolve(entry)) + : ["ts", "js", "mjs", "cjs"] + .flatMap((ext) => [ + ...globSync(join(corpus, `*.${ext}`)), + ...globSync(join(corpus, `*/main.${ext}`)), + ]) + .sort() + .filter((_, index) => index % Number(values.stride) === Number(values.offset)) + .slice(0, Number(values.limit)); + + const child = (entry, outPath, journal) => + new Promise((done) => { + const env = { ...process.env, SCRIPTC_NO_CACHE: "1", SCRIPTC_TIMING: "1" }; + if (journal) env["SCRIPTC_LOWER_JOURNAL"] = "serial"; + else delete env["SCRIPTC_LOWER_JOURNAL"]; + const proc = spawn( + process.execPath, + ["--max-old-space-size=8192", fileURLToPath(import.meta.url), "--child", entry, outPath], + { stdio: ["ignore", "ignore", "pipe"], env }, + ); + let stderr = ""; + proc.stderr.on("data", (chunk) => (stderr += chunk)); + proc.on("close", (code) => done({ ok: code === 0, stderr })); + }); + + /** The unit-records line SCRIPTC_TIMING prints, or null. */ + const recordStats = (stderr) => { + const line = stderr.split("\n").find((text) => text.includes('"phase":"unit-records"')); + return line === undefined ? null : JSON.parse(line.slice(line.indexOf("{"))); + }; + + const results = []; + let next = 0; + const work = async () => { + for (;;) { + const index = next++; + if (index >= sampled.length) return; + const entry = sampled[index]; + const name = entry.slice(corpus.length + 1).replaceAll("/", "__"); + const plainPath = join(out, `${name}.plain.ll`); + const recordedPath = join(out, `${name}.recorded.ll`); + const plain = await child(entry, plainPath, false); + const recorded = await child(entry, recordedPath, true); + const result = { entry, plain: plain.ok, recorded: recorded.ok, stats: null, same: false }; + result.stats = recordStats(recorded.stderr); + if (plain.ok && recorded.ok) + result.same = readFileSync(plainPath, "utf8") === readFileSync(recordedPath, "utf8"); + if (result.same || (!plain.ok && !recorded.ok)) { + rmSync(plainPath, { force: true }); + rmSync(recordedPath, { force: true }); + } + if (plain.ok !== recorded.ok) + result.error = (plain.ok ? recorded.stderr : plain.stderr).slice(-2000); + results.push(result); + } + }; + await Promise.all(Array.from({ length: Number(values.jobs) }, work)); + results.sort((a, b) => (a.entry < b.entry ? -1 : a.entry > b.entry ? 1 : 0)); + writeFileSync(join(out, "results.json"), JSON.stringify(results, null, 2)); + + const built = results.filter((result) => result.plain && result.recorded); + const differ = built.filter((result) => !result.same); + const oneMode = results.filter((result) => result.plain !== result.recorded); + const totals = { units: 0, recorded: 0, unrecorded: 0, reasons: {} }; + for (const { stats } of results) { + if (stats === null) continue; + totals.units += stats.units; + totals.recorded += stats.recorded; + totals.unrecorded += stats.unrecorded; + for (const [reason, count] of Object.entries(stats.reasons)) + totals.reasons[reason] = (totals.reasons[reason] ?? 0) + count; + } + console.log( + `${results.length} programs, ${built.length} built, ${differ.length} differ, ${oneMode.length} build in one mode only`, + ); + console.log( + `units: ${totals.units}, recorded ${totals.recorded}, without a record ${totals.unrecorded}`, + ); + for (const [reason, count] of Object.entries(totals.reasons).sort((a, b) => b[1] - a[1])) + console.log(` ${reason}: ${count}`); + for (const result of differ) console.log(`differs: ${result.entry}`); + for (const result of oneMode) + console.log(`builds ${result.plain ? "plain" : "recorded"} only: ${result.entry}`); + console.log(`results: ${join(out, "results.json")}`); + if (differ.length > 0 || oneMode.length > 0) process.exitCode = 1; +}