diff --git a/native/llvm-codegen/src/emit.cpp b/native/llvm-codegen/src/emit.cpp index 1ab0f6ecd..d3d868fae 100644 --- a/native/llvm-codegen/src/emit.cpp +++ b/native/llvm-codegen/src/emit.cpp @@ -1241,6 +1241,7 @@ emitSplitCodegen(const EmitOptions &Options, Module &M, TargetMachine &Machine, size_t Count = Options.Outputs.size(); std::optional Split = splitForCodegen(M, Count); std::string Identifier = M.getModuleIdentifier(); + std::string SourceFileName = M.getSourceFileName(); Triple TargetTriple = M.getTargetTriple(); DataLayout Layout = M.getDataLayout(); std::vector>> Temporaries(Count); @@ -1274,6 +1275,7 @@ emitSplitCodegen(const EmitOptions &Options, Module &M, TargetMachine &Machine, keepCodegenPart(*Part, Split->PartOf, I); } else { Part = std::make_unique(Identifier, Context); + Part->setSourceFileName(SourceFileName); Part->setTargetTriple(TargetTriple); Part->setDataLayout(Layout); } diff --git a/packages/cli/src/bootstrap.ts b/packages/cli/src/bootstrap.ts index 516d923f7..87736778f 100644 --- a/packages/cli/src/bootstrap.ts +++ b/packages/cli/src/bootstrap.ts @@ -74,7 +74,11 @@ async function tryFastPath(): Promise { values["target-cpu"] !== undefined || (values["external-types"] ?? []).length > 0 || // The full CLI validates the Node island policy flags. - hasNodeIslandFlags(values) + hasNodeIslandFlags(values) || + // The full compiler writes the representation report and keys the heap + // census build (backend/llvm/repr-report.ts, heap-census.ts). + (process.env["SCRIPTC_REPR_REPORT"] ?? "") !== "" || + (process.env["SCRIPTC_HEAP_CENSUS"] ?? "") !== "" ) return null; const backend = values.backend; diff --git a/packages/cli/test/executable-cache.test.ts b/packages/cli/test/executable-cache.test.ts index ee8027765..eef8e97a0 100644 --- a/packages/cli/test/executable-cache.test.ts +++ b/packages/cli/test/executable-cache.test.ts @@ -198,3 +198,57 @@ test.skipIf(process.platform === "win32")( }, 120_000, ); + +test.skipIf(process.platform !== "linux")( + "a warm executable cache still builds the heap census and writes the representation report", + async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-cli-diagnostic-cache-")); + const cacheRoot = join(dir, "cache"); + const entry = join(dir, "main.ts"); + const outPath = join(dir, "program"); + try { + await mkdir(cacheRoot, { mode: 0o700 }); + // `new` and construction through a class value both count. + await writeFile( + entry, + [ + "class Item { constructor(readonly id: number) {} }", + "const kind: typeof Item = Item;", + "const items = [new Item(1), new kind(2), new kind(3)];", + "console.log(items.length);", + "", + ].join("\n"), + ); + const build = async (env: NodeJS.ProcessEnv = {}): Promise => + ( + await execFileAsync( + process.execPath, + ["--import", tsxLoader, cliEntry, "build", entry, "-o", outPath], + { + env: { ...process.env, SCRIPTC_CACHE_DIR: cacheRoot, SCRIPTC_TIMING: "1", ...env }, + maxBuffer: 4 * 1024 * 1024, + }, + ) + ).stderr; + expect(await build()).toContain("scriptc lowering"); + expect(await build()).not.toContain("scriptc lowering"); + + // The census build is its own cache entry. + const census = join(dir, "census.tsv"); + await build({ SCRIPTC_HEAP_CENSUS: "1" }); + const { stdout } = await execFileAsync(outPath, [], { + env: { ...process.env, SCRIPTC_HEAP_CENSUS: census }, + }); + expect(stdout).toBe("3\n"); + expect(await readFile(census, "utf8")).toMatch(/^new\tItem\t\t3\t[1-9]\d*$/m); + + // A report is written even when the executable is cached. + const report = join(dir, "repr.tsv"); + await build({ SCRIPTC_REPR_REPORT: report }); + expect(await readFile(report, "utf8")).toMatch(/^field\tItem\tid\t/m); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }, + 600_000, +); diff --git a/packages/compiler/src/backend/llvm/classes.ts b/packages/compiler/src/backend/llvm/classes.ts index 695acc273..94cfcaf25 100644 --- a/packages/compiler/src/backend/llvm/classes.ts +++ b/packages/compiler/src/backend/llvm/classes.ts @@ -1712,6 +1712,8 @@ export function emitClassObjDefs( `define internal ptr @${mangleCtorThunk(className)}(ptr %class${paramDecls ? ", " + paramDecls : ""}) ${FN_ATTRS} { ; construct thunk ${className}`, `entry:`, ` %o = call ptr @${mangleClassNew(className)}()`, + // Construction through a class value: the census counts it here. + ...(host.heapCensus?.newLines(className) ?? []), ...(meta.def.localCaptures !== undefined ? [ ` %class.owned = call ptr @scr_classobj_retain_v(ptr %class)`, diff --git a/packages/compiler/src/backend/llvm/emit-statements.ts b/packages/compiler/src/backend/llvm/emit-statements.ts index 74c84ae86..7732fb8be 100644 --- a/packages/compiler/src/backend/llvm/emit-statements.ts +++ b/packages/compiler/src/backend/llvm/emit-statements.ts @@ -617,6 +617,14 @@ export function emitStmtBody(emitter: LlEmitter, s: IrStmt): void { // (The guard precedes the value here: the nullable store // evaluates it itself.) emitter.emitStoreGuard(obj.name, s); + emitter.heapCensus?.countStore( + (line) => B.line(line), + () => B.tmp(), + s.className, + s.field, + s.value, + null, + ); emitter.emitNullableFieldStore(obj.name, s.className, s.field, nullable, s.value); break; } @@ -636,8 +644,17 @@ export function emitStmtBody(emitter: LlEmitter, s: IrStmt): void { ? emitter.recordFieldInline(s.shapeId, s.field) : emitter.inlineUnions.classField(s.className, s.field), ); - if (s.kind === "fieldSet") emitter.emitStoreGuard(obj.name, s); - else if (emitter.publishedTypes?.recordGuarded(s.shapeId) === true) + if (s.kind === "fieldSet") { + emitter.emitStoreGuard(obj.name, s); + emitter.heapCensus?.countStore( + (line) => B.line(line), + () => B.tmp(), + s.className, + s.field, + s.value, + v.name, + ); + } else if (emitter.publishedTypes?.recordGuarded(s.shapeId) === true) emitter.emitPublishedFieldGuard(obj.name, null, s.field); const { ptr, type, inline } = s.kind === "fieldSet" diff --git a/packages/compiler/src/backend/llvm/emitter.ts b/packages/compiler/src/backend/llvm/emitter.ts index f3b141b9a..cd72644bd 100644 --- a/packages/compiler/src/backend/llvm/emitter.ts +++ b/packages/compiler/src/backend/llvm/emitter.ts @@ -130,6 +130,8 @@ import { } from "./int32-abi.js"; import { type Int32Local } from "./int32-locals.js"; import { findIntegerClones, NO_INTEGER_CLONES, type IntegerClones } from "./integer-clones.js"; +import { writeRepresentationReport } from "./repr-report.js"; +import { heapCensusOf, type HeapCensus } from "./heap-census.js"; import { exactInteger, narrowInteger, widenInteger, type LlInteger } from "./integer-values.js"; import { findInitializerBindings } from "../../ir/initializer-bindings.js"; import { findConstantNumericTables, type ConstantNumericTable } from "../../ir/constant-tables.js"; @@ -797,7 +799,9 @@ function emitModuleText(input: IrModule, options: LlvmTargetOptions): string[] { for (let excluded = options.unwindExcluded ?? new Set(); ;) { const emitter = moduleEmitter(input, { ...options, unwindExcluded: excluded }); try { - return emitter.emitParts(); + const parts = emitter.emitParts(); + writeRepresentationReport(emitter); + return parts; } catch (error) { if (!(error instanceof UnwindAbiViolation)) throw error; const names = new Set(excluded); @@ -1614,6 +1618,8 @@ export class LlEmitter { readonly inlineUnions: InlineUnionFields; /** Unions whose values are nullable arm pointers (nullable-unions.ts). */ readonly nullableUnions: NullableUnions; + /** The counters of a build instrumented for the heap census, or null. */ + readonly heapCensus: HeapCensus | null; /** The union table without nullable unions: the stack-box analyses. */ readonly boxedUnionsById: Map; readonly recordsById = new Map(); @@ -2006,6 +2012,8 @@ export class LlEmitter { options.debugSources === undefined, this.nodeIndex, ); + this.heapCensus = heapCensusOf(mod, this.nullableUnions); + for (const decl of this.heapCensus?.declarations() ?? []) this.declare(decl); this.mockTimers = this.runtimeFeatures.mockTimers; this.debug = options.debugSources === undefined @@ -2363,6 +2371,7 @@ export class LlEmitter { virtualEntry: (implFn) => this.virtualEntry(implFn), programLifetime: this.programLifetime, regionAliasTag: (word) => this.memoryAliasAttachment(`region.${word}`), + heapCensus: this.heapCensus, }; // An instance of a class that extends Array is an array: a slot of its // element array type can hold it, and the collector visits it there as @@ -2646,7 +2655,8 @@ export class LlEmitter { emitParts(): string[] { const parts = emitParts(this); const bridged = this.unwindAbi.size > 0 ? bridgePlainUnwindCalls(parts) : parts; - return this.cpuDispatch ? cloneForCpuDispatch(bridged) : bridged; + const out = this.cpuDispatch ? cloneForCpuDispatch(bridged) : bridged; + return this.heapCensus === null ? out : [...out, this.heapCensus.definitions().join("\n")]; } /** The dispose-observer gates the emitted frees and record writers test @@ -4686,6 +4696,7 @@ export class LlEmitter { `${p} = getelementptr inbounds %${classStructSym(className)}, ptr ${objName}, i64 0, i32 ${index}`, ); this.markFieldPointer(p, field); + this.heapCensus?.countAccess((line) => this.B.line(line), className, field); if (compressed) this.compressedFieldPtrs.add(p); return { ptr: p, type, inline: this.inlineUnions.classField(className, field), compressed }; } diff --git a/packages/compiler/src/backend/llvm/expr-calls.ts b/packages/compiler/src/backend/llvm/expr-calls.ts index 18181de50..9b2f2e4d8 100644 --- a/packages/compiler/src/backend/llvm/expr-calls.ts +++ b/packages/compiler/src/backend/llvm/expr-calls.ts @@ -1105,6 +1105,7 @@ export function emitCallExpr( ); const o = B.tmp(); B.line(`${o} = call ptr @${host.allocSym(mangleClassNew(e.className), e)}()`); + host.heapCensus?.countNew((line) => B.line(line), e.className); const out = host.own({ name: o, type: e.type }); emitConstruction(host, e, o, ctor); return out; diff --git a/packages/compiler/src/backend/llvm/heap-census.ts b/packages/compiler/src/backend/llvm/heap-census.ts new file mode 100644 index 000000000..b55e9341b --- /dev/null +++ b/packages/compiler/src/backend/llvm/heap-census.ts @@ -0,0 +1,241 @@ +/* The heap census. A program built with SCRIPTC_HEAP_CENSUS set counts, for + * ranking representation work: + * - the instances of each class that `new` or a class value constructs, + * with the class's size; + * - for each array-typed class field, its stores (with the stored array's + * length in buckets and the total of those lengths), the stores whose + * value is an array literal written at the store, and its accesses; + * - for each record-typed class field, its stores and accesses. + * Accesses count every field address the backend computes for the field, + * so a store is also an access. At exit the program writes one + * tab-separated line per counter to the file the run's SCRIPTC_HEAP_CENSUS + * names, or to stderr. Linux only: the runtime calls the writer through a + * weak reference, as it does the profile writer. Counters are module + * globals updated with monotonic atomic adds, so they include every + * thread's work. Ordinary builds emit none of it. */ +import type { IrClassDef, IrExpr, IrModule, IrType } from "../../ir/ir.js"; +import { classStructSym } from "./classes.js"; +import { llvmBytes, wtf8Bytes } from "../literals.js"; +import type { NullableUnions } from "./nullable-unions.js"; + +/** Whether this build counts the heap census: SCRIPTC_HEAP_CENSUS is set + * and nonempty. The executable cache keys on it. */ +export function heapCensusRequested(env: NodeJS.ProcessEnv = process.env): boolean { + const flag = env["SCRIPTC_HEAP_CENSUS"]; + return flag !== undefined && flag !== ""; +} + +/** The census of the module being built, or null when it is off. */ +export function heapCensusOf(mod: IrModule, unions: NullableUnions): HeapCensus | null { + if (!heapCensusRequested() || mod.lib !== undefined) return null; + return new HeapCensus(mod, unions); +} + +/** A NUL-terminated i8 array constant of `text`. */ +function cString(text: string): string { + const bytes = wtf8Bytes(text); + return `[${bytes.length + 1} x i8] c"${llvmBytes(bytes)}"`; +} + +const LENGTH_BUCKETS = ["len0", "len1", "len2", "len3-4", "len5-8", "len9-16", "len17+"]; +/** Upper bounds of LENGTH_BUCKETS but the last. */ +const BUCKET_LIMITS = [0, 1, 2, 4, 8, 16]; + +interface FieldCounters { + /** First counter: stores, accesses, then (arrays) the literal stores, + * the length total and the buckets. */ + base: number; + array: boolean; + /** The stored value is a pointer that is NULL for the union's unit arm. */ + nullable: boolean; +} + +export class HeapCensus { + private readonly names: string[] = []; + private readonly sizes: (string | null)[] = []; + private readonly instances = new Map(); + private readonly fields = new Map(); + + constructor(mod: IrModule, unions: NullableUnions) { + for (const cls of mod.classes ?? []) { + this.instances.set(cls.name, this.add(`new\t${cls.name}\t`, classStructSym(cls.name))); + for (const field of cls.fields) this.addField(cls, field.name, field.type, unions); + } + } + + /** Counts one instance of `className`. */ + countNew(line: (text: string) => void, className: string): void { + for (const text of this.newLines(className)) line(text); + } + + /** The lines that count one instance of `className`: a `new`, or a + * construct thunk for class values (`newValue`). */ + newLines(className: string): string[] { + const index = this.instances.get(className); + return index === undefined ? [] : [this.increment(index, "1")]; + } + + /** Counts an access of a field; true when the field is counted. */ + countAccess(line: (text: string) => void, className: string, field: string): void { + const counters = this.fields.get(`${className}\0${field}`); + if (counters !== undefined) line(this.increment(counters.base + 1, "1")); + } + + /** Counts a store of `value` (an LLVM pointer, computed from `source`) + * into a field: its length when the field holds arrays. `tmp` names fresh + * temporaries. */ + countStore( + line: (text: string) => void, + tmp: () => string, + className: string, + field: string, + source: IrExpr, + value: string | null, + ): void { + const counters = this.fields.get(`${className}\0${field}`); + if (counters === undefined) return; + line(this.increment(counters.base, "1")); + if (!counters.array) return; + const literal = source.kind === "unionWrap" ? source.value : source; + if (literal.kind === "arrayLit") line(this.increment(counters.base + 2, "1")); + if (value === null) return; + // A NULL pointer (the unit arm) counts as length 0: its length is read + // through a select that points NULL at a zeroed array header. + let array = value; + if (counters.nullable) { + const isNull = tmp(); + array = tmp(); + line(`${isNull} = icmp eq ptr ${value}, null`); + line(`${array} = select i1 ${isNull}, ptr @sc_census_zero_arr, ptr ${value}`); + } + const lenPtr = tmp(), + len32 = tmp(), + len = tmp(); + line(`${lenPtr} = getelementptr inbounds %ScrArr, ptr ${array}, i32 0, i32 1`); + line(`${len32} = load i32, ptr ${lenPtr}`); + line(`${len} = zext i32 ${len32} to i64`); + line(this.increment(counters.base + 3, len)); + // The bucket: the number of limits below the length. + let bucket = "0"; + for (const limit of BUCKET_LIMITS) { + const above = tmp(), + step = tmp(), + next = tmp(); + line(`${above} = icmp ugt i64 ${len}, ${limit}`); + line(`${step} = zext i1 ${above} to i64`); + line(`${next} = add i64 ${bucket}, ${step}`); + bucket = next; + } + const slot = tmp(); + line(`${slot} = add i64 ${bucket}, ${counters.base + 4}`); + const ptr = tmp(); + line( + `${ptr} = getelementptr inbounds [${this.names.length} x i64], ptr @sc_census, i64 0, i64 ${slot}`, + ); + line(`atomicrmw add ptr ${ptr}, i64 1 monotonic`); + } + + /** The module definitions: the counters, their names, the zero array + * header NULL stores read their length from, and the writer the runtime + * calls at exit (scr_census_write). */ + definitions(): string[] { + const n = this.names.length; + const out = [`@sc_census = internal global [${n} x i64] zeroinitializer`]; + // A zeroed %ScrArr: length 0. + out.push(`@sc_census_zero_arr = internal constant %ScrArr zeroinitializer`); + const names = this.names.map((name, i) => { + const bytes = cString(name); + out.push(`@sc_census_name.${i} = private unnamed_addr constant ${bytes}`); + return `ptr @sc_census_name.${i}`; + }); + out.push(`@sc_census_names = internal constant [${n} x ptr] [${names.join(", ")}]`); + const sizes = this.sizes.map((sym) => + sym === null ? "i64 0" : `i64 ptrtoint (ptr getelementptr (%${sym}, ptr null, i32 1) to i64)`, + ); + out.push(`@sc_census_sizes = internal constant [${n} x i64] [${sizes.join(", ")}]`); + const env = cString("SCRIPTC_HEAP_CENSUS"); + out.push(`@sc_census_env = private unnamed_addr constant ${env}`); + out.push(`@sc_census_mode = private unnamed_addr constant ${cString("w")}`); + out.push(`@sc_census_fmt = private unnamed_addr constant ${cString("%s\t%llu\t%llu\n")}`); + out.push( + `define void @scr_census_write() {`, + `entry:`, + ` %path = call ptr @getenv(ptr @sc_census_env)`, + ` %named = icmp ne ptr %path, null`, + ` br i1 %named, label %open, label %err`, + `open:`, + ` %file = call ptr @fopen(ptr %path, ptr @sc_census_mode)`, + ` %opened = icmp ne ptr %file, null`, + ` br i1 %opened, label %write, label %err`, + `err:`, + ` %stderr = load ptr, ptr @stderr`, + ` br label %write`, + `write:`, + ` %out = phi ptr [ %file, %open ], [ %stderr, %err ]`, + ` br label %loop`, + `loop:`, + ` %i = phi i64 [ 0, %write ], [ %next, %loop ]`, + ` %namep = getelementptr inbounds [${n} x ptr], ptr @sc_census_names, i64 0, i64 %i`, + ` %name = load ptr, ptr %namep`, + ` %countp = getelementptr inbounds [${n} x i64], ptr @sc_census, i64 0, i64 %i`, + ` %count = load atomic i64, ptr %countp monotonic, align 8`, + ` %sizep = getelementptr inbounds [${n} x i64], ptr @sc_census_sizes, i64 0, i64 %i`, + ` %size = load i64, ptr %sizep`, + ` %bytes = mul i64 %count, %size`, + ` call i32 (ptr, ptr, ...) @fprintf(ptr %out, ptr @sc_census_fmt, ptr %name, i64 %count, i64 %bytes)`, + ` %next = add i64 %i, 1`, + ` %done = icmp eq i64 %next, ${n}`, + ` br i1 %done, label %end, label %loop`, + `end:`, + ` call i32 @fflush(ptr %out)`, + ` ret void`, + `}`, + ); + return out; + } + + /** External declarations the writer uses. */ + declarations(): string[] { + return [ + "declare ptr @getenv(ptr)", + "declare ptr @fopen(ptr, ptr)", + "declare i32 @fprintf(ptr, ptr, ...)", + "declare i32 @fflush(ptr)", + "@stderr = external global ptr", + ]; + } + + private add(name: string, sizeSym: string | null = null): number { + this.names.push(name); + this.sizes.push(sizeSym); + return this.names.length - 1; + } + + private addField(cls: IrClassDef, field: string, type: IrType, unions: NullableUnions): void { + let array = type.kind === "array"; + let nullable = false; + let record = type.kind === "record"; + if (type.kind === "union") { + const single = unions.single(type.unionId); + if (single !== null && single.nullTag < 0) { + array = single.arm.kind === "array"; + record = single.arm.kind === "record"; + nullable = true; + } + } + if (!array && !record) return; + const key = `${cls.name}\t${field}`; + const base = this.add(`store\t${key}`); + this.add(`access\t${key}`); + if (array) { + this.add(`literal\t${key}`); + this.add(`elements\t${key}`); + for (const bucket of LENGTH_BUCKETS) this.add(`${bucket}\t${key}`); + } + this.fields.set(`${cls.name}\0${field}`, { base, array, nullable }); + } + + private increment(index: number, by: string): string { + return `atomicrmw add ptr getelementptr inbounds ([${this.names.length} x i64], ptr @sc_census, i64 0, i64 ${index}), i64 ${by} monotonic`; + } +} diff --git a/packages/compiler/src/backend/llvm/repr-report.test.ts b/packages/compiler/src/backend/llvm/repr-report.test.ts new file mode 100644 index 000000000..ec7b9b5ed --- /dev/null +++ b/packages/compiler/src/backend/llvm/repr-report.test.ts @@ -0,0 +1,65 @@ +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, expect, test } from "vitest"; +import { compile, deserializeModule } from "../../index.js"; +import { emitLlvmModule } from "./emitter.js"; +import { REPR_REPORT_HEADER } from "./repr-report.js"; + +const saved = process.env["SCRIPTC_REPR_REPORT"]; +afterEach(() => { + if (saved === undefined) delete process.env["SCRIPTC_REPR_REPORT"]; + else process.env["SCRIPTC_REPR_REPORT"] = saved; +}); + +test("the representation report names each slot's storage and why a number stays a double", async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-repr-report-")); + try { + const entry = join(dir, "main.ts"), + outPath = join(dir, "main.ir.json"), + report = join(dir, "repr.tsv"); + await writeFile( + entry, + ` +class Box { + id: number; + hits: number; + items: number[]; + constructor(id: number, items: number[]) { this.id = id | 0; this.hits = 0; this.items = items; } +} +function scale(value: number, by: number = 2): number { return (value * by) | 0; } +function half(value: number): number { return value / 2; } +const boxes: Box[] = []; +for (let i = 0; i < 4; i++) boxes.push(new Box(i, [i, i + 1])); +for (const b of boxes) b.hits++; +console.log(scale(boxes[1]!.id), scale(3, 4), half(boxes.length), boxes[2]!.items.length, boxes[3]!.hits); +`, + ); + const result = await compile(entry, { outDir: dir, outPath, outputKind: "ir" }); + if (!result.ok) throw new Error(result.diagnostics.map((d) => d.message).join("\n")); + const mod = deserializeModule(await readFile(outPath, "utf8")); + process.env["SCRIPTC_REPR_REPORT"] = report; + emitLlvmModule(mod); + const lines = (await readFile(report, "utf8")).trimEnd().split("\n"); + expect(lines[0]).toBe(REPR_REPORT_HEADER); + const rows = lines.slice(1).map((line) => line.split("\t")); + const row = (kind: string, owner: string, name: string) => + rows.find((r) => r[0] === kind && r[1] === owner && r[2] === name); + expect(row("field", "Box", "id")?.slice(3, 5)).toEqual(["f64", "i32"]); + // `++` on a field keeps it a double, and the report says so. + expect(row("field", "Box", "hits")?.[4]).toBe("double"); + expect(row("field", "Box", "hits")?.[5]).not.toBe(""); + expect(row("field", "Box", "items")?.slice(3, 5)).toEqual(["array", "ptr"]); + expect(row("param", "scale", "value")?.[4]).toBe("i32"); + // A defaulted parameter is a `number | undefined` box; the variant for + // calls that pass a number takes an i32. + expect(row("param", "scale", "by")?.slice(3, 5)).toEqual(["f64 | undefined", "box"]); + expect(row("param", "scale.nativeNumber", "by")?.[4]).toBe("i32"); + // An array length may exceed the int32 range. + expect(row("param", "half", "value")?.[4]).toBe("double"); + expect(row("param", "half", "value")?.[5]).toMatch(/^argument in .* is not int32$/); + expect(row("return", "half", "")?.slice(4)).toEqual(["double", "return is not int32"]); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); diff --git a/packages/compiler/src/backend/llvm/repr-report.ts b/packages/compiler/src/backend/llvm/repr-report.ts new file mode 100644 index 000000000..8d6e48f30 --- /dev/null +++ b/packages/compiler/src/backend/llvm/repr-report.ts @@ -0,0 +1,141 @@ +/* The representation report (SCRIPTC_REPR_REPORT=): after a module is + * emitted, one tab-separated line per class field, function parameter, + * function result and union, naming the storage the backend chose for it + * and, for numbers kept as doubles, the reason the int32 analysis recorded + * (int32-slots.ts Int32Slots.reasons). It is a diagnostic for ranking + * representation work; nothing reads it back. */ +import { writeFileSync } from "node:fs"; +import { typeKey, type IrFunction, type IrModule, type IrType } from "../../ir/ir.js"; +import type { LlEmitter } from "./emitter.js"; +import { classFieldTypes } from "./classes.js"; +import { INLINE_UNION_FIELD } from "./inline-unions.js"; + +export const REPR_REPORT_HEADER = "kind\towner\tname\ttype\tstorage\treason"; + +/** The file SCRIPTC_REPR_REPORT names, or null. A build that writes the + * report skips the executable cache, whose hit would emit no module. */ +export function representationReportPath(env: NodeJS.ProcessEnv = process.env): string | null { + const path = env["SCRIPTC_REPR_REPORT"]; + return path === undefined || path === "" ? null : path; +} + +/** Writes the report of `emitter`'s module when SCRIPTC_REPR_REPORT names a + * file. */ +export function writeRepresentationReport(emitter: LlEmitter): void { + const path = representationReportPath(); + if (path !== null) writeFileSync(path, representationReport(emitter)); +} + +export function representationReport(emitter: LlEmitter): string { + const mod = emitter.mod; + const slots = emitter.int32Slots; + const lines = [REPR_REPORT_HEADER]; + const row = (...cells: string[]): void => { + lines.push(cells.map((cell) => cell.replace(/[\t\n]/g, " ")).join("\t")); + }; + const typeLabel = typeLabeler(mod); + // A union value that is not a plain box: a nullable or shape pointer. + const unionStorage = (t: IrType): string | null => { + if (t.kind !== "union") return null; + return emitter.nullableUnions.get(t.unionId)?.repr ?? "box"; + }; + + for (const cls of mod.classes ?? []) { + const meta = emitter.classMeta.get(cls.name); + const types = classFieldTypes(emitter.shapeHost, cls, slots); + // Inherited fields come first in def.fields: report each field once, + // on the class that declares it. + const inherited = meta?.base?.def.fields.length ?? 0; + cls.fields.forEach((field, i) => { + if (i < inherited) return; + let storage = types[i]!; + if (storage === INLINE_UNION_FIELD) storage = "inline union"; + else if (meta?.compressed[i] === true) storage = "ref32"; + else if (storage === "ptr") storage = unionStorage(field.type) ?? "ptr"; + let reason = ""; + if (field.type.kind === "f64" && storage === "double") { + const family = slots.family(cls.name, field.name); + reason = + family === null + ? "no field family" + : (slots.reasons.get(`field:${family}`) ?? "not a candidate"); + } + row("field", cls.name, field.name, typeLabel(field.type), storage, reason); + }); + } + + for (const fn of mod.functions) { + const abi = emitter.int32Abis.get(fn.name); + fn.params.forEach((param, i) => { + let storage: string; + let reason = ""; + if (param.type.kind === "f64") { + storage = abi?.params[i] === true ? "i32" : "double"; + if (storage === "double") reason = parameterReason(emitter, fn, param.localId); + } else storage = unionStorage(param.type) ?? scalarStorage(param.type); + row("param", fn.name, param.name, typeLabel(param.type), storage, reason); + }); + const ret = fn.returnType; + if (ret.kind === "void") continue; + let storage: string; + let reason = ""; + if (ret.kind === "f64") { + storage = abi?.result === true ? "i32" : "double"; + if (storage === "double") + reason = slots.reasons.get(`ret:${fn.name}`) ?? notCandidate(emitter, fn); + } else if (ret.kind === "union" && abi?.union === true) storage = "register pair"; + else storage = unionStorage(ret) ?? scalarStorage(ret); + row("return", fn.name, "", typeLabel(ret), storage, reason); + } + + for (const def of mod.unions ?? []) + row( + "union", + def.id, + "", + typeLabel({ kind: "union", unionId: def.id }), + unionStorage({ kind: "union", unionId: def.id })!, + "", + ); + return lines.join("\n") + "\n"; +} + +function scalarStorage(t: IrType): string { + switch (t.kind) { + case "f64": + return "double"; + case "bool": + return "i1"; + default: + return "ptr"; + } +} + +function parameterReason(emitter: LlEmitter, fn: IrFunction, localId: string): string { + const slots = emitter.int32Slots; + const recorded = slots.reasons.get(`param:${fn.name}\0${localId}`); + if (recorded !== undefined) return recorded; + // Proven, but the function has no i32 entry (findInt32Abis). + if (slots.hasParam(fn.name, localId)) return "no i32 entry"; + return notCandidate(emitter, fn); +} + +/** Why a function's numbers were never int32 candidates: the analysis only + * proves functions that direct calls alone reach. */ +function notCandidate(emitter: LlEmitter, fn: IrFunction): string { + if (fn.async) return "async function"; + if (fn.generator) return "generator"; + if (fn.captures !== undefined) return "closure"; + if (!emitter.int32Slots.called.has(fn.name)) return "no direct call"; + return "escapes or is dispatched"; +} + +/** Type labels with union arms spelled out. */ +function typeLabeler(mod: IrModule): (t: IrType) => string { + const arms = new Map((mod.unions ?? []).map((def) => [def.id, def.arms])); + return (t) => { + if (t.kind !== "union") return typeKey(t); + const list = arms.get(t.unionId); + return list === undefined ? typeKey(t) : list.map(typeKey).join(" | "); + }; +} diff --git a/packages/compiler/src/backend/llvm/shapes.ts b/packages/compiler/src/backend/llvm/shapes.ts index b5b1e735a..e48ab8e8c 100644 --- a/packages/compiler/src/backend/llvm/shapes.ts +++ b/packages/compiler/src/backend/llvm/shapes.ts @@ -2,6 +2,7 @@ * Containers store erased adapters; shape helpers retain, release and trace * their fields. Every allocation has a size_t reference count at offset zero * and participates in the runtime allocation audit. */ +import type { HeapCensus } from "./heap-census.js"; import type { IrClassDef, IrModule, IrRecordShape, IrType } from "../../ir/ir.js"; import { isIdentityCollectionKey, @@ -78,6 +79,8 @@ const CYC_MAX_TYPES = 32768; /** What the tables need from the emitter: the extern-declaration ledger * and the module-wide cycle/shape indexes. */ export interface ShapeHost { + /** The counters of a build instrumented for the heap census (heap-census.ts). */ + readonly heapCensus?: HeapCensus | null; /** Chunk emitters only (emit-journal.ts): EmitJournal's allocated, built, * opened, used and entry. */ journalAllocated?: ((name: string, op: string, args: readonly unknown[]) => void) | undefined; diff --git a/packages/compiler/src/backend/native-tools.test.ts b/packages/compiler/src/backend/native-tools.test.ts new file mode 100644 index 000000000..11bb8f3ba --- /dev/null +++ b/packages/compiler/src/backend/native-tools.test.ts @@ -0,0 +1,92 @@ +import { execFileSync } from "node:child_process"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, expect, test, vi } from "vitest"; +import { + NATIVE_CODEGEN_LLVM_VERSION, + NATIVE_CODEGEN_PROTOCOL_VERSION, +} from "./native-codegen-core.js"; +import { verifyNativeHelper } from "./native-tools.js"; +import { LINUX_X64_GNU_TARGET } from "./targets.js"; + +vi.mock("node:child_process", async (original) => ({ + ...(await original()), + execFileSync: vi.fn(), +})); + +const directories: string[] = []; +afterEach(() => { + vi.resetAllMocks(); + for (const path of directories.splice(0)) rmSync(path, { recursive: true, force: true }); +}); + +/** A helper package directory and the options that verify it. */ +function helperOptions() { + const packageRoot = mkdtempSync(join(tmpdir(), "scriptc-helper-")); + directories.push(packageRoot); + const target = LINUX_X64_GNU_TARGET; + const helper = target.helper; + writeFileSync( + join(packageRoot, "package.json"), + JSON.stringify({ name: helper.packageName, version: "1.2.3" }), + ); + vi.mocked(execFileSync).mockReturnValue( + JSON.stringify({ + ok: true, + protocol_version: NATIVE_CODEGEN_PROTOCOL_VERSION, + scriptc_package_version: "1.2.3", + llvm_version: NATIVE_CODEGEN_LLVM_VERSION, + default_target: helper.defaultTarget, + data_layout: helper.defaultDataLayout, + targets: [target.llvmBackend], + supported_targets: [target.llvmTriple], + host_triple: target.llvmTriple, + }), + ); + return { + executable: join(packageRoot, "helper"), + packageRoot, + compilerVersion: "1.2.3", + target, + helper, + }; +} + +test("a helper the verification record knows is not asked for its version again", () => { + const options = helperOptions(); + let known = false; + const remember = vi.fn(() => { + known = true; + }); + const verified = { known: () => known, remember }; + verifyNativeHelper({ ...options, verified }); + expect(execFileSync).toHaveBeenCalledTimes(1); + expect(remember).toHaveBeenCalledTimes(1); + verifyNativeHelper({ ...options, verified }); + expect(execFileSync).toHaveBeenCalledTimes(1); + expect(remember).toHaveBeenCalledTimes(1); +}); + +test("a helper whose version answer is rejected is not recorded", () => { + const options = helperOptions(); + vi.mocked(execFileSync).mockReturnValue(JSON.stringify({ ok: false })); + const remember = vi.fn(); + expect(() => + verifyNativeHelper({ ...options, verified: { known: () => false, remember } }), + ).toThrow(/incompatible/); + expect(remember).not.toHaveBeenCalled(); +}); + +test("the package identity is checked even for a helper the record knows", () => { + const options = helperOptions(); + const remember = vi.fn(); + expect(() => + verifyNativeHelper({ + ...options, + compilerVersion: "9.9.9", + verified: { known: () => true, remember }, + }), + ).toThrow(/package mismatch/); + expect(execFileSync).not.toHaveBeenCalled(); +}); diff --git a/packages/compiler/src/backend/native-tools.ts b/packages/compiler/src/backend/native-tools.ts index 71ad26111..5056de491 100644 --- a/packages/compiler/src/backend/native-tools.ts +++ b/packages/compiler/src/backend/native-tools.ts @@ -56,6 +56,7 @@ export function emitNativeObject(options: { partitionCache?: string; /** Profile-guided optimization: instrumentation or an indexed profile. */ profile?: NativeProfileOptions; + verified?: NativeHelperVerification; }): { outputs: string[]; profileMatch: NativeProfileMatch | null } { verifyNativeHelper(options); const outputs = nativePartitionPaths(options.outputPath, options.partitions ?? 1); @@ -111,13 +112,24 @@ export function mergeNativeProfiles(options: { ]); } -/** Verify installed identity even when a caller can reuse a cached object. */ +/** Helpers that answered `version` as expected: `known` is true when the + * helper file, as it is now, already did, and `remember` records that this + * one just did. */ +export interface NativeHelperVerification { + known(): boolean; + remember(): void; +} + +/** Verify installed identity even when a caller can reuse a cached object. + * With `verified`, the version probe runs only for a helper it does not + * know yet. */ export function verifyNativeHelper(options: { executable: string; packageRoot: string; compilerVersion: string; target: NativeTargetSpec; helper: NativeHelperSpec; + verified?: NativeHelperVerification; }): void { const identity = JSON.parse(readFileSync(options.packageRoot + "/package.json", "utf8")) as { name: string; @@ -131,8 +143,10 @@ export function verifyNativeHelper(options: { `LLVM helper package mismatch: expected ${options.helper.packageName}@${options.compilerVersion}`, ); } + if (options.verified?.known() === true) return; const version = JSON.parse( runNativeTool(options.executable, ["version", "--format=json"]), ) as Record; validateNativeCodegenVersion(version, options.target, options.helper, options.compilerVersion); + options.verified?.remember(); } diff --git a/packages/compiler/src/executable/executable-cache.ts b/packages/compiler/src/executable/executable-cache.ts index a261df8fd..92090015f 100644 --- a/packages/compiler/src/executable/executable-cache.ts +++ b/packages/compiler/src/executable/executable-cache.ts @@ -178,6 +178,8 @@ export interface EarlyExecutableCacheOptions { checker?: string; /** Omitted unless SCRIPTC_YOUNG_NURSERY=1 (backend/young-nursery.ts). */ youngNursery?: true; + /** Omitted unless SCRIPTC_HEAP_CENSUS is set (backend/llvm/heap-census.ts). */ + heapCensus?: true; npmStatic: readonly string[] | "auto" | null; /** Omitted unless shape inference's overlay compiles the packages * (frontend/shape-inference). */ @@ -334,6 +336,7 @@ function cacheKey(options: EarlyExecutableCacheOptions): string { ...(options.functionLayout !== undefined ? [`layout:${options.functionLayout}`] : []), ...(options.checker !== undefined ? [`checker:${options.checker}`] : []), ...(options.youngNursery ? ["young-nursery"] : []), + ...(options.heapCensus ? ["heap-census"] : []), options.npmStatic === null ? "" : options.npmStatic === "auto" @@ -380,6 +383,7 @@ function routeKey(options: Omit { + const dir = await mkdtemp(join(tmpdir(), "scriptc-inputs-")); + scratch.push(dir); + const source = join(dir, "entry.ts"); + await writeFile(source, "export const answer = 1;\n"); + const settledRead = (tracker: FrontendInputTracker, later: number) => { + vi.useFakeTimers({ toFake: ["Date"] }); + vi.setSystemTime(Date.now() + later); + try { + tracker.runSynchronous(() => trackedReadFile(source)); + } finally { + vi.useRealTimers(); + } + const probe = tracker.snapshot().probes.find((entry) => entry.op === "file"); + return probe?.op === "file" ? probe : undefined; + }; + const first = settledRead(new FrontendInputTracker(), 5000)!; + expect(first.identity).toBeDefined(); + // A digest only the previous snapshot holds shows it was not computed again. + const marker = "f".repeat(64); + const previous = { version: 1 as const, stable: true, probes: [{ ...first, digest: marker }] }; + expect(settledRead(new FrontendInputTracker(previous), 5000)?.digest).toBe(marker); + await writeFile(source, "export const answer = 2;\n"); + const rewritten = settledRead(new FrontendInputTracker(previous), 10000); + expect(rewritten?.identity).not.toBe(first.identity); + expect(rewritten?.digest).toBe( + createHash("sha256").update("export const answer = 2;\n").digest("hex"), + ); +}); + test("a file changed shortly before the read is hashed on every check", async () => { const dir = await mkdtemp(join(tmpdir(), "scriptc-inputs-")); scratch.push(dir); diff --git a/packages/compiler/src/frontend/input-tracker.ts b/packages/compiler/src/frontend/input-tracker.ts index 5dd80dde5..ade739435 100644 --- a/packages/compiler/src/frontend/input-tracker.ts +++ b/packages/compiler/src/frontend/input-tracker.ts @@ -127,6 +127,23 @@ let synchronousTracker: FrontendInputTracker | undefined; export class FrontendInputTracker { private readonly probes = new Map(); private stable = true; + /** The digests `previous` recorded for files read with a settled identity. */ + private readonly known = new Map(); + + /** `previous` is the snapshot of an earlier run whose cache entry is being + * rebuilt: a file whose settled identity has not changed since then has the + * content it hashed, so its digest is not computed again. */ + constructor(previous?: FrontendInputSnapshot) { + for (const probe of previous?.probes ?? []) + if (probe.op === "file" && probe.identity !== undefined) + this.known.set(probe.path, { identity: probe.identity, digest: probe.digest }); + } + + /** The digest an earlier run recorded for `path` with this identity. */ + knownDigest(path: string, identity: string): string | undefined { + const known = this.known.get(path); + return known?.identity === identity ? known.digest : undefined; + } run(fn: () => T): T { const parent = activeTracker.getStore(); @@ -254,7 +271,11 @@ export function trackedReadFile(path: string): string | null { if (virtual !== undefined) return virtual; try { const { text, identity } = readWithIdentity(path); - const digest = frontendSourceDigest(text); + const known = + identity === null + ? undefined + : (synchronousTracker ?? activeTracker.getStore())?.knownDigest(path, identity); + const digest = known ?? frontendSourceDigest(text); record( identity === null ? { op: "file", path, digest } : { op: "file", path, digest, identity }, ); diff --git a/packages/compiler/src/index.ts b/packages/compiler/src/index.ts index 3659c092f..2a4d2ea14 100644 --- a/packages/compiler/src/index.ts +++ b/packages/compiler/src/index.ts @@ -165,6 +165,8 @@ import { clearResolveCaches } from "./frontend/resolve.js"; import { loadFfiProfile, type FfiProfile } from "./ffi/ffi-manifest.js"; import { executableLinkFeatures } from "./backend/executable-features.js"; import { youngNurseryRequested } from "./backend/young-nursery.js"; +import { heapCensusRequested } from "./backend/llvm/heap-census.js"; +import { representationReportPath } from "./backend/llvm/repr-report.js"; import { type FrontendInputSnapshot, FrontendInputTracker, @@ -1742,6 +1744,7 @@ async function compileTracked( ...(functionLayoutDigest === undefined ? {} : { functionLayout: functionLayoutDigest }), ...checkerField(), ...(youngNurseryRequested() ? { youngNursery: true as const } : {}), + ...(heapCensusRequested() ? { heapCensus: true as const } : {}), npmStatic: opts.npmStatic ?? null, ...(shapeOverlayEnabled() ? { shapeOverlay: true as const } : {}), ffiProfile: @@ -1785,8 +1788,9 @@ async function compileTracked( implementationDependencies: implementation.dependencies, }; } + // A cache hit emits no module, so it would write no representation report. const cachedHit = - earlyCacheOptions === null + earlyCacheOptions === null || representationReportPath() !== null ? null : await readEarlyExecutableCache(cacheRoot, earlyCacheOptions); // A --libnode=beside build that calls into Node places its island modules diff --git a/packages/compiler/src/native/cache.ts b/packages/compiler/src/native/cache.ts index 0250e9618..551a151a1 100644 --- a/packages/compiler/src/native/cache.ts +++ b/packages/compiler/src/native/cache.ts @@ -170,6 +170,7 @@ export class NativeCache { "kept", "linker-job", "runtime-pack", + "helper", "binary", "dsym", "coverage", diff --git a/packages/compiler/src/native/compiler.ts b/packages/compiler/src/native/compiler.ts index 1cf4f7eac..2ed2cbc16 100644 --- a/packages/compiler/src/native/compiler.ts +++ b/packages/compiler/src/native/compiler.ts @@ -28,6 +28,7 @@ import { requireNativeArtifact, runNativeTool, verifyNativeHelper, + type NativeHelperVerification, } from "../backend/native-tools.js"; import { type NativeRuntimeSelection, @@ -192,6 +193,44 @@ export class NativeCompiler { }; } + /** The helpers whose `version` answer this cache recorded, by file identity: + * the probe runs again only when the helper file changes. */ + private helperVerification(): { verified?: NativeHelperVerification } { + const cache = this.cache; + const toolchain = this.toolchain; + if (cache === null) return {}; + let identity: string; + let key: string; + try { + identity = nativeFileIdentity(toolchain.helperExecutable); + key = contentDigest( + JSON.stringify({ + schema: 1, + helper: identity, + compiler: nativeFileIdentity(process.execPath), + version: toolchain.compilerVersion, + target: toolchain.target.name, + package: toolchain.helper.packageName, + }), + ); + } catch { + return {}; + } + return { + verified: { + known: () => cache.read("helper", key) !== null, + remember: () => { + try { + if (nativeFileIdentity(toolchain.helperExecutable) === identity) + cache.write("helper", key, "verified\n"); + } catch { + /* A failed write only means probing again next time. */ + } + }, + }, + }; + } + private emitObject( input: string, output: string, @@ -211,6 +250,7 @@ export class NativeCompiler { compilerVersion: toolchain.compilerVersion, target: toolchain.target, helper: toolchain.helper, + ...this.helperVerification(), }; let key: string | null = null; let identity: string | null = null; @@ -266,11 +306,7 @@ export class NativeCompiler { ? null : this.cache.partitionDirectory(identity); const { profileMatch } = emitNativeObject({ - executable: toolchain.helperExecutable, - packageRoot: toolchain.helperPackageRoot, - compilerVersion: toolchain.compilerVersion, - target: toolchain.target, - helper: toolchain.helper, + ...helperOptions, inputPath: input, outputPath: output, sourcePath: source, diff --git a/packages/compiler/src/native/prepare.ts b/packages/compiler/src/native/prepare.ts index 4300c3c25..339a50faa 100644 --- a/packages/compiler/src/native/prepare.ts +++ b/packages/compiler/src/native/prepare.ts @@ -370,6 +370,8 @@ export function prepareNativeExecutable( const exclusions = { outputPaths, outputDirectories: [...directories] }; const irInput = process.env[BOOTSTRAP_IR_INPUT] ?? ""; let key: string | null = null; + // The entry this build replaces: its probes give the digests of unchanged files. + let previous: FrontendInputSnapshot | undefined; if (cache !== null && irInput === "" && provenanceSources() === null) { try { key = contentDigest( @@ -389,6 +391,7 @@ export function prepareNativeExecutable( timing("frontend-cache-hit"); return cached.input; } + if (cached !== null) previous = cached.stale.probes; const equivalent = cached === null || outputKind === "ir" ? null @@ -403,7 +406,7 @@ export function prepareNativeExecutable( } } timing("frontend-cache-miss"); - const tracker = new FrontendInputTracker(); + const tracker = new FrontendInputTracker(previous); const prepared = irInput !== "" ? preparedFromIr(irInput, entry, options) diff --git a/packages/compiler/test/native-partitions.test.ts b/packages/compiler/test/native-partitions.test.ts index c86a37aa4..18d0aa749 100644 --- a/packages/compiler/test/native-partitions.test.ts +++ b/packages/compiler/test/native-partitions.test.ts @@ -327,6 +327,16 @@ describe.runIf(helper !== null)("LLVM program partitions", () => { const first = await readFile(parts[0]!); const [single] = await emit(dir, source, { outputs: 1, optLevel: "2", name: "whole" }); expect(await readFile(single!)).toEqual(first); + // The empty parts name the source path, so they do not depend on the + // input's file name either. + const renamed = await emit(dir, source, { + outputs: 3, + optLevel: "2", + split: true, + name: "renamed", + }); + for (let index = 0; index < parts.length; index++) + expect(await readFile(renamed[index]!)).toEqual(await readFile(parts[index]!)); }); test("split code generation requires several optimized outputs and no cache", async () => { diff --git a/packages/runtime/src/scr_lib.c b/packages/runtime/src/scr_lib.c index 0b780603b..ad42a5824 100644 --- a/packages/runtime/src/scr_lib.c +++ b/packages/runtime/src/scr_lib.c @@ -225,7 +225,18 @@ static void scr_module_registry_cleanup(void) { * own cleanup only when the value is first materialized. That is equivalent * at process exit and retains the RC-audit cleanup guarantee for the values a * program actually observes. */ +#if defined(__linux__) && !defined(SCR_LIB) +/* The heap census of a program built with SCRIPTC_HEAP_CENSUS + * (heap-census.ts) writes its counters at exit, from here and from + * scr_process_exit; every other program leaves the weak reference null. + * Libraries have no census, and their archives declare no such symbol. */ +extern void scr_census_write(void) __attribute__((weak)); +#endif + static void scr_lib_cleanup(void) { +#if defined(__linux__) && !defined(SCR_LIB) + if (scr_census_write != NULL) scr_census_write(); +#endif scr_module_registry_cleanup(); scr_arr_release(scr_argv_arr); scr_argv_arr = NULL; @@ -2137,6 +2148,9 @@ void scr_process_exit(double code) { fflush(stdout); #if defined(__linux__) if (__llvm_profile_write_file != NULL) (void)__llvm_profile_write_file(); +#ifndef SCR_LIB + if (scr_census_write != NULL) scr_census_write(); +#endif scr_detach_teardown(); #endif _Exit((int)code); diff --git a/packages/runtime/src/scr_lib.h b/packages/runtime/src/scr_lib.h index e01bc7b23..1d8d91d97 100644 --- a/packages/runtime/src/scr_lib.h +++ b/packages/runtime/src/scr_lib.h @@ -150,6 +150,10 @@ void scr_process_exit(double code); * helper process (Linux, `--detached-exit`). scr_lib.c defines a weak 0; * the program object of such an executable defines it as 1. */ extern const unsigned char scr_detached_exit; +/* The heap census writer of a program built with SCRIPTC_HEAP_CENSUS + * (heap-census.ts): that program's object defines it, and scr_lib.c calls + * it at exit through a weak reference (programs on Linux only). */ +void scr_census_write(void); void scr_process_set_exit_code(double code); double scr_process_exit_code_or_zero(void); /* The writer and the presence test of programs that read process.exitCode: diff --git a/tests/harness/heap-census.test.ts b/tests/harness/heap-census.test.ts new file mode 100644 index 000000000..b8ca2b799 --- /dev/null +++ b/tests/harness/heap-census.test.ts @@ -0,0 +1,113 @@ +/* The heap census (backend/llvm/heap-census.ts): a program built with + * SCRIPTC_HEAP_CENSUS counts class instances and the stores, lengths and + * accesses of array-typed fields, and writes them at exit, on a normal exit + * and on process.exit alike. */ +import { execFile } from "node:child_process"; +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { promisify } from "node:util"; +import { expect, test } from "vitest"; +import { compile } from "@scriptc/compiler"; + +const exec = promisify(execFile); +const sanitize = process.env["SCRIPTC_SAN"] === "1"; + +const PROGRAM = ` +class Node { + kids: Node[]; + tags: string[] | undefined; + constructor(kids: Node[]) { + this.kids = kids; + if (kids.length > 1) this.tags = ["a", "b"]; + else this.tags = undefined; + } +} +let total = 0; +const roots: Node[] = []; +for (let i = 0; i < 100; i++) { + const leaves: Node[] = []; + for (let j = 0; j < i % 6; j++) leaves.push(new Node([])); + roots.push(new Node(leaves)); +} +for (const r of roots) total += r.kids.length + (r.tags?.length ?? 0); +console.log(total); +`; + +async function census(source: string): Promise<{ stdout: string; counts: Map }> { + const dir = await mkdtemp(join(tmpdir(), "scriptc-heap-census-")); + const saved = process.env["SCRIPTC_HEAP_CENSUS"]; + try { + const entry = join(dir, "main.ts"); + await writeFile(entry, source); + process.env["SCRIPTC_HEAP_CENSUS"] = "1"; + const result = await compile(entry, { + backend: "llvm", + sanitize, + outDir: dir, + outPath: join(dir, "program"), + }); + if (saved === undefined) delete process.env["SCRIPTC_HEAP_CENSUS"]; + else process.env["SCRIPTC_HEAP_CENSUS"] = saved; + expect(result.ok, JSON.stringify(result.diagnostics)).toBe(true); + if (!result.ok) throw new Error("compile failed"); + const out = join(dir, "census.tsv"); + const { stdout } = await exec(result.binaryPath, [], { + env: { ...process.env, SCRIPTC_HEAP_CENSUS: out }, + timeout: 120_000, + }); + const counts = new Map(); + for (const line of (await readFile(out, "utf8")).trimEnd().split("\n")) { + const [kind, cls, field, count] = line.split("\t"); + counts.set(`${kind} ${cls}${field ? "." + field : ""}`, Number(count)); + } + return { stdout, counts }; + } finally { + if (saved === undefined) delete process.env["SCRIPTC_HEAP_CENSUS"]; + else process.env["SCRIPTC_HEAP_CENSUS"] = saved; + await rm(dir, { recursive: true, force: true }); + } +} + +const EXPECTED: [string, number][] = [ + // 100 roots and 246 leaves (i % 6 children each). + ["new Node", 346], + ["store Node.kids", 346], + ["elements Node.kids", 246], + ["len0 Node.kids", 263], + ["len1 Node.kids", 17], + ["len2 Node.kids", 17], + ["len3-4 Node.kids", 33], + ["len5-8 Node.kids", 16], + ["len9-16 Node.kids", 0], + // Literals count where the store itself writes one. + ["literal Node.kids", 0], + ["store Node.tags", 346], + ["literal Node.tags", 66], + // An undefined tags field counts as length 0. + ["len0 Node.tags", 280], + ["len2 Node.tags", 66], + ["elements Node.tags", 132], +]; + +test.skipIf(process.platform !== "linux")( + "the census counts instances and array field stores at a normal exit", + async () => { + const { stdout, counts } = await census(PROGRAM); + expect(stdout).toBe("378\n"); + for (const [key, count] of EXPECTED) expect([key, counts.get(key)]).toEqual([key, count]); + // Every store also computes the field's address. + expect(counts.get("access Node.kids")).toBeGreaterThanOrEqual(346 + 100); + }, + 240_000, +); + +test.skipIf(process.platform !== "linux")( + "the census is written on process.exit too", + async () => { + const { stdout, counts } = await census(PROGRAM + "process.exit(0);\n"); + expect(stdout).toBe("378\n"); + expect(counts.get("new Node")).toBe(346); + }, + 240_000, +);