From 8b256639f4144bdb7d37c328ad9a8a54ae53be17 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Tue, 27 Oct 2026 01:31:00 -0500 Subject: [PATCH 1/6] Add a representation report and a heap census for ranking value-layout work --- .../src/backend/llvm/emit-statements.ts | 21 +- packages/compiler/src/backend/llvm/emitter.ts | 14 +- .../compiler/src/backend/llvm/expr-calls.ts | 1 + .../compiler/src/backend/llvm/heap-census.ts | 228 ++++++++++++++++++ .../src/backend/llvm/repr-report.test.ts | 65 +++++ .../compiler/src/backend/llvm/repr-report.ts | 135 +++++++++++ packages/runtime/src/scr_lib.c | 11 + tests/harness/heap-census.test.ts | 113 +++++++++ 8 files changed, 584 insertions(+), 4 deletions(-) create mode 100644 packages/compiler/src/backend/llvm/heap-census.ts create mode 100644 packages/compiler/src/backend/llvm/repr-report.test.ts create mode 100644 packages/compiler/src/backend/llvm/repr-report.ts create mode 100644 tests/harness/heap-census.test.ts diff --git a/packages/compiler/src/backend/llvm/emit-statements.ts b/packages/compiler/src/backend/llvm/emit-statements.ts index 74c84ae86..7732fb8be 100644 --- a/packages/compiler/src/backend/llvm/emit-statements.ts +++ b/packages/compiler/src/backend/llvm/emit-statements.ts @@ -617,6 +617,14 @@ export function emitStmtBody(emitter: LlEmitter, s: IrStmt): void { // (The guard precedes the value here: the nullable store // evaluates it itself.) emitter.emitStoreGuard(obj.name, s); + emitter.heapCensus?.countStore( + (line) => B.line(line), + () => B.tmp(), + s.className, + s.field, + s.value, + null, + ); emitter.emitNullableFieldStore(obj.name, s.className, s.field, nullable, s.value); break; } @@ -636,8 +644,17 @@ export function emitStmtBody(emitter: LlEmitter, s: IrStmt): void { ? emitter.recordFieldInline(s.shapeId, s.field) : emitter.inlineUnions.classField(s.className, s.field), ); - if (s.kind === "fieldSet") emitter.emitStoreGuard(obj.name, s); - else if (emitter.publishedTypes?.recordGuarded(s.shapeId) === true) + if (s.kind === "fieldSet") { + emitter.emitStoreGuard(obj.name, s); + emitter.heapCensus?.countStore( + (line) => B.line(line), + () => B.tmp(), + s.className, + s.field, + s.value, + v.name, + ); + } else if (emitter.publishedTypes?.recordGuarded(s.shapeId) === true) emitter.emitPublishedFieldGuard(obj.name, null, s.field); const { ptr, type, inline } = s.kind === "fieldSet" diff --git a/packages/compiler/src/backend/llvm/emitter.ts b/packages/compiler/src/backend/llvm/emitter.ts index 2d2ebfb8d..9278613da 100644 --- a/packages/compiler/src/backend/llvm/emitter.ts +++ b/packages/compiler/src/backend/llvm/emitter.ts @@ -130,6 +130,8 @@ import { } from "./int32-abi.js"; import { type Int32Local } from "./int32-locals.js"; import { findIntegerClones, NO_INTEGER_CLONES, type IntegerClones } from "./integer-clones.js"; +import { writeRepresentationReport } from "./repr-report.js"; +import { heapCensusOf, type HeapCensus } from "./heap-census.js"; import { exactInteger, narrowInteger, widenInteger, type LlInteger } from "./integer-values.js"; import { findInitializerBindings } from "../../ir/initializer-bindings.js"; import { findConstantNumericTables, type ConstantNumericTable } from "../../ir/constant-tables.js"; @@ -794,7 +796,9 @@ function emitModuleText(input: IrModule, options: LlvmTargetOptions): string[] { for (let excluded = options.unwindExcluded ?? new Set(); ;) { const emitter = moduleEmitter(input, { ...options, unwindExcluded: excluded }); try { - return emitter.emitParts(); + const parts = emitter.emitParts(); + writeRepresentationReport(emitter); + return parts; } catch (error) { if (!(error instanceof UnwindAbiViolation)) throw error; const names = new Set(excluded); @@ -1610,6 +1614,8 @@ export class LlEmitter { readonly inlineUnions: InlineUnionFields; /** Unions whose values are nullable arm pointers (nullable-unions.ts). */ readonly nullableUnions: NullableUnions; + /** The counters of a build instrumented for the heap census, or null. */ + readonly heapCensus: HeapCensus | null; /** The union table without nullable unions: the stack-box analyses. */ readonly boxedUnionsById: Map; readonly recordsById = new Map(); @@ -2002,6 +2008,8 @@ export class LlEmitter { options.debugSources === undefined, this.nodeIndex, ); + this.heapCensus = heapCensusOf(mod, this.nullableUnions); + for (const decl of this.heapCensus?.declarations() ?? []) this.declare(decl); this.mockTimers = this.runtimeFeatures.mockTimers; this.debug = options.debugSources === undefined @@ -2641,7 +2649,8 @@ export class LlEmitter { emitParts(): string[] { const parts = emitParts(this); const bridged = this.unwindAbi.size > 0 ? bridgePlainUnwindCalls(parts) : parts; - return this.cpuDispatch ? cloneForCpuDispatch(bridged) : bridged; + const out = this.cpuDispatch ? cloneForCpuDispatch(bridged) : bridged; + return this.heapCensus === null ? out : [...out, this.heapCensus.definitions().join("\n")]; } /** The dispose-observer gates the emitted frees and record writers test @@ -4681,6 +4690,7 @@ export class LlEmitter { `${p} = getelementptr inbounds %${classStructSym(className)}, ptr ${objName}, i64 0, i32 ${index}`, ); this.markFieldPointer(p, field); + this.heapCensus?.countAccess((line) => this.B.line(line), className, field); if (compressed) this.compressedFieldPtrs.add(p); return { ptr: p, type, inline: this.inlineUnions.classField(className, field), compressed }; } diff --git a/packages/compiler/src/backend/llvm/expr-calls.ts b/packages/compiler/src/backend/llvm/expr-calls.ts index 18181de50..9b2f2e4d8 100644 --- a/packages/compiler/src/backend/llvm/expr-calls.ts +++ b/packages/compiler/src/backend/llvm/expr-calls.ts @@ -1105,6 +1105,7 @@ export function emitCallExpr( ); const o = B.tmp(); B.line(`${o} = call ptr @${host.allocSym(mangleClassNew(e.className), e)}()`); + host.heapCensus?.countNew((line) => B.line(line), e.className); const out = host.own({ name: o, type: e.type }); emitConstruction(host, e, o, ctor); return out; diff --git a/packages/compiler/src/backend/llvm/heap-census.ts b/packages/compiler/src/backend/llvm/heap-census.ts new file mode 100644 index 000000000..91b9839cd --- /dev/null +++ b/packages/compiler/src/backend/llvm/heap-census.ts @@ -0,0 +1,228 @@ +/* The heap census. A program built with SCRIPTC_HEAP_CENSUS set counts, for + * ranking representation work: + * - the instances each `new` of a class creates, with the class's size; + * - for each array-typed class field, its stores (with the stored array's + * length in buckets and the total of those lengths), the stores whose + * value is an array literal written at the store, and its accesses; + * - for each record-typed class field, its stores and accesses. + * Accesses count every field address the backend computes for the field, + * so a store is also an access. At exit the program writes one + * tab-separated line per counter to the file the run's SCRIPTC_HEAP_CENSUS + * names, or to stderr. Linux only: the runtime calls the writer through a + * weak reference, as it does the profile writer. Counters are module + * globals updated with monotonic atomic adds, so they include every + * thread's work. Ordinary builds emit none of it. */ +import type { IrClassDef, IrExpr, IrModule, IrType } from "../../ir/ir.js"; +import { classStructSym } from "./classes.js"; +import { llvmBytes, wtf8Bytes } from "../literals.js"; +import type { NullableUnions } from "./nullable-unions.js"; + +/** The census of the module being built, or null when it is off. */ +export function heapCensusOf(mod: IrModule, unions: NullableUnions): HeapCensus | null { + const flag = process.env["SCRIPTC_HEAP_CENSUS"]; + if (flag === undefined || flag === "" || mod.lib !== undefined) return null; + return new HeapCensus(mod, unions); +} + +/** A NUL-terminated i8 array constant of `text`. */ +function cString(text: string): string { + const bytes = wtf8Bytes(text); + return `[${bytes.length + 1} x i8] c"${llvmBytes(bytes)}"`; +} + +const LENGTH_BUCKETS = ["len0", "len1", "len2", "len3-4", "len5-8", "len9-16", "len17+"]; +/** Upper bounds of LENGTH_BUCKETS but the last. */ +const BUCKET_LIMITS = [0, 1, 2, 4, 8, 16]; + +interface FieldCounters { + /** First counter: stores, accesses, then (arrays) the literal stores, + * the length total and the buckets. */ + base: number; + array: boolean; + /** The stored value is a pointer that is NULL for the union's unit arm. */ + nullable: boolean; +} + +export class HeapCensus { + private readonly names: string[] = []; + private readonly sizes: (string | null)[] = []; + private readonly instances = new Map(); + private readonly fields = new Map(); + + constructor(mod: IrModule, unions: NullableUnions) { + for (const cls of mod.classes ?? []) { + this.instances.set(cls.name, this.add(`new\t${cls.name}\t`, classStructSym(cls.name))); + for (const field of cls.fields) this.addField(cls, field.name, field.type, unions); + } + } + + /** Counts one instance of `className`. */ + countNew(line: (text: string) => void, className: string): void { + const index = this.instances.get(className); + if (index !== undefined) line(this.increment(index, "1")); + } + + /** Counts an access of a field; true when the field is counted. */ + countAccess(line: (text: string) => void, className: string, field: string): void { + const counters = this.fields.get(`${className}\0${field}`); + if (counters !== undefined) line(this.increment(counters.base + 1, "1")); + } + + /** Counts a store of `value` (an LLVM pointer, computed from `source`) + * into a field: its length when the field holds arrays. `tmp` names fresh + * temporaries. */ + countStore( + line: (text: string) => void, + tmp: () => string, + className: string, + field: string, + source: IrExpr, + value: string | null, + ): void { + const counters = this.fields.get(`${className}\0${field}`); + if (counters === undefined) return; + line(this.increment(counters.base, "1")); + if (!counters.array) return; + const literal = source.kind === "unionWrap" ? source.value : source; + if (literal.kind === "arrayLit") line(this.increment(counters.base + 2, "1")); + if (value === null) return; + // A NULL pointer (the unit arm) counts as length 0: its length is read + // through a select that points NULL at a zeroed array header. + let array = value; + if (counters.nullable) { + const isNull = tmp(); + array = tmp(); + line(`${isNull} = icmp eq ptr ${value}, null`); + line(`${array} = select i1 ${isNull}, ptr @sc_census_zero_arr, ptr ${value}`); + } + const lenPtr = tmp(), + len32 = tmp(), + len = tmp(); + line(`${lenPtr} = getelementptr inbounds %ScrArr, ptr ${array}, i32 0, i32 1`); + line(`${len32} = load i32, ptr ${lenPtr}`); + line(`${len} = zext i32 ${len32} to i64`); + line(this.increment(counters.base + 3, len)); + // The bucket: the number of limits below the length. + let bucket = "0"; + for (const limit of BUCKET_LIMITS) { + const above = tmp(), + step = tmp(), + next = tmp(); + line(`${above} = icmp ugt i64 ${len}, ${limit}`); + line(`${step} = zext i1 ${above} to i64`); + line(`${next} = add i64 ${bucket}, ${step}`); + bucket = next; + } + const slot = tmp(); + line(`${slot} = add i64 ${bucket}, ${counters.base + 4}`); + const ptr = tmp(); + line( + `${ptr} = getelementptr inbounds [${this.names.length} x i64], ptr @sc_census, i64 0, i64 ${slot}`, + ); + line(`atomicrmw add ptr ${ptr}, i64 1 monotonic`); + } + + /** The module definitions: the counters, their names, the zero array + * header NULL stores read their length from, and the writer the runtime + * calls at exit (scr_census_write). */ + definitions(): string[] { + const n = this.names.length; + const out = [`@sc_census = internal global [${n} x i64] zeroinitializer`]; + // A zeroed %ScrArr: length 0. + out.push(`@sc_census_zero_arr = internal constant %ScrArr zeroinitializer`); + const names = this.names.map((name, i) => { + const bytes = cString(name); + out.push(`@sc_census_name.${i} = private unnamed_addr constant ${bytes}`); + return `ptr @sc_census_name.${i}`; + }); + out.push(`@sc_census_names = internal constant [${n} x ptr] [${names.join(", ")}]`); + const sizes = this.sizes.map((sym) => + sym === null ? "i64 0" : `i64 ptrtoint (ptr getelementptr (%${sym}, ptr null, i32 1) to i64)`, + ); + out.push(`@sc_census_sizes = internal constant [${n} x i64] [${sizes.join(", ")}]`); + const env = cString("SCRIPTC_HEAP_CENSUS"); + out.push(`@sc_census_env = private unnamed_addr constant ${env}`); + out.push(`@sc_census_mode = private unnamed_addr constant ${cString("w")}`); + out.push(`@sc_census_fmt = private unnamed_addr constant ${cString("%s\t%llu\t%llu\n")}`); + out.push( + `define void @scr_census_write() {`, + `entry:`, + ` %path = call ptr @getenv(ptr @sc_census_env)`, + ` %named = icmp ne ptr %path, null`, + ` br i1 %named, label %open, label %err`, + `open:`, + ` %file = call ptr @fopen(ptr %path, ptr @sc_census_mode)`, + ` %opened = icmp ne ptr %file, null`, + ` br i1 %opened, label %write, label %err`, + `err:`, + ` %stderr = load ptr, ptr @stderr`, + ` br label %write`, + `write:`, + ` %out = phi ptr [ %file, %open ], [ %stderr, %err ]`, + ` br label %loop`, + `loop:`, + ` %i = phi i64 [ 0, %write ], [ %next, %loop ]`, + ` %namep = getelementptr inbounds [${n} x ptr], ptr @sc_census_names, i64 0, i64 %i`, + ` %name = load ptr, ptr %namep`, + ` %countp = getelementptr inbounds [${n} x i64], ptr @sc_census, i64 0, i64 %i`, + ` %count = load atomic i64, ptr %countp monotonic, align 8`, + ` %sizep = getelementptr inbounds [${n} x i64], ptr @sc_census_sizes, i64 0, i64 %i`, + ` %size = load i64, ptr %sizep`, + ` %bytes = mul i64 %count, %size`, + ` call i32 (ptr, ptr, ...) @fprintf(ptr %out, ptr @sc_census_fmt, ptr %name, i64 %count, i64 %bytes)`, + ` %next = add i64 %i, 1`, + ` %done = icmp eq i64 %next, ${n}`, + ` br i1 %done, label %end, label %loop`, + `end:`, + ` call i32 @fflush(ptr %out)`, + ` ret void`, + `}`, + ); + return out; + } + + /** External declarations the writer uses. */ + declarations(): string[] { + return [ + "declare ptr @getenv(ptr)", + "declare ptr @fopen(ptr, ptr)", + "declare i32 @fprintf(ptr, ptr, ...)", + "declare i32 @fflush(ptr)", + "@stderr = external global ptr", + ]; + } + + private add(name: string, sizeSym: string | null = null): number { + this.names.push(name); + this.sizes.push(sizeSym); + return this.names.length - 1; + } + + private addField(cls: IrClassDef, field: string, type: IrType, unions: NullableUnions): void { + let array = type.kind === "array"; + let nullable = false; + let record = type.kind === "record"; + if (type.kind === "union") { + const single = unions.single(type.unionId); + if (single !== null && single.nullTag < 0) { + array = single.arm.kind === "array"; + record = single.arm.kind === "record"; + nullable = true; + } + } + if (!array && !record) return; + const key = `${cls.name}\t${field}`; + const base = this.add(`store\t${key}`); + this.add(`access\t${key}`); + if (array) { + this.add(`literal\t${key}`); + this.add(`elements\t${key}`); + for (const bucket of LENGTH_BUCKETS) this.add(`${bucket}\t${key}`); + } + this.fields.set(`${cls.name}\0${field}`, { base, array, nullable }); + } + + private increment(index: number, by: string): string { + return `atomicrmw add ptr getelementptr inbounds ([${this.names.length} x i64], ptr @sc_census, i64 0, i64 ${index}), i64 ${by} monotonic`; + } +} diff --git a/packages/compiler/src/backend/llvm/repr-report.test.ts b/packages/compiler/src/backend/llvm/repr-report.test.ts new file mode 100644 index 000000000..ec7b9b5ed --- /dev/null +++ b/packages/compiler/src/backend/llvm/repr-report.test.ts @@ -0,0 +1,65 @@ +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, expect, test } from "vitest"; +import { compile, deserializeModule } from "../../index.js"; +import { emitLlvmModule } from "./emitter.js"; +import { REPR_REPORT_HEADER } from "./repr-report.js"; + +const saved = process.env["SCRIPTC_REPR_REPORT"]; +afterEach(() => { + if (saved === undefined) delete process.env["SCRIPTC_REPR_REPORT"]; + else process.env["SCRIPTC_REPR_REPORT"] = saved; +}); + +test("the representation report names each slot's storage and why a number stays a double", async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-repr-report-")); + try { + const entry = join(dir, "main.ts"), + outPath = join(dir, "main.ir.json"), + report = join(dir, "repr.tsv"); + await writeFile( + entry, + ` +class Box { + id: number; + hits: number; + items: number[]; + constructor(id: number, items: number[]) { this.id = id | 0; this.hits = 0; this.items = items; } +} +function scale(value: number, by: number = 2): number { return (value * by) | 0; } +function half(value: number): number { return value / 2; } +const boxes: Box[] = []; +for (let i = 0; i < 4; i++) boxes.push(new Box(i, [i, i + 1])); +for (const b of boxes) b.hits++; +console.log(scale(boxes[1]!.id), scale(3, 4), half(boxes.length), boxes[2]!.items.length, boxes[3]!.hits); +`, + ); + const result = await compile(entry, { outDir: dir, outPath, outputKind: "ir" }); + if (!result.ok) throw new Error(result.diagnostics.map((d) => d.message).join("\n")); + const mod = deserializeModule(await readFile(outPath, "utf8")); + process.env["SCRIPTC_REPR_REPORT"] = report; + emitLlvmModule(mod); + const lines = (await readFile(report, "utf8")).trimEnd().split("\n"); + expect(lines[0]).toBe(REPR_REPORT_HEADER); + const rows = lines.slice(1).map((line) => line.split("\t")); + const row = (kind: string, owner: string, name: string) => + rows.find((r) => r[0] === kind && r[1] === owner && r[2] === name); + expect(row("field", "Box", "id")?.slice(3, 5)).toEqual(["f64", "i32"]); + // `++` on a field keeps it a double, and the report says so. + expect(row("field", "Box", "hits")?.[4]).toBe("double"); + expect(row("field", "Box", "hits")?.[5]).not.toBe(""); + expect(row("field", "Box", "items")?.slice(3, 5)).toEqual(["array", "ptr"]); + expect(row("param", "scale", "value")?.[4]).toBe("i32"); + // A defaulted parameter is a `number | undefined` box; the variant for + // calls that pass a number takes an i32. + expect(row("param", "scale", "by")?.slice(3, 5)).toEqual(["f64 | undefined", "box"]); + expect(row("param", "scale.nativeNumber", "by")?.[4]).toBe("i32"); + // An array length may exceed the int32 range. + expect(row("param", "half", "value")?.[4]).toBe("double"); + expect(row("param", "half", "value")?.[5]).toMatch(/^argument in .* is not int32$/); + expect(row("return", "half", "")?.slice(4)).toEqual(["double", "return is not int32"]); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); diff --git a/packages/compiler/src/backend/llvm/repr-report.ts b/packages/compiler/src/backend/llvm/repr-report.ts new file mode 100644 index 000000000..82104277c --- /dev/null +++ b/packages/compiler/src/backend/llvm/repr-report.ts @@ -0,0 +1,135 @@ +/* The representation report (SCRIPTC_REPR_REPORT=): after a module is + * emitted, one tab-separated line per class field, function parameter, + * function result and union, naming the storage the backend chose for it + * and, for numbers kept as doubles, the reason the int32 analysis recorded + * (int32-slots.ts Int32Slots.reasons). It is a diagnostic for ranking + * representation work; nothing reads it back. */ +import { writeFileSync } from "node:fs"; +import { typeKey, type IrFunction, type IrModule, type IrType } from "../../ir/ir.js"; +import type { LlEmitter } from "./emitter.js"; +import { classFieldTypes } from "./classes.js"; +import { INLINE_UNION_FIELD } from "./inline-unions.js"; + +export const REPR_REPORT_HEADER = "kind\towner\tname\ttype\tstorage\treason"; + +/** Writes the report of `emitter`'s module when SCRIPTC_REPR_REPORT names a + * file. */ +export function writeRepresentationReport(emitter: LlEmitter): void { + const path = process.env["SCRIPTC_REPR_REPORT"]; + if (path === undefined || path === "") return; + writeFileSync(path, representationReport(emitter)); +} + +export function representationReport(emitter: LlEmitter): string { + const mod = emitter.mod; + const slots = emitter.int32Slots; + const lines = [REPR_REPORT_HEADER]; + const row = (...cells: string[]): void => { + lines.push(cells.map((cell) => cell.replace(/[\t\n]/g, " ")).join("\t")); + }; + const typeLabel = typeLabeler(mod); + // A union value that is not a plain box: a nullable or shape pointer. + const unionStorage = (t: IrType): string | null => { + if (t.kind !== "union") return null; + return emitter.nullableUnions.get(t.unionId)?.repr ?? "box"; + }; + + for (const cls of mod.classes ?? []) { + const meta = emitter.classMeta.get(cls.name); + const types = classFieldTypes(emitter.shapeHost, cls, slots); + // Inherited fields come first in def.fields: report each field once, + // on the class that declares it. + const inherited = meta?.base?.def.fields.length ?? 0; + cls.fields.forEach((field, i) => { + if (i < inherited) return; + let storage = types[i]!; + if (storage === INLINE_UNION_FIELD) storage = "inline union"; + else if (meta?.compressed[i] === true) storage = "ref32"; + else if (storage === "ptr") storage = unionStorage(field.type) ?? "ptr"; + let reason = ""; + if (field.type.kind === "f64" && storage === "double") { + const family = slots.family(cls.name, field.name); + reason = + family === null + ? "no field family" + : (slots.reasons.get(`field:${family}`) ?? "not a candidate"); + } + row("field", cls.name, field.name, typeLabel(field.type), storage, reason); + }); + } + + for (const fn of mod.functions) { + const abi = emitter.int32Abis.get(fn.name); + fn.params.forEach((param, i) => { + let storage: string; + let reason = ""; + if (param.type.kind === "f64") { + storage = abi?.params[i] === true ? "i32" : "double"; + if (storage === "double") reason = parameterReason(emitter, fn, param.localId); + } else storage = unionStorage(param.type) ?? scalarStorage(param.type); + row("param", fn.name, param.name, typeLabel(param.type), storage, reason); + }); + const ret = fn.returnType; + if (ret.kind === "void") continue; + let storage: string; + let reason = ""; + if (ret.kind === "f64") { + storage = abi?.result === true ? "i32" : "double"; + if (storage === "double") + reason = slots.reasons.get(`ret:${fn.name}`) ?? notCandidate(emitter, fn); + } else if (ret.kind === "union" && abi?.union === true) storage = "register pair"; + else storage = unionStorage(ret) ?? scalarStorage(ret); + row("return", fn.name, "", typeLabel(ret), storage, reason); + } + + for (const def of mod.unions ?? []) + row( + "union", + def.id, + "", + typeLabel({ kind: "union", unionId: def.id }), + unionStorage({ kind: "union", unionId: def.id })!, + "", + ); + return lines.join("\n") + "\n"; +} + +function scalarStorage(t: IrType): string { + switch (t.kind) { + case "f64": + return "double"; + case "bool": + return "i1"; + default: + return "ptr"; + } +} + +function parameterReason(emitter: LlEmitter, fn: IrFunction, localId: string): string { + const slots = emitter.int32Slots; + const recorded = slots.reasons.get(`param:${fn.name}\0${localId}`); + if (recorded !== undefined) return recorded; + // Proven, but the function has no i32 entry (findInt32Abis). + if (slots.hasParam(fn.name, localId)) return "no i32 entry"; + return notCandidate(emitter, fn); +} + +/** Why a function's numbers were never int32 candidates: the analysis only + * proves functions that direct calls alone reach. */ +function notCandidate(emitter: LlEmitter, fn: IrFunction): string { + if (fn.async) return "async function"; + if (fn.generator) return "generator"; + if (fn.captures !== undefined) return "closure"; + if (!emitter.int32Slots.called.has(fn.name)) return "no direct call"; + return "escapes or is dispatched"; +} + +/** Type labels with union arms spelled out. */ +function typeLabeler(mod: IrModule): (t: IrType) => string { + const arms = new Map((mod.unions ?? []).map((def) => [def.id, def.arms])); + return (t) => { + if (t.kind !== "union") return typeKey(t); + const list = arms.get(t.unionId); + return list === undefined ? typeKey(t) : list.map(typeKey).join(" | "); + }; +} diff --git a/packages/runtime/src/scr_lib.c b/packages/runtime/src/scr_lib.c index 0b780603b..df36f063e 100644 --- a/packages/runtime/src/scr_lib.c +++ b/packages/runtime/src/scr_lib.c @@ -225,7 +225,17 @@ static void scr_module_registry_cleanup(void) { * own cleanup only when the value is first materialized. That is equivalent * at process exit and retains the RC-audit cleanup guarantee for the values a * program actually observes. */ +#if defined(__linux__) +/* The heap census of a program built with SCRIPTC_HEAP_CENSUS + * (heap-census.ts) writes its counters at exit, from here and from + * scr_process_exit; every other build leaves the weak reference null. */ +extern void scr_census_write(void) __attribute__((weak)); +#endif + static void scr_lib_cleanup(void) { +#if defined(__linux__) + if (scr_census_write != NULL) scr_census_write(); +#endif scr_module_registry_cleanup(); scr_arr_release(scr_argv_arr); scr_argv_arr = NULL; @@ -2137,6 +2147,7 @@ void scr_process_exit(double code) { fflush(stdout); #if defined(__linux__) if (__llvm_profile_write_file != NULL) (void)__llvm_profile_write_file(); + if (scr_census_write != NULL) scr_census_write(); scr_detach_teardown(); #endif _Exit((int)code); diff --git a/tests/harness/heap-census.test.ts b/tests/harness/heap-census.test.ts new file mode 100644 index 000000000..b8ca2b799 --- /dev/null +++ b/tests/harness/heap-census.test.ts @@ -0,0 +1,113 @@ +/* The heap census (backend/llvm/heap-census.ts): a program built with + * SCRIPTC_HEAP_CENSUS counts class instances and the stores, lengths and + * accesses of array-typed fields, and writes them at exit, on a normal exit + * and on process.exit alike. */ +import { execFile } from "node:child_process"; +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { promisify } from "node:util"; +import { expect, test } from "vitest"; +import { compile } from "@scriptc/compiler"; + +const exec = promisify(execFile); +const sanitize = process.env["SCRIPTC_SAN"] === "1"; + +const PROGRAM = ` +class Node { + kids: Node[]; + tags: string[] | undefined; + constructor(kids: Node[]) { + this.kids = kids; + if (kids.length > 1) this.tags = ["a", "b"]; + else this.tags = undefined; + } +} +let total = 0; +const roots: Node[] = []; +for (let i = 0; i < 100; i++) { + const leaves: Node[] = []; + for (let j = 0; j < i % 6; j++) leaves.push(new Node([])); + roots.push(new Node(leaves)); +} +for (const r of roots) total += r.kids.length + (r.tags?.length ?? 0); +console.log(total); +`; + +async function census(source: string): Promise<{ stdout: string; counts: Map }> { + const dir = await mkdtemp(join(tmpdir(), "scriptc-heap-census-")); + const saved = process.env["SCRIPTC_HEAP_CENSUS"]; + try { + const entry = join(dir, "main.ts"); + await writeFile(entry, source); + process.env["SCRIPTC_HEAP_CENSUS"] = "1"; + const result = await compile(entry, { + backend: "llvm", + sanitize, + outDir: dir, + outPath: join(dir, "program"), + }); + if (saved === undefined) delete process.env["SCRIPTC_HEAP_CENSUS"]; + else process.env["SCRIPTC_HEAP_CENSUS"] = saved; + expect(result.ok, JSON.stringify(result.diagnostics)).toBe(true); + if (!result.ok) throw new Error("compile failed"); + const out = join(dir, "census.tsv"); + const { stdout } = await exec(result.binaryPath, [], { + env: { ...process.env, SCRIPTC_HEAP_CENSUS: out }, + timeout: 120_000, + }); + const counts = new Map(); + for (const line of (await readFile(out, "utf8")).trimEnd().split("\n")) { + const [kind, cls, field, count] = line.split("\t"); + counts.set(`${kind} ${cls}${field ? "." + field : ""}`, Number(count)); + } + return { stdout, counts }; + } finally { + if (saved === undefined) delete process.env["SCRIPTC_HEAP_CENSUS"]; + else process.env["SCRIPTC_HEAP_CENSUS"] = saved; + await rm(dir, { recursive: true, force: true }); + } +} + +const EXPECTED: [string, number][] = [ + // 100 roots and 246 leaves (i % 6 children each). + ["new Node", 346], + ["store Node.kids", 346], + ["elements Node.kids", 246], + ["len0 Node.kids", 263], + ["len1 Node.kids", 17], + ["len2 Node.kids", 17], + ["len3-4 Node.kids", 33], + ["len5-8 Node.kids", 16], + ["len9-16 Node.kids", 0], + // Literals count where the store itself writes one. + ["literal Node.kids", 0], + ["store Node.tags", 346], + ["literal Node.tags", 66], + // An undefined tags field counts as length 0. + ["len0 Node.tags", 280], + ["len2 Node.tags", 66], + ["elements Node.tags", 132], +]; + +test.skipIf(process.platform !== "linux")( + "the census counts instances and array field stores at a normal exit", + async () => { + const { stdout, counts } = await census(PROGRAM); + expect(stdout).toBe("378\n"); + for (const [key, count] of EXPECTED) expect([key, counts.get(key)]).toEqual([key, count]); + // Every store also computes the field's address. + expect(counts.get("access Node.kids")).toBeGreaterThanOrEqual(346 + 100); + }, + 240_000, +); + +test.skipIf(process.platform !== "linux")( + "the census is written on process.exit too", + async () => { + const { stdout, counts } = await census(PROGRAM + "process.exit(0);\n"); + expect(stdout).toBe("378\n"); + expect(counts.get("new Node")).toBe(346); + }, + 240_000, +); From 60d5dc8ab178293f36045c3a0341bd37eb467cb2 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Tue, 27 Oct 2026 01:54:48 -0500 Subject: [PATCH 2/6] Count class-value constructions in the census, and key caches on both diagnostics - The heap census also counts instances a class value constructs: the construct thunk that `newValue` calls counts them after its allocation, as `new` does at its own. - A build with SCRIPTC_HEAP_CENSUS set is its own executable cache entry (`heapCensus` in the early cache's key and route key), and one with SCRIPTC_REPR_REPORT set does not read the early cache, since a hit emits no module and so writes no report. The CLI's startup route defers both to the full compiler. - Test: a warm cache still builds the census executable, counting `new` and class-value constructions, and still writes the report. --- packages/cli/src/bootstrap.ts | 6 ++- packages/cli/test/executable-cache.test.ts | 54 +++++++++++++++++++ packages/compiler/src/backend/llvm/classes.ts | 2 + packages/compiler/src/backend/llvm/emitter.ts | 1 + .../compiler/src/backend/llvm/heap-census.ts | 21 ++++++-- .../compiler/src/backend/llvm/repr-report.ts | 12 +++-- packages/compiler/src/backend/llvm/shapes.ts | 3 ++ .../src/executable/executable-cache.ts | 4 ++ packages/compiler/src/index.ts | 6 ++- 9 files changed, 100 insertions(+), 9 deletions(-) diff --git a/packages/cli/src/bootstrap.ts b/packages/cli/src/bootstrap.ts index f6efec134..0c09c4435 100644 --- a/packages/cli/src/bootstrap.ts +++ b/packages/cli/src/bootstrap.ts @@ -74,7 +74,11 @@ async function tryFastPath(): Promise { values["target-cpu"] !== undefined || (values["external-types"] ?? []).length > 0 || // The full CLI validates the Node island policy flags. - hasNodeIslandFlags(values) + hasNodeIslandFlags(values) || + // The full compiler writes the representation report and keys the heap + // census build (backend/llvm/repr-report.ts, heap-census.ts). + (process.env["SCRIPTC_REPR_REPORT"] ?? "") !== "" || + (process.env["SCRIPTC_HEAP_CENSUS"] ?? "") !== "" ) return null; const backend = values.backend; diff --git a/packages/cli/test/executable-cache.test.ts b/packages/cli/test/executable-cache.test.ts index ee8027765..eef8e97a0 100644 --- a/packages/cli/test/executable-cache.test.ts +++ b/packages/cli/test/executable-cache.test.ts @@ -198,3 +198,57 @@ test.skipIf(process.platform === "win32")( }, 120_000, ); + +test.skipIf(process.platform !== "linux")( + "a warm executable cache still builds the heap census and writes the representation report", + async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-cli-diagnostic-cache-")); + const cacheRoot = join(dir, "cache"); + const entry = join(dir, "main.ts"); + const outPath = join(dir, "program"); + try { + await mkdir(cacheRoot, { mode: 0o700 }); + // `new` and construction through a class value both count. + await writeFile( + entry, + [ + "class Item { constructor(readonly id: number) {} }", + "const kind: typeof Item = Item;", + "const items = [new Item(1), new kind(2), new kind(3)];", + "console.log(items.length);", + "", + ].join("\n"), + ); + const build = async (env: NodeJS.ProcessEnv = {}): Promise => + ( + await execFileAsync( + process.execPath, + ["--import", tsxLoader, cliEntry, "build", entry, "-o", outPath], + { + env: { ...process.env, SCRIPTC_CACHE_DIR: cacheRoot, SCRIPTC_TIMING: "1", ...env }, + maxBuffer: 4 * 1024 * 1024, + }, + ) + ).stderr; + expect(await build()).toContain("scriptc lowering"); + expect(await build()).not.toContain("scriptc lowering"); + + // The census build is its own cache entry. + const census = join(dir, "census.tsv"); + await build({ SCRIPTC_HEAP_CENSUS: "1" }); + const { stdout } = await execFileAsync(outPath, [], { + env: { ...process.env, SCRIPTC_HEAP_CENSUS: census }, + }); + expect(stdout).toBe("3\n"); + expect(await readFile(census, "utf8")).toMatch(/^new\tItem\t\t3\t[1-9]\d*$/m); + + // A report is written even when the executable is cached. + const report = join(dir, "repr.tsv"); + await build({ SCRIPTC_REPR_REPORT: report }); + expect(await readFile(report, "utf8")).toMatch(/^field\tItem\tid\t/m); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }, + 600_000, +); diff --git a/packages/compiler/src/backend/llvm/classes.ts b/packages/compiler/src/backend/llvm/classes.ts index 695acc273..94cfcaf25 100644 --- a/packages/compiler/src/backend/llvm/classes.ts +++ b/packages/compiler/src/backend/llvm/classes.ts @@ -1712,6 +1712,8 @@ export function emitClassObjDefs( `define internal ptr @${mangleCtorThunk(className)}(ptr %class${paramDecls ? ", " + paramDecls : ""}) ${FN_ATTRS} { ; construct thunk ${className}`, `entry:`, ` %o = call ptr @${mangleClassNew(className)}()`, + // Construction through a class value: the census counts it here. + ...(host.heapCensus?.newLines(className) ?? []), ...(meta.def.localCaptures !== undefined ? [ ` %class.owned = call ptr @scr_classobj_retain_v(ptr %class)`, diff --git a/packages/compiler/src/backend/llvm/emitter.ts b/packages/compiler/src/backend/llvm/emitter.ts index 9278613da..0a20e1b1a 100644 --- a/packages/compiler/src/backend/llvm/emitter.ts +++ b/packages/compiler/src/backend/llvm/emitter.ts @@ -2366,6 +2366,7 @@ export class LlEmitter { virtualEntry: (implFn) => this.virtualEntry(implFn), programLifetime: this.programLifetime, regionAliasTag: (word) => this.memoryAliasAttachment(`region.${word}`), + heapCensus: this.heapCensus, }; // An instance of a class that extends Array is an array: a slot of its // element array type can hold it, and the collector visits it there as diff --git a/packages/compiler/src/backend/llvm/heap-census.ts b/packages/compiler/src/backend/llvm/heap-census.ts index 91b9839cd..b55e9341b 100644 --- a/packages/compiler/src/backend/llvm/heap-census.ts +++ b/packages/compiler/src/backend/llvm/heap-census.ts @@ -1,6 +1,7 @@ /* The heap census. A program built with SCRIPTC_HEAP_CENSUS set counts, for * ranking representation work: - * - the instances each `new` of a class creates, with the class's size; + * - the instances of each class that `new` or a class value constructs, + * with the class's size; * - for each array-typed class field, its stores (with the stored array's * length in buckets and the total of those lengths), the stores whose * value is an array literal written at the store, and its accesses; @@ -17,10 +18,16 @@ import { classStructSym } from "./classes.js"; import { llvmBytes, wtf8Bytes } from "../literals.js"; import type { NullableUnions } from "./nullable-unions.js"; +/** Whether this build counts the heap census: SCRIPTC_HEAP_CENSUS is set + * and nonempty. The executable cache keys on it. */ +export function heapCensusRequested(env: NodeJS.ProcessEnv = process.env): boolean { + const flag = env["SCRIPTC_HEAP_CENSUS"]; + return flag !== undefined && flag !== ""; +} + /** The census of the module being built, or null when it is off. */ export function heapCensusOf(mod: IrModule, unions: NullableUnions): HeapCensus | null { - const flag = process.env["SCRIPTC_HEAP_CENSUS"]; - if (flag === undefined || flag === "" || mod.lib !== undefined) return null; + if (!heapCensusRequested() || mod.lib !== undefined) return null; return new HeapCensus(mod, unions); } @@ -58,8 +65,14 @@ export class HeapCensus { /** Counts one instance of `className`. */ countNew(line: (text: string) => void, className: string): void { + for (const text of this.newLines(className)) line(text); + } + + /** The lines that count one instance of `className`: a `new`, or a + * construct thunk for class values (`newValue`). */ + newLines(className: string): string[] { const index = this.instances.get(className); - if (index !== undefined) line(this.increment(index, "1")); + return index === undefined ? [] : [this.increment(index, "1")]; } /** Counts an access of a field; true when the field is counted. */ diff --git a/packages/compiler/src/backend/llvm/repr-report.ts b/packages/compiler/src/backend/llvm/repr-report.ts index 82104277c..8d6e48f30 100644 --- a/packages/compiler/src/backend/llvm/repr-report.ts +++ b/packages/compiler/src/backend/llvm/repr-report.ts @@ -12,12 +12,18 @@ import { INLINE_UNION_FIELD } from "./inline-unions.js"; export const REPR_REPORT_HEADER = "kind\towner\tname\ttype\tstorage\treason"; +/** The file SCRIPTC_REPR_REPORT names, or null. A build that writes the + * report skips the executable cache, whose hit would emit no module. */ +export function representationReportPath(env: NodeJS.ProcessEnv = process.env): string | null { + const path = env["SCRIPTC_REPR_REPORT"]; + return path === undefined || path === "" ? null : path; +} + /** Writes the report of `emitter`'s module when SCRIPTC_REPR_REPORT names a * file. */ export function writeRepresentationReport(emitter: LlEmitter): void { - const path = process.env["SCRIPTC_REPR_REPORT"]; - if (path === undefined || path === "") return; - writeFileSync(path, representationReport(emitter)); + const path = representationReportPath(); + if (path !== null) writeFileSync(path, representationReport(emitter)); } export function representationReport(emitter: LlEmitter): string { diff --git a/packages/compiler/src/backend/llvm/shapes.ts b/packages/compiler/src/backend/llvm/shapes.ts index b5b1e735a..e48ab8e8c 100644 --- a/packages/compiler/src/backend/llvm/shapes.ts +++ b/packages/compiler/src/backend/llvm/shapes.ts @@ -2,6 +2,7 @@ * Containers store erased adapters; shape helpers retain, release and trace * their fields. Every allocation has a size_t reference count at offset zero * and participates in the runtime allocation audit. */ +import type { HeapCensus } from "./heap-census.js"; import type { IrClassDef, IrModule, IrRecordShape, IrType } from "../../ir/ir.js"; import { isIdentityCollectionKey, @@ -78,6 +79,8 @@ const CYC_MAX_TYPES = 32768; /** What the tables need from the emitter: the extern-declaration ledger * and the module-wide cycle/shape indexes. */ export interface ShapeHost { + /** The counters of a build instrumented for the heap census (heap-census.ts). */ + readonly heapCensus?: HeapCensus | null; /** Chunk emitters only (emit-journal.ts): EmitJournal's allocated, built, * opened, used and entry. */ journalAllocated?: ((name: string, op: string, args: readonly unknown[]) => void) | undefined; diff --git a/packages/compiler/src/executable/executable-cache.ts b/packages/compiler/src/executable/executable-cache.ts index 6124a6edf..8f90a9d0c 100644 --- a/packages/compiler/src/executable/executable-cache.ts +++ b/packages/compiler/src/executable/executable-cache.ts @@ -168,6 +168,8 @@ export interface EarlyExecutableCacheOptions { checker?: string; /** Omitted unless SCRIPTC_YOUNG_NURSERY=1 (backend/young-nursery.ts). */ youngNursery?: true; + /** Omitted unless SCRIPTC_HEAP_CENSUS is set (backend/llvm/heap-census.ts). */ + heapCensus?: true; npmStatic: readonly string[] | "auto" | null; /** Omitted unless shape inference's overlay compiles the packages * (frontend/shape-inference). */ @@ -324,6 +326,7 @@ function cacheKey(options: EarlyExecutableCacheOptions): string { ...(options.functionLayout !== undefined ? [`layout:${options.functionLayout}`] : []), ...(options.checker !== undefined ? [`checker:${options.checker}`] : []), ...(options.youngNursery ? ["young-nursery"] : []), + ...(options.heapCensus ? ["heap-census"] : []), options.npmStatic === null ? "" : options.npmStatic === "auto" @@ -370,6 +373,7 @@ function routeKey(options: Omit Date: Tue, 27 Oct 2026 02:49:05 -0500 Subject: [PATCH 3/6] Keep the heap census hook out of library archives Libraries never build the census (heapCensusOf), but the runtime's weak scr_census_write reference made library archives list an external symbol outside their declared set. Declare and call it only in programs. --- packages/runtime/src/scr_lib.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/packages/runtime/src/scr_lib.c b/packages/runtime/src/scr_lib.c index df36f063e..ad42a5824 100644 --- a/packages/runtime/src/scr_lib.c +++ b/packages/runtime/src/scr_lib.c @@ -225,15 +225,16 @@ static void scr_module_registry_cleanup(void) { * own cleanup only when the value is first materialized. That is equivalent * at process exit and retains the RC-audit cleanup guarantee for the values a * program actually observes. */ -#if defined(__linux__) +#if defined(__linux__) && !defined(SCR_LIB) /* The heap census of a program built with SCRIPTC_HEAP_CENSUS * (heap-census.ts) writes its counters at exit, from here and from - * scr_process_exit; every other build leaves the weak reference null. */ + * scr_process_exit; every other program leaves the weak reference null. + * Libraries have no census, and their archives declare no such symbol. */ extern void scr_census_write(void) __attribute__((weak)); #endif static void scr_lib_cleanup(void) { -#if defined(__linux__) +#if defined(__linux__) && !defined(SCR_LIB) if (scr_census_write != NULL) scr_census_write(); #endif scr_module_registry_cleanup(); @@ -2147,7 +2148,9 @@ void scr_process_exit(double code) { fflush(stdout); #if defined(__linux__) if (__llvm_profile_write_file != NULL) (void)__llvm_profile_write_file(); +#ifndef SCR_LIB if (scr_census_write != NULL) scr_census_write(); +#endif scr_detach_teardown(); #endif _Exit((int)code); From 9179a881a4f1a065338214ccd04554a3ff6e31da Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Tue, 27 Oct 2026 03:24:16 -0500 Subject: [PATCH 4/6] Skip the helper version probe and unchanged files' digests on rebuilds Every native build started the LLVM helper once for `version` before emitting, and hashed every file the frontend read, including the supplied declaration files that do not change between builds. - verifyNativeHelper takes an optional record of helpers that already answered `version` as expected. The native compiler keeps it in its cache (family "helper"), keyed by the helper's and the compiler's file identity, the compiler version and the target. The probe runs again when the helper file changes. The package name and version are still checked every time. - FrontendInputTracker takes the snapshot of the cache entry a build replaces. A file whose settled identity has not changed since that snapshot keeps the digest recorded there, which is the trust frontendInputsStillMatch already gives an unchanged identity. Builds produce the same executables. --- .../compiler/src/backend/native-tools.test.ts | 92 +++++++++++++++++++ packages/compiler/src/backend/native-tools.ts | 16 +++- .../src/frontend/input-tracker.test.ts | 31 +++++++ .../compiler/src/frontend/input-tracker.ts | 23 ++++- packages/compiler/src/native/cache.ts | 1 + packages/compiler/src/native/compiler.ts | 46 +++++++++- packages/compiler/src/native/prepare.ts | 5 +- 7 files changed, 206 insertions(+), 8 deletions(-) create mode 100644 packages/compiler/src/backend/native-tools.test.ts diff --git a/packages/compiler/src/backend/native-tools.test.ts b/packages/compiler/src/backend/native-tools.test.ts new file mode 100644 index 000000000..11bb8f3ba --- /dev/null +++ b/packages/compiler/src/backend/native-tools.test.ts @@ -0,0 +1,92 @@ +import { execFileSync } from "node:child_process"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, expect, test, vi } from "vitest"; +import { + NATIVE_CODEGEN_LLVM_VERSION, + NATIVE_CODEGEN_PROTOCOL_VERSION, +} from "./native-codegen-core.js"; +import { verifyNativeHelper } from "./native-tools.js"; +import { LINUX_X64_GNU_TARGET } from "./targets.js"; + +vi.mock("node:child_process", async (original) => ({ + ...(await original()), + execFileSync: vi.fn(), +})); + +const directories: string[] = []; +afterEach(() => { + vi.resetAllMocks(); + for (const path of directories.splice(0)) rmSync(path, { recursive: true, force: true }); +}); + +/** A helper package directory and the options that verify it. */ +function helperOptions() { + const packageRoot = mkdtempSync(join(tmpdir(), "scriptc-helper-")); + directories.push(packageRoot); + const target = LINUX_X64_GNU_TARGET; + const helper = target.helper; + writeFileSync( + join(packageRoot, "package.json"), + JSON.stringify({ name: helper.packageName, version: "1.2.3" }), + ); + vi.mocked(execFileSync).mockReturnValue( + JSON.stringify({ + ok: true, + protocol_version: NATIVE_CODEGEN_PROTOCOL_VERSION, + scriptc_package_version: "1.2.3", + llvm_version: NATIVE_CODEGEN_LLVM_VERSION, + default_target: helper.defaultTarget, + data_layout: helper.defaultDataLayout, + targets: [target.llvmBackend], + supported_targets: [target.llvmTriple], + host_triple: target.llvmTriple, + }), + ); + return { + executable: join(packageRoot, "helper"), + packageRoot, + compilerVersion: "1.2.3", + target, + helper, + }; +} + +test("a helper the verification record knows is not asked for its version again", () => { + const options = helperOptions(); + let known = false; + const remember = vi.fn(() => { + known = true; + }); + const verified = { known: () => known, remember }; + verifyNativeHelper({ ...options, verified }); + expect(execFileSync).toHaveBeenCalledTimes(1); + expect(remember).toHaveBeenCalledTimes(1); + verifyNativeHelper({ ...options, verified }); + expect(execFileSync).toHaveBeenCalledTimes(1); + expect(remember).toHaveBeenCalledTimes(1); +}); + +test("a helper whose version answer is rejected is not recorded", () => { + const options = helperOptions(); + vi.mocked(execFileSync).mockReturnValue(JSON.stringify({ ok: false })); + const remember = vi.fn(); + expect(() => + verifyNativeHelper({ ...options, verified: { known: () => false, remember } }), + ).toThrow(/incompatible/); + expect(remember).not.toHaveBeenCalled(); +}); + +test("the package identity is checked even for a helper the record knows", () => { + const options = helperOptions(); + const remember = vi.fn(); + expect(() => + verifyNativeHelper({ + ...options, + compilerVersion: "9.9.9", + verified: { known: () => true, remember }, + }), + ).toThrow(/package mismatch/); + expect(execFileSync).not.toHaveBeenCalled(); +}); diff --git a/packages/compiler/src/backend/native-tools.ts b/packages/compiler/src/backend/native-tools.ts index 71ad26111..5056de491 100644 --- a/packages/compiler/src/backend/native-tools.ts +++ b/packages/compiler/src/backend/native-tools.ts @@ -56,6 +56,7 @@ export function emitNativeObject(options: { partitionCache?: string; /** Profile-guided optimization: instrumentation or an indexed profile. */ profile?: NativeProfileOptions; + verified?: NativeHelperVerification; }): { outputs: string[]; profileMatch: NativeProfileMatch | null } { verifyNativeHelper(options); const outputs = nativePartitionPaths(options.outputPath, options.partitions ?? 1); @@ -111,13 +112,24 @@ export function mergeNativeProfiles(options: { ]); } -/** Verify installed identity even when a caller can reuse a cached object. */ +/** Helpers that answered `version` as expected: `known` is true when the + * helper file, as it is now, already did, and `remember` records that this + * one just did. */ +export interface NativeHelperVerification { + known(): boolean; + remember(): void; +} + +/** Verify installed identity even when a caller can reuse a cached object. + * With `verified`, the version probe runs only for a helper it does not + * know yet. */ export function verifyNativeHelper(options: { executable: string; packageRoot: string; compilerVersion: string; target: NativeTargetSpec; helper: NativeHelperSpec; + verified?: NativeHelperVerification; }): void { const identity = JSON.parse(readFileSync(options.packageRoot + "/package.json", "utf8")) as { name: string; @@ -131,8 +143,10 @@ export function verifyNativeHelper(options: { `LLVM helper package mismatch: expected ${options.helper.packageName}@${options.compilerVersion}`, ); } + if (options.verified?.known() === true) return; const version = JSON.parse( runNativeTool(options.executable, ["version", "--format=json"]), ) as Record; validateNativeCodegenVersion(version, options.target, options.helper, options.compilerVersion); + options.verified?.remember(); } diff --git a/packages/compiler/src/frontend/input-tracker.test.ts b/packages/compiler/src/frontend/input-tracker.test.ts index a320eb2bd..754a59c2f 100644 --- a/packages/compiler/src/frontend/input-tracker.test.ts +++ b/packages/compiler/src/frontend/input-tracker.test.ts @@ -1,3 +1,4 @@ +import { createHash } from "node:crypto"; import * as fs from "node:fs"; import { chmod, mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; @@ -301,6 +302,36 @@ test("a settled file is checked by its identity, and any write invalidates it", expect(frontendInputsStillMatch(snapshot)).toBe(false); }); +test("a settled file an earlier run read keeps that run's digest until it is written", async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-inputs-")); + scratch.push(dir); + const source = join(dir, "entry.ts"); + await writeFile(source, "export const answer = 1;\n"); + const settledRead = (tracker: FrontendInputTracker, later: number) => { + vi.useFakeTimers({ toFake: ["Date"] }); + vi.setSystemTime(Date.now() + later); + try { + tracker.runSynchronous(() => trackedReadFile(source)); + } finally { + vi.useRealTimers(); + } + const probe = tracker.snapshot().probes.find((entry) => entry.op === "file"); + return probe?.op === "file" ? probe : undefined; + }; + const first = settledRead(new FrontendInputTracker(), 5000)!; + expect(first.identity).toBeDefined(); + // A digest only the previous snapshot holds shows it was not computed again. + const marker = "f".repeat(64); + const previous = { version: 1 as const, stable: true, probes: [{ ...first, digest: marker }] }; + expect(settledRead(new FrontendInputTracker(previous), 5000)?.digest).toBe(marker); + await writeFile(source, "export const answer = 2;\n"); + const rewritten = settledRead(new FrontendInputTracker(previous), 10000); + expect(rewritten?.identity).not.toBe(first.identity); + expect(rewritten?.digest).toBe( + createHash("sha256").update("export const answer = 2;\n").digest("hex"), + ); +}); + test("a file changed shortly before the read is hashed on every check", async () => { const dir = await mkdtemp(join(tmpdir(), "scriptc-inputs-")); scratch.push(dir); diff --git a/packages/compiler/src/frontend/input-tracker.ts b/packages/compiler/src/frontend/input-tracker.ts index 18081576b..f91e926c4 100644 --- a/packages/compiler/src/frontend/input-tracker.ts +++ b/packages/compiler/src/frontend/input-tracker.ts @@ -125,6 +125,23 @@ let synchronousTracker: FrontendInputTracker | undefined; export class FrontendInputTracker { private readonly probes = new Map(); private stable = true; + /** The digests `previous` recorded for files read with a settled identity. */ + private readonly known = new Map(); + + /** `previous` is the snapshot of an earlier run whose cache entry is being + * rebuilt: a file whose settled identity has not changed since then has the + * content it hashed, so its digest is not computed again. */ + constructor(previous?: FrontendInputSnapshot) { + for (const probe of previous?.probes ?? []) + if (probe.op === "file" && probe.identity !== undefined) + this.known.set(probe.path, { identity: probe.identity, digest: probe.digest }); + } + + /** The digest an earlier run recorded for `path` with this identity. */ + knownDigest(path: string, identity: string): string | undefined { + const known = this.known.get(path); + return known?.identity === identity ? known.digest : undefined; + } run(fn: () => T): T { const parent = activeTracker.getStore(); @@ -252,7 +269,11 @@ export function trackedReadFile(path: string): string | null { if (virtual !== undefined) return virtual; try { const { text, identity } = readWithIdentity(path); - const digest = frontendSourceDigest(text); + const known = + identity === null + ? undefined + : (synchronousTracker ?? activeTracker.getStore())?.knownDigest(path, identity); + const digest = known ?? frontendSourceDigest(text); record( identity === null ? { op: "file", path, digest } : { op: "file", path, digest, identity }, ); diff --git a/packages/compiler/src/native/cache.ts b/packages/compiler/src/native/cache.ts index 0250e9618..551a151a1 100644 --- a/packages/compiler/src/native/cache.ts +++ b/packages/compiler/src/native/cache.ts @@ -170,6 +170,7 @@ export class NativeCache { "kept", "linker-job", "runtime-pack", + "helper", "binary", "dsym", "coverage", diff --git a/packages/compiler/src/native/compiler.ts b/packages/compiler/src/native/compiler.ts index 1cf4f7eac..2ed2cbc16 100644 --- a/packages/compiler/src/native/compiler.ts +++ b/packages/compiler/src/native/compiler.ts @@ -28,6 +28,7 @@ import { requireNativeArtifact, runNativeTool, verifyNativeHelper, + type NativeHelperVerification, } from "../backend/native-tools.js"; import { type NativeRuntimeSelection, @@ -192,6 +193,44 @@ export class NativeCompiler { }; } + /** The helpers whose `version` answer this cache recorded, by file identity: + * the probe runs again only when the helper file changes. */ + private helperVerification(): { verified?: NativeHelperVerification } { + const cache = this.cache; + const toolchain = this.toolchain; + if (cache === null) return {}; + let identity: string; + let key: string; + try { + identity = nativeFileIdentity(toolchain.helperExecutable); + key = contentDigest( + JSON.stringify({ + schema: 1, + helper: identity, + compiler: nativeFileIdentity(process.execPath), + version: toolchain.compilerVersion, + target: toolchain.target.name, + package: toolchain.helper.packageName, + }), + ); + } catch { + return {}; + } + return { + verified: { + known: () => cache.read("helper", key) !== null, + remember: () => { + try { + if (nativeFileIdentity(toolchain.helperExecutable) === identity) + cache.write("helper", key, "verified\n"); + } catch { + /* A failed write only means probing again next time. */ + } + }, + }, + }; + } + private emitObject( input: string, output: string, @@ -211,6 +250,7 @@ export class NativeCompiler { compilerVersion: toolchain.compilerVersion, target: toolchain.target, helper: toolchain.helper, + ...this.helperVerification(), }; let key: string | null = null; let identity: string | null = null; @@ -266,11 +306,7 @@ export class NativeCompiler { ? null : this.cache.partitionDirectory(identity); const { profileMatch } = emitNativeObject({ - executable: toolchain.helperExecutable, - packageRoot: toolchain.helperPackageRoot, - compilerVersion: toolchain.compilerVersion, - target: toolchain.target, - helper: toolchain.helper, + ...helperOptions, inputPath: input, outputPath: output, sourcePath: source, diff --git a/packages/compiler/src/native/prepare.ts b/packages/compiler/src/native/prepare.ts index 4300c3c25..339a50faa 100644 --- a/packages/compiler/src/native/prepare.ts +++ b/packages/compiler/src/native/prepare.ts @@ -370,6 +370,8 @@ export function prepareNativeExecutable( const exclusions = { outputPaths, outputDirectories: [...directories] }; const irInput = process.env[BOOTSTRAP_IR_INPUT] ?? ""; let key: string | null = null; + // The entry this build replaces: its probes give the digests of unchanged files. + let previous: FrontendInputSnapshot | undefined; if (cache !== null && irInput === "" && provenanceSources() === null) { try { key = contentDigest( @@ -389,6 +391,7 @@ export function prepareNativeExecutable( timing("frontend-cache-hit"); return cached.input; } + if (cached !== null) previous = cached.stale.probes; const equivalent = cached === null || outputKind === "ir" ? null @@ -403,7 +406,7 @@ export function prepareNativeExecutable( } } timing("frontend-cache-miss"); - const tracker = new FrontendInputTracker(); + const tracker = new FrontendInputTracker(previous); const prepared = irInput !== "" ? preparedFromIr(irInput, entry, options) From 60372c09a457f826c954dc9d87e59e8f840eb6ab Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Tue, 27 Oct 2026 03:28:32 -0500 Subject: [PATCH 5/6] Declare the heap census writer in scr_lib.h The LLVM backend can emit scr_census_write, which a census program defines and the runtime calls through a weak reference, so the header declares it as it does scr_detached_exit. --- packages/runtime/src/scr_lib.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/runtime/src/scr_lib.h b/packages/runtime/src/scr_lib.h index e01bc7b23..1d8d91d97 100644 --- a/packages/runtime/src/scr_lib.h +++ b/packages/runtime/src/scr_lib.h @@ -150,6 +150,10 @@ void scr_process_exit(double code); * helper process (Linux, `--detached-exit`). scr_lib.c defines a weak 0; * the program object of such an executable defines it as 1. */ extern const unsigned char scr_detached_exit; +/* The heap census writer of a program built with SCRIPTC_HEAP_CENSUS + * (heap-census.ts): that program's object defines it, and scr_lib.c calls + * it at exit through a weak reference (programs on Linux only). */ +void scr_census_write(void); void scr_process_set_exit_code(double code); double scr_process_exit_code_or_zero(void); /* The writer and the presence test of programs that read process.exitCode: From 7c770d08877c27a651c60625b8ddec745acb35b2 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Tue, 27 Oct 2026 03:30:15 -0500 Subject: [PATCH 6/6] Name the source file in empty split codegen parts When --split codegen keeps the module whole, the extra outputs are new empty modules, and their source file name defaulted to the module identifier: the path of the private copy of the LLVM input, which has a random name. That name reached each part's STT_FILE symbol, so two release builds of one program differed in .strtab. The empty parts now take the module's source file name, as the first part has it. --- native/llvm-codegen/src/emit.cpp | 2 ++ packages/compiler/test/native-partitions.test.ts | 10 ++++++++++ 2 files changed, 12 insertions(+) diff --git a/native/llvm-codegen/src/emit.cpp b/native/llvm-codegen/src/emit.cpp index 1ab0f6ecd..d3d868fae 100644 --- a/native/llvm-codegen/src/emit.cpp +++ b/native/llvm-codegen/src/emit.cpp @@ -1241,6 +1241,7 @@ emitSplitCodegen(const EmitOptions &Options, Module &M, TargetMachine &Machine, size_t Count = Options.Outputs.size(); std::optional Split = splitForCodegen(M, Count); std::string Identifier = M.getModuleIdentifier(); + std::string SourceFileName = M.getSourceFileName(); Triple TargetTriple = M.getTargetTriple(); DataLayout Layout = M.getDataLayout(); std::vector>> Temporaries(Count); @@ -1274,6 +1275,7 @@ emitSplitCodegen(const EmitOptions &Options, Module &M, TargetMachine &Machine, keepCodegenPart(*Part, Split->PartOf, I); } else { Part = std::make_unique(Identifier, Context); + Part->setSourceFileName(SourceFileName); Part->setTargetTriple(TargetTriple); Part->setDataLayout(Layout); } diff --git a/packages/compiler/test/native-partitions.test.ts b/packages/compiler/test/native-partitions.test.ts index c86a37aa4..18d0aa749 100644 --- a/packages/compiler/test/native-partitions.test.ts +++ b/packages/compiler/test/native-partitions.test.ts @@ -327,6 +327,16 @@ describe.runIf(helper !== null)("LLVM program partitions", () => { const first = await readFile(parts[0]!); const [single] = await emit(dir, source, { outputs: 1, optLevel: "2", name: "whole" }); expect(await readFile(single!)).toEqual(first); + // The empty parts name the source path, so they do not depend on the + // input's file name either. + const renamed = await emit(dir, source, { + outputs: 3, + optLevel: "2", + split: true, + name: "renamed", + }); + for (let index = 0; index < parts.length; index++) + expect(await readFile(renamed[index]!)).toEqual(await readFile(parts[index]!)); }); test("split code generation requires several optimized outputs and no cache", async () => {