mirror of
https://github.com/vercel-labs/scriptc.git
synced 2026-10-30 07:14:39 +08:00
Merge main
This commit is contained in:
@@ -1241,6 +1241,7 @@ emitSplitCodegen(const EmitOptions &Options, Module &M, TargetMachine &Machine,
|
||||
size_t Count = Options.Outputs.size();
|
||||
std::optional<CodegenSplit> Split = splitForCodegen(M, Count);
|
||||
std::string Identifier = M.getModuleIdentifier();
|
||||
std::string SourceFileName = M.getSourceFileName();
|
||||
Triple TargetTriple = M.getTargetTriple();
|
||||
DataLayout Layout = M.getDataLayout();
|
||||
std::vector<std::vector<SmallString<256>>> Temporaries(Count);
|
||||
@@ -1274,6 +1275,7 @@ emitSplitCodegen(const EmitOptions &Options, Module &M, TargetMachine &Machine,
|
||||
keepCodegenPart(*Part, Split->PartOf, I);
|
||||
} else {
|
||||
Part = std::make_unique<Module>(Identifier, Context);
|
||||
Part->setSourceFileName(SourceFileName);
|
||||
Part->setTargetTriple(TargetTriple);
|
||||
Part->setDataLayout(Layout);
|
||||
}
|
||||
|
||||
@@ -74,7 +74,11 @@ async function tryFastPath(): Promise<number | null> {
|
||||
values["target-cpu"] !== undefined ||
|
||||
(values["external-types"] ?? []).length > 0 ||
|
||||
// The full CLI validates the Node island policy flags.
|
||||
hasNodeIslandFlags(values)
|
||||
hasNodeIslandFlags(values) ||
|
||||
// The full compiler writes the representation report and keys the heap
|
||||
// census build (backend/llvm/repr-report.ts, heap-census.ts).
|
||||
(process.env["SCRIPTC_REPR_REPORT"] ?? "") !== "" ||
|
||||
(process.env["SCRIPTC_HEAP_CENSUS"] ?? "") !== ""
|
||||
)
|
||||
return null;
|
||||
const backend = values.backend;
|
||||
|
||||
@@ -198,3 +198,57 @@ test.skipIf(process.platform === "win32")(
|
||||
},
|
||||
120_000,
|
||||
);
|
||||
|
||||
test.skipIf(process.platform !== "linux")(
|
||||
"a warm executable cache still builds the heap census and writes the representation report",
|
||||
async () => {
|
||||
const dir = await mkdtemp(join(tmpdir(), "scriptc-cli-diagnostic-cache-"));
|
||||
const cacheRoot = join(dir, "cache");
|
||||
const entry = join(dir, "main.ts");
|
||||
const outPath = join(dir, "program");
|
||||
try {
|
||||
await mkdir(cacheRoot, { mode: 0o700 });
|
||||
// `new` and construction through a class value both count.
|
||||
await writeFile(
|
||||
entry,
|
||||
[
|
||||
"class Item { constructor(readonly id: number) {} }",
|
||||
"const kind: typeof Item = Item;",
|
||||
"const items = [new Item(1), new kind(2), new kind(3)];",
|
||||
"console.log(items.length);",
|
||||
"",
|
||||
].join("\n"),
|
||||
);
|
||||
const build = async (env: NodeJS.ProcessEnv = {}): Promise<string> =>
|
||||
(
|
||||
await execFileAsync(
|
||||
process.execPath,
|
||||
["--import", tsxLoader, cliEntry, "build", entry, "-o", outPath],
|
||||
{
|
||||
env: { ...process.env, SCRIPTC_CACHE_DIR: cacheRoot, SCRIPTC_TIMING: "1", ...env },
|
||||
maxBuffer: 4 * 1024 * 1024,
|
||||
},
|
||||
)
|
||||
).stderr;
|
||||
expect(await build()).toContain("scriptc lowering");
|
||||
expect(await build()).not.toContain("scriptc lowering");
|
||||
|
||||
// The census build is its own cache entry.
|
||||
const census = join(dir, "census.tsv");
|
||||
await build({ SCRIPTC_HEAP_CENSUS: "1" });
|
||||
const { stdout } = await execFileAsync(outPath, [], {
|
||||
env: { ...process.env, SCRIPTC_HEAP_CENSUS: census },
|
||||
});
|
||||
expect(stdout).toBe("3\n");
|
||||
expect(await readFile(census, "utf8")).toMatch(/^new\tItem\t\t3\t[1-9]\d*$/m);
|
||||
|
||||
// A report is written even when the executable is cached.
|
||||
const report = join(dir, "repr.tsv");
|
||||
await build({ SCRIPTC_REPR_REPORT: report });
|
||||
expect(await readFile(report, "utf8")).toMatch(/^field\tItem\tid\t/m);
|
||||
} finally {
|
||||
await rm(dir, { recursive: true, force: true });
|
||||
}
|
||||
},
|
||||
600_000,
|
||||
);
|
||||
|
||||
@@ -1712,6 +1712,8 @@ export function emitClassObjDefs(
|
||||
`define internal ptr @${mangleCtorThunk(className)}(ptr %class${paramDecls ? ", " + paramDecls : ""}) ${FN_ATTRS} { ; construct thunk ${className}`,
|
||||
`entry:`,
|
||||
` %o = call ptr @${mangleClassNew(className)}()`,
|
||||
// Construction through a class value: the census counts it here.
|
||||
...(host.heapCensus?.newLines(className) ?? []),
|
||||
...(meta.def.localCaptures !== undefined
|
||||
? [
|
||||
` %class.owned = call ptr @scr_classobj_retain_v(ptr %class)`,
|
||||
|
||||
@@ -617,6 +617,14 @@ export function emitStmtBody(emitter: LlEmitter, s: IrStmt): void {
|
||||
// (The guard precedes the value here: the nullable store
|
||||
// evaluates it itself.)
|
||||
emitter.emitStoreGuard(obj.name, s);
|
||||
emitter.heapCensus?.countStore(
|
||||
(line) => B.line(line),
|
||||
() => B.tmp(),
|
||||
s.className,
|
||||
s.field,
|
||||
s.value,
|
||||
null,
|
||||
);
|
||||
emitter.emitNullableFieldStore(obj.name, s.className, s.field, nullable, s.value);
|
||||
break;
|
||||
}
|
||||
@@ -636,8 +644,17 @@ export function emitStmtBody(emitter: LlEmitter, s: IrStmt): void {
|
||||
? emitter.recordFieldInline(s.shapeId, s.field)
|
||||
: emitter.inlineUnions.classField(s.className, s.field),
|
||||
);
|
||||
if (s.kind === "fieldSet") emitter.emitStoreGuard(obj.name, s);
|
||||
else if (emitter.publishedTypes?.recordGuarded(s.shapeId) === true)
|
||||
if (s.kind === "fieldSet") {
|
||||
emitter.emitStoreGuard(obj.name, s);
|
||||
emitter.heapCensus?.countStore(
|
||||
(line) => B.line(line),
|
||||
() => B.tmp(),
|
||||
s.className,
|
||||
s.field,
|
||||
s.value,
|
||||
v.name,
|
||||
);
|
||||
} else if (emitter.publishedTypes?.recordGuarded(s.shapeId) === true)
|
||||
emitter.emitPublishedFieldGuard(obj.name, null, s.field);
|
||||
const { ptr, type, inline } =
|
||||
s.kind === "fieldSet"
|
||||
|
||||
@@ -130,6 +130,8 @@ import {
|
||||
} from "./int32-abi.js";
|
||||
import { type Int32Local } from "./int32-locals.js";
|
||||
import { findIntegerClones, NO_INTEGER_CLONES, type IntegerClones } from "./integer-clones.js";
|
||||
import { writeRepresentationReport } from "./repr-report.js";
|
||||
import { heapCensusOf, type HeapCensus } from "./heap-census.js";
|
||||
import { exactInteger, narrowInteger, widenInteger, type LlInteger } from "./integer-values.js";
|
||||
import { findInitializerBindings } from "../../ir/initializer-bindings.js";
|
||||
import { findConstantNumericTables, type ConstantNumericTable } from "../../ir/constant-tables.js";
|
||||
@@ -797,7 +799,9 @@ function emitModuleText(input: IrModule, options: LlvmTargetOptions): string[] {
|
||||
for (let excluded = options.unwindExcluded ?? new Set<string>(); ;) {
|
||||
const emitter = moduleEmitter(input, { ...options, unwindExcluded: excluded });
|
||||
try {
|
||||
return emitter.emitParts();
|
||||
const parts = emitter.emitParts();
|
||||
writeRepresentationReport(emitter);
|
||||
return parts;
|
||||
} catch (error) {
|
||||
if (!(error instanceof UnwindAbiViolation)) throw error;
|
||||
const names = new Set(excluded);
|
||||
@@ -1614,6 +1618,8 @@ export class LlEmitter {
|
||||
readonly inlineUnions: InlineUnionFields;
|
||||
/** Unions whose values are nullable arm pointers (nullable-unions.ts). */
|
||||
readonly nullableUnions: NullableUnions;
|
||||
/** The counters of a build instrumented for the heap census, or null. */
|
||||
readonly heapCensus: HeapCensus | null;
|
||||
/** The union table without nullable unions: the stack-box analyses. */
|
||||
readonly boxedUnionsById: Map<string, IrUnionDef>;
|
||||
readonly recordsById = new Map<string, IrRecordShape>();
|
||||
@@ -2006,6 +2012,8 @@ export class LlEmitter {
|
||||
options.debugSources === undefined,
|
||||
this.nodeIndex,
|
||||
);
|
||||
this.heapCensus = heapCensusOf(mod, this.nullableUnions);
|
||||
for (const decl of this.heapCensus?.declarations() ?? []) this.declare(decl);
|
||||
this.mockTimers = this.runtimeFeatures.mockTimers;
|
||||
this.debug =
|
||||
options.debugSources === undefined
|
||||
@@ -2363,6 +2371,7 @@ export class LlEmitter {
|
||||
virtualEntry: (implFn) => this.virtualEntry(implFn),
|
||||
programLifetime: this.programLifetime,
|
||||
regionAliasTag: (word) => this.memoryAliasAttachment(`region.${word}`),
|
||||
heapCensus: this.heapCensus,
|
||||
};
|
||||
// An instance of a class that extends Array is an array: a slot of its
|
||||
// element array type can hold it, and the collector visits it there as
|
||||
@@ -2646,7 +2655,8 @@ export class LlEmitter {
|
||||
emitParts(): string[] {
|
||||
const parts = emitParts(this);
|
||||
const bridged = this.unwindAbi.size > 0 ? bridgePlainUnwindCalls(parts) : parts;
|
||||
return this.cpuDispatch ? cloneForCpuDispatch(bridged) : bridged;
|
||||
const out = this.cpuDispatch ? cloneForCpuDispatch(bridged) : bridged;
|
||||
return this.heapCensus === null ? out : [...out, this.heapCensus.definitions().join("\n")];
|
||||
}
|
||||
|
||||
/** The dispose-observer gates the emitted frees and record writers test
|
||||
@@ -4686,6 +4696,7 @@ export class LlEmitter {
|
||||
`${p} = getelementptr inbounds %${classStructSym(className)}, ptr ${objName}, i64 0, i32 ${index}`,
|
||||
);
|
||||
this.markFieldPointer(p, field);
|
||||
this.heapCensus?.countAccess((line) => this.B.line(line), className, field);
|
||||
if (compressed) this.compressedFieldPtrs.add(p);
|
||||
return { ptr: p, type, inline: this.inlineUnions.classField(className, field), compressed };
|
||||
}
|
||||
|
||||
@@ -1105,6 +1105,7 @@ export function emitCallExpr(
|
||||
);
|
||||
const o = B.tmp();
|
||||
B.line(`${o} = call ptr @${host.allocSym(mangleClassNew(e.className), e)}()`);
|
||||
host.heapCensus?.countNew((line) => B.line(line), e.className);
|
||||
const out = host.own({ name: o, type: e.type });
|
||||
emitConstruction(host, e, o, ctor);
|
||||
return out;
|
||||
|
||||
@@ -0,0 +1,241 @@
|
||||
/* The heap census. A program built with SCRIPTC_HEAP_CENSUS set counts, for
|
||||
* ranking representation work:
|
||||
* - the instances of each class that `new` or a class value constructs,
|
||||
* with the class's size;
|
||||
* - for each array-typed class field, its stores (with the stored array's
|
||||
* length in buckets and the total of those lengths), the stores whose
|
||||
* value is an array literal written at the store, and its accesses;
|
||||
* - for each record-typed class field, its stores and accesses.
|
||||
* Accesses count every field address the backend computes for the field,
|
||||
* so a store is also an access. At exit the program writes one
|
||||
* tab-separated line per counter to the file the run's SCRIPTC_HEAP_CENSUS
|
||||
* names, or to stderr. Linux only: the runtime calls the writer through a
|
||||
* weak reference, as it does the profile writer. Counters are module
|
||||
* globals updated with monotonic atomic adds, so they include every
|
||||
* thread's work. Ordinary builds emit none of it. */
|
||||
import type { IrClassDef, IrExpr, IrModule, IrType } from "../../ir/ir.js";
|
||||
import { classStructSym } from "./classes.js";
|
||||
import { llvmBytes, wtf8Bytes } from "../literals.js";
|
||||
import type { NullableUnions } from "./nullable-unions.js";
|
||||
|
||||
/** Whether this build counts the heap census: SCRIPTC_HEAP_CENSUS is set
|
||||
* and nonempty. The executable cache keys on it. */
|
||||
export function heapCensusRequested(env: NodeJS.ProcessEnv = process.env): boolean {
|
||||
const flag = env["SCRIPTC_HEAP_CENSUS"];
|
||||
return flag !== undefined && flag !== "";
|
||||
}
|
||||
|
||||
/** The census of the module being built, or null when it is off. */
|
||||
export function heapCensusOf(mod: IrModule, unions: NullableUnions): HeapCensus | null {
|
||||
if (!heapCensusRequested() || mod.lib !== undefined) return null;
|
||||
return new HeapCensus(mod, unions);
|
||||
}
|
||||
|
||||
/** A NUL-terminated i8 array constant of `text`. */
|
||||
function cString(text: string): string {
|
||||
const bytes = wtf8Bytes(text);
|
||||
return `[${bytes.length + 1} x i8] c"${llvmBytes(bytes)}"`;
|
||||
}
|
||||
|
||||
const LENGTH_BUCKETS = ["len0", "len1", "len2", "len3-4", "len5-8", "len9-16", "len17+"];
|
||||
/** Upper bounds of LENGTH_BUCKETS but the last. */
|
||||
const BUCKET_LIMITS = [0, 1, 2, 4, 8, 16];
|
||||
|
||||
interface FieldCounters {
|
||||
/** First counter: stores, accesses, then (arrays) the literal stores,
|
||||
* the length total and the buckets. */
|
||||
base: number;
|
||||
array: boolean;
|
||||
/** The stored value is a pointer that is NULL for the union's unit arm. */
|
||||
nullable: boolean;
|
||||
}
|
||||
|
||||
export class HeapCensus {
|
||||
private readonly names: string[] = [];
|
||||
private readonly sizes: (string | null)[] = [];
|
||||
private readonly instances = new Map<string, number>();
|
||||
private readonly fields = new Map<string, FieldCounters>();
|
||||
|
||||
constructor(mod: IrModule, unions: NullableUnions) {
|
||||
for (const cls of mod.classes ?? []) {
|
||||
this.instances.set(cls.name, this.add(`new\t${cls.name}\t`, classStructSym(cls.name)));
|
||||
for (const field of cls.fields) this.addField(cls, field.name, field.type, unions);
|
||||
}
|
||||
}
|
||||
|
||||
/** Counts one instance of `className`. */
|
||||
countNew(line: (text: string) => void, className: string): void {
|
||||
for (const text of this.newLines(className)) line(text);
|
||||
}
|
||||
|
||||
/** The lines that count one instance of `className`: a `new`, or a
|
||||
* construct thunk for class values (`newValue`). */
|
||||
newLines(className: string): string[] {
|
||||
const index = this.instances.get(className);
|
||||
return index === undefined ? [] : [this.increment(index, "1")];
|
||||
}
|
||||
|
||||
/** Counts an access of a field; true when the field is counted. */
|
||||
countAccess(line: (text: string) => void, className: string, field: string): void {
|
||||
const counters = this.fields.get(`${className}\0${field}`);
|
||||
if (counters !== undefined) line(this.increment(counters.base + 1, "1"));
|
||||
}
|
||||
|
||||
/** Counts a store of `value` (an LLVM pointer, computed from `source`)
|
||||
* into a field: its length when the field holds arrays. `tmp` names fresh
|
||||
* temporaries. */
|
||||
countStore(
|
||||
line: (text: string) => void,
|
||||
tmp: () => string,
|
||||
className: string,
|
||||
field: string,
|
||||
source: IrExpr,
|
||||
value: string | null,
|
||||
): void {
|
||||
const counters = this.fields.get(`${className}\0${field}`);
|
||||
if (counters === undefined) return;
|
||||
line(this.increment(counters.base, "1"));
|
||||
if (!counters.array) return;
|
||||
const literal = source.kind === "unionWrap" ? source.value : source;
|
||||
if (literal.kind === "arrayLit") line(this.increment(counters.base + 2, "1"));
|
||||
if (value === null) return;
|
||||
// A NULL pointer (the unit arm) counts as length 0: its length is read
|
||||
// through a select that points NULL at a zeroed array header.
|
||||
let array = value;
|
||||
if (counters.nullable) {
|
||||
const isNull = tmp();
|
||||
array = tmp();
|
||||
line(`${isNull} = icmp eq ptr ${value}, null`);
|
||||
line(`${array} = select i1 ${isNull}, ptr @sc_census_zero_arr, ptr ${value}`);
|
||||
}
|
||||
const lenPtr = tmp(),
|
||||
len32 = tmp(),
|
||||
len = tmp();
|
||||
line(`${lenPtr} = getelementptr inbounds %ScrArr, ptr ${array}, i32 0, i32 1`);
|
||||
line(`${len32} = load i32, ptr ${lenPtr}`);
|
||||
line(`${len} = zext i32 ${len32} to i64`);
|
||||
line(this.increment(counters.base + 3, len));
|
||||
// The bucket: the number of limits below the length.
|
||||
let bucket = "0";
|
||||
for (const limit of BUCKET_LIMITS) {
|
||||
const above = tmp(),
|
||||
step = tmp(),
|
||||
next = tmp();
|
||||
line(`${above} = icmp ugt i64 ${len}, ${limit}`);
|
||||
line(`${step} = zext i1 ${above} to i64`);
|
||||
line(`${next} = add i64 ${bucket}, ${step}`);
|
||||
bucket = next;
|
||||
}
|
||||
const slot = tmp();
|
||||
line(`${slot} = add i64 ${bucket}, ${counters.base + 4}`);
|
||||
const ptr = tmp();
|
||||
line(
|
||||
`${ptr} = getelementptr inbounds [${this.names.length} x i64], ptr @sc_census, i64 0, i64 ${slot}`,
|
||||
);
|
||||
line(`atomicrmw add ptr ${ptr}, i64 1 monotonic`);
|
||||
}
|
||||
|
||||
/** The module definitions: the counters, their names, the zero array
|
||||
* header NULL stores read their length from, and the writer the runtime
|
||||
* calls at exit (scr_census_write). */
|
||||
definitions(): string[] {
|
||||
const n = this.names.length;
|
||||
const out = [`@sc_census = internal global [${n} x i64] zeroinitializer`];
|
||||
// A zeroed %ScrArr: length 0.
|
||||
out.push(`@sc_census_zero_arr = internal constant %ScrArr zeroinitializer`);
|
||||
const names = this.names.map((name, i) => {
|
||||
const bytes = cString(name);
|
||||
out.push(`@sc_census_name.${i} = private unnamed_addr constant ${bytes}`);
|
||||
return `ptr @sc_census_name.${i}`;
|
||||
});
|
||||
out.push(`@sc_census_names = internal constant [${n} x ptr] [${names.join(", ")}]`);
|
||||
const sizes = this.sizes.map((sym) =>
|
||||
sym === null ? "i64 0" : `i64 ptrtoint (ptr getelementptr (%${sym}, ptr null, i32 1) to i64)`,
|
||||
);
|
||||
out.push(`@sc_census_sizes = internal constant [${n} x i64] [${sizes.join(", ")}]`);
|
||||
const env = cString("SCRIPTC_HEAP_CENSUS");
|
||||
out.push(`@sc_census_env = private unnamed_addr constant ${env}`);
|
||||
out.push(`@sc_census_mode = private unnamed_addr constant ${cString("w")}`);
|
||||
out.push(`@sc_census_fmt = private unnamed_addr constant ${cString("%s\t%llu\t%llu\n")}`);
|
||||
out.push(
|
||||
`define void @scr_census_write() {`,
|
||||
`entry:`,
|
||||
` %path = call ptr @getenv(ptr @sc_census_env)`,
|
||||
` %named = icmp ne ptr %path, null`,
|
||||
` br i1 %named, label %open, label %err`,
|
||||
`open:`,
|
||||
` %file = call ptr @fopen(ptr %path, ptr @sc_census_mode)`,
|
||||
` %opened = icmp ne ptr %file, null`,
|
||||
` br i1 %opened, label %write, label %err`,
|
||||
`err:`,
|
||||
` %stderr = load ptr, ptr @stderr`,
|
||||
` br label %write`,
|
||||
`write:`,
|
||||
` %out = phi ptr [ %file, %open ], [ %stderr, %err ]`,
|
||||
` br label %loop`,
|
||||
`loop:`,
|
||||
` %i = phi i64 [ 0, %write ], [ %next, %loop ]`,
|
||||
` %namep = getelementptr inbounds [${n} x ptr], ptr @sc_census_names, i64 0, i64 %i`,
|
||||
` %name = load ptr, ptr %namep`,
|
||||
` %countp = getelementptr inbounds [${n} x i64], ptr @sc_census, i64 0, i64 %i`,
|
||||
` %count = load atomic i64, ptr %countp monotonic, align 8`,
|
||||
` %sizep = getelementptr inbounds [${n} x i64], ptr @sc_census_sizes, i64 0, i64 %i`,
|
||||
` %size = load i64, ptr %sizep`,
|
||||
` %bytes = mul i64 %count, %size`,
|
||||
` call i32 (ptr, ptr, ...) @fprintf(ptr %out, ptr @sc_census_fmt, ptr %name, i64 %count, i64 %bytes)`,
|
||||
` %next = add i64 %i, 1`,
|
||||
` %done = icmp eq i64 %next, ${n}`,
|
||||
` br i1 %done, label %end, label %loop`,
|
||||
`end:`,
|
||||
` call i32 @fflush(ptr %out)`,
|
||||
` ret void`,
|
||||
`}`,
|
||||
);
|
||||
return out;
|
||||
}
|
||||
|
||||
/** External declarations the writer uses. */
|
||||
declarations(): string[] {
|
||||
return [
|
||||
"declare ptr @getenv(ptr)",
|
||||
"declare ptr @fopen(ptr, ptr)",
|
||||
"declare i32 @fprintf(ptr, ptr, ...)",
|
||||
"declare i32 @fflush(ptr)",
|
||||
"@stderr = external global ptr",
|
||||
];
|
||||
}
|
||||
|
||||
private add(name: string, sizeSym: string | null = null): number {
|
||||
this.names.push(name);
|
||||
this.sizes.push(sizeSym);
|
||||
return this.names.length - 1;
|
||||
}
|
||||
|
||||
private addField(cls: IrClassDef, field: string, type: IrType, unions: NullableUnions): void {
|
||||
let array = type.kind === "array";
|
||||
let nullable = false;
|
||||
let record = type.kind === "record";
|
||||
if (type.kind === "union") {
|
||||
const single = unions.single(type.unionId);
|
||||
if (single !== null && single.nullTag < 0) {
|
||||
array = single.arm.kind === "array";
|
||||
record = single.arm.kind === "record";
|
||||
nullable = true;
|
||||
}
|
||||
}
|
||||
if (!array && !record) return;
|
||||
const key = `${cls.name}\t${field}`;
|
||||
const base = this.add(`store\t${key}`);
|
||||
this.add(`access\t${key}`);
|
||||
if (array) {
|
||||
this.add(`literal\t${key}`);
|
||||
this.add(`elements\t${key}`);
|
||||
for (const bucket of LENGTH_BUCKETS) this.add(`${bucket}\t${key}`);
|
||||
}
|
||||
this.fields.set(`${cls.name}\0${field}`, { base, array, nullable });
|
||||
}
|
||||
|
||||
private increment(index: number, by: string): string {
|
||||
return `atomicrmw add ptr getelementptr inbounds ([${this.names.length} x i64], ptr @sc_census, i64 0, i64 ${index}), i64 ${by} monotonic`;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { afterEach, expect, test } from "vitest";
|
||||
import { compile, deserializeModule } from "../../index.js";
|
||||
import { emitLlvmModule } from "./emitter.js";
|
||||
import { REPR_REPORT_HEADER } from "./repr-report.js";
|
||||
|
||||
const saved = process.env["SCRIPTC_REPR_REPORT"];
|
||||
afterEach(() => {
|
||||
if (saved === undefined) delete process.env["SCRIPTC_REPR_REPORT"];
|
||||
else process.env["SCRIPTC_REPR_REPORT"] = saved;
|
||||
});
|
||||
|
||||
test("the representation report names each slot's storage and why a number stays a double", async () => {
|
||||
const dir = await mkdtemp(join(tmpdir(), "scriptc-repr-report-"));
|
||||
try {
|
||||
const entry = join(dir, "main.ts"),
|
||||
outPath = join(dir, "main.ir.json"),
|
||||
report = join(dir, "repr.tsv");
|
||||
await writeFile(
|
||||
entry,
|
||||
`
|
||||
class Box {
|
||||
id: number;
|
||||
hits: number;
|
||||
items: number[];
|
||||
constructor(id: number, items: number[]) { this.id = id | 0; this.hits = 0; this.items = items; }
|
||||
}
|
||||
function scale(value: number, by: number = 2): number { return (value * by) | 0; }
|
||||
function half(value: number): number { return value / 2; }
|
||||
const boxes: Box[] = [];
|
||||
for (let i = 0; i < 4; i++) boxes.push(new Box(i, [i, i + 1]));
|
||||
for (const b of boxes) b.hits++;
|
||||
console.log(scale(boxes[1]!.id), scale(3, 4), half(boxes.length), boxes[2]!.items.length, boxes[3]!.hits);
|
||||
`,
|
||||
);
|
||||
const result = await compile(entry, { outDir: dir, outPath, outputKind: "ir" });
|
||||
if (!result.ok) throw new Error(result.diagnostics.map((d) => d.message).join("\n"));
|
||||
const mod = deserializeModule(await readFile(outPath, "utf8"));
|
||||
process.env["SCRIPTC_REPR_REPORT"] = report;
|
||||
emitLlvmModule(mod);
|
||||
const lines = (await readFile(report, "utf8")).trimEnd().split("\n");
|
||||
expect(lines[0]).toBe(REPR_REPORT_HEADER);
|
||||
const rows = lines.slice(1).map((line) => line.split("\t"));
|
||||
const row = (kind: string, owner: string, name: string) =>
|
||||
rows.find((r) => r[0] === kind && r[1] === owner && r[2] === name);
|
||||
expect(row("field", "Box", "id")?.slice(3, 5)).toEqual(["f64", "i32"]);
|
||||
// `++` on a field keeps it a double, and the report says so.
|
||||
expect(row("field", "Box", "hits")?.[4]).toBe("double");
|
||||
expect(row("field", "Box", "hits")?.[5]).not.toBe("");
|
||||
expect(row("field", "Box", "items")?.slice(3, 5)).toEqual(["array<f64>", "ptr"]);
|
||||
expect(row("param", "scale", "value")?.[4]).toBe("i32");
|
||||
// A defaulted parameter is a `number | undefined` box; the variant for
|
||||
// calls that pass a number takes an i32.
|
||||
expect(row("param", "scale", "by")?.slice(3, 5)).toEqual(["f64 | undefined", "box"]);
|
||||
expect(row("param", "scale.nativeNumber", "by")?.[4]).toBe("i32");
|
||||
// An array length may exceed the int32 range.
|
||||
expect(row("param", "half", "value")?.[4]).toBe("double");
|
||||
expect(row("param", "half", "value")?.[5]).toMatch(/^argument in .* is not int32$/);
|
||||
expect(row("return", "half", "")?.slice(4)).toEqual(["double", "return is not int32"]);
|
||||
} finally {
|
||||
await rm(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,141 @@
|
||||
/* The representation report (SCRIPTC_REPR_REPORT=<file>): after a module is
|
||||
* emitted, one tab-separated line per class field, function parameter,
|
||||
* function result and union, naming the storage the backend chose for it
|
||||
* and, for numbers kept as doubles, the reason the int32 analysis recorded
|
||||
* (int32-slots.ts Int32Slots.reasons). It is a diagnostic for ranking
|
||||
* representation work; nothing reads it back. */
|
||||
import { writeFileSync } from "node:fs";
|
||||
import { typeKey, type IrFunction, type IrModule, type IrType } from "../../ir/ir.js";
|
||||
import type { LlEmitter } from "./emitter.js";
|
||||
import { classFieldTypes } from "./classes.js";
|
||||
import { INLINE_UNION_FIELD } from "./inline-unions.js";
|
||||
|
||||
export const REPR_REPORT_HEADER = "kind\towner\tname\ttype\tstorage\treason";
|
||||
|
||||
/** The file SCRIPTC_REPR_REPORT names, or null. A build that writes the
|
||||
* report skips the executable cache, whose hit would emit no module. */
|
||||
export function representationReportPath(env: NodeJS.ProcessEnv = process.env): string | null {
|
||||
const path = env["SCRIPTC_REPR_REPORT"];
|
||||
return path === undefined || path === "" ? null : path;
|
||||
}
|
||||
|
||||
/** Writes the report of `emitter`'s module when SCRIPTC_REPR_REPORT names a
|
||||
* file. */
|
||||
export function writeRepresentationReport(emitter: LlEmitter): void {
|
||||
const path = representationReportPath();
|
||||
if (path !== null) writeFileSync(path, representationReport(emitter));
|
||||
}
|
||||
|
||||
export function representationReport(emitter: LlEmitter): string {
|
||||
const mod = emitter.mod;
|
||||
const slots = emitter.int32Slots;
|
||||
const lines = [REPR_REPORT_HEADER];
|
||||
const row = (...cells: string[]): void => {
|
||||
lines.push(cells.map((cell) => cell.replace(/[\t\n]/g, " ")).join("\t"));
|
||||
};
|
||||
const typeLabel = typeLabeler(mod);
|
||||
// A union value that is not a plain box: a nullable or shape pointer.
|
||||
const unionStorage = (t: IrType): string | null => {
|
||||
if (t.kind !== "union") return null;
|
||||
return emitter.nullableUnions.get(t.unionId)?.repr ?? "box";
|
||||
};
|
||||
|
||||
for (const cls of mod.classes ?? []) {
|
||||
const meta = emitter.classMeta.get(cls.name);
|
||||
const types = classFieldTypes(emitter.shapeHost, cls, slots);
|
||||
// Inherited fields come first in def.fields: report each field once,
|
||||
// on the class that declares it.
|
||||
const inherited = meta?.base?.def.fields.length ?? 0;
|
||||
cls.fields.forEach((field, i) => {
|
||||
if (i < inherited) return;
|
||||
let storage = types[i]!;
|
||||
if (storage === INLINE_UNION_FIELD) storage = "inline union";
|
||||
else if (meta?.compressed[i] === true) storage = "ref32";
|
||||
else if (storage === "ptr") storage = unionStorage(field.type) ?? "ptr";
|
||||
let reason = "";
|
||||
if (field.type.kind === "f64" && storage === "double") {
|
||||
const family = slots.family(cls.name, field.name);
|
||||
reason =
|
||||
family === null
|
||||
? "no field family"
|
||||
: (slots.reasons.get(`field:${family}`) ?? "not a candidate");
|
||||
}
|
||||
row("field", cls.name, field.name, typeLabel(field.type), storage, reason);
|
||||
});
|
||||
}
|
||||
|
||||
for (const fn of mod.functions) {
|
||||
const abi = emitter.int32Abis.get(fn.name);
|
||||
fn.params.forEach((param, i) => {
|
||||
let storage: string;
|
||||
let reason = "";
|
||||
if (param.type.kind === "f64") {
|
||||
storage = abi?.params[i] === true ? "i32" : "double";
|
||||
if (storage === "double") reason = parameterReason(emitter, fn, param.localId);
|
||||
} else storage = unionStorage(param.type) ?? scalarStorage(param.type);
|
||||
row("param", fn.name, param.name, typeLabel(param.type), storage, reason);
|
||||
});
|
||||
const ret = fn.returnType;
|
||||
if (ret.kind === "void") continue;
|
||||
let storage: string;
|
||||
let reason = "";
|
||||
if (ret.kind === "f64") {
|
||||
storage = abi?.result === true ? "i32" : "double";
|
||||
if (storage === "double")
|
||||
reason = slots.reasons.get(`ret:${fn.name}`) ?? notCandidate(emitter, fn);
|
||||
} else if (ret.kind === "union" && abi?.union === true) storage = "register pair";
|
||||
else storage = unionStorage(ret) ?? scalarStorage(ret);
|
||||
row("return", fn.name, "", typeLabel(ret), storage, reason);
|
||||
}
|
||||
|
||||
for (const def of mod.unions ?? [])
|
||||
row(
|
||||
"union",
|
||||
def.id,
|
||||
"",
|
||||
typeLabel({ kind: "union", unionId: def.id }),
|
||||
unionStorage({ kind: "union", unionId: def.id })!,
|
||||
"",
|
||||
);
|
||||
return lines.join("\n") + "\n";
|
||||
}
|
||||
|
||||
function scalarStorage(t: IrType): string {
|
||||
switch (t.kind) {
|
||||
case "f64":
|
||||
return "double";
|
||||
case "bool":
|
||||
return "i1";
|
||||
default:
|
||||
return "ptr";
|
||||
}
|
||||
}
|
||||
|
||||
function parameterReason(emitter: LlEmitter, fn: IrFunction, localId: string): string {
|
||||
const slots = emitter.int32Slots;
|
||||
const recorded = slots.reasons.get(`param:${fn.name}\0${localId}`);
|
||||
if (recorded !== undefined) return recorded;
|
||||
// Proven, but the function has no i32 entry (findInt32Abis).
|
||||
if (slots.hasParam(fn.name, localId)) return "no i32 entry";
|
||||
return notCandidate(emitter, fn);
|
||||
}
|
||||
|
||||
/** Why a function's numbers were never int32 candidates: the analysis only
|
||||
* proves functions that direct calls alone reach. */
|
||||
function notCandidate(emitter: LlEmitter, fn: IrFunction): string {
|
||||
if (fn.async) return "async function";
|
||||
if (fn.generator) return "generator";
|
||||
if (fn.captures !== undefined) return "closure";
|
||||
if (!emitter.int32Slots.called.has(fn.name)) return "no direct call";
|
||||
return "escapes or is dispatched";
|
||||
}
|
||||
|
||||
/** Type labels with union arms spelled out. */
|
||||
function typeLabeler(mod: IrModule): (t: IrType) => string {
|
||||
const arms = new Map((mod.unions ?? []).map((def) => [def.id, def.arms]));
|
||||
return (t) => {
|
||||
if (t.kind !== "union") return typeKey(t);
|
||||
const list = arms.get(t.unionId);
|
||||
return list === undefined ? typeKey(t) : list.map(typeKey).join(" | ");
|
||||
};
|
||||
}
|
||||
@@ -2,6 +2,7 @@
|
||||
* Containers store erased adapters; shape helpers retain, release and trace
|
||||
* their fields. Every allocation has a size_t reference count at offset zero
|
||||
* and participates in the runtime allocation audit. */
|
||||
import type { HeapCensus } from "./heap-census.js";
|
||||
import type { IrClassDef, IrModule, IrRecordShape, IrType } from "../../ir/ir.js";
|
||||
import {
|
||||
isIdentityCollectionKey,
|
||||
@@ -78,6 +79,8 @@ const CYC_MAX_TYPES = 32768;
|
||||
/** What the tables need from the emitter: the extern-declaration ledger
|
||||
* and the module-wide cycle/shape indexes. */
|
||||
export interface ShapeHost {
|
||||
/** The counters of a build instrumented for the heap census (heap-census.ts). */
|
||||
readonly heapCensus?: HeapCensus | null;
|
||||
/** Chunk emitters only (emit-journal.ts): EmitJournal's allocated, built,
|
||||
* opened, used and entry. */
|
||||
journalAllocated?: ((name: string, op: string, args: readonly unknown[]) => void) | undefined;
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { afterEach, expect, test, vi } from "vitest";
|
||||
import {
|
||||
NATIVE_CODEGEN_LLVM_VERSION,
|
||||
NATIVE_CODEGEN_PROTOCOL_VERSION,
|
||||
} from "./native-codegen-core.js";
|
||||
import { verifyNativeHelper } from "./native-tools.js";
|
||||
import { LINUX_X64_GNU_TARGET } from "./targets.js";
|
||||
|
||||
vi.mock("node:child_process", async (original) => ({
|
||||
...(await original<typeof import("node:child_process")>()),
|
||||
execFileSync: vi.fn(),
|
||||
}));
|
||||
|
||||
const directories: string[] = [];
|
||||
afterEach(() => {
|
||||
vi.resetAllMocks();
|
||||
for (const path of directories.splice(0)) rmSync(path, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
/** A helper package directory and the options that verify it. */
|
||||
function helperOptions() {
|
||||
const packageRoot = mkdtempSync(join(tmpdir(), "scriptc-helper-"));
|
||||
directories.push(packageRoot);
|
||||
const target = LINUX_X64_GNU_TARGET;
|
||||
const helper = target.helper;
|
||||
writeFileSync(
|
||||
join(packageRoot, "package.json"),
|
||||
JSON.stringify({ name: helper.packageName, version: "1.2.3" }),
|
||||
);
|
||||
vi.mocked(execFileSync).mockReturnValue(
|
||||
JSON.stringify({
|
||||
ok: true,
|
||||
protocol_version: NATIVE_CODEGEN_PROTOCOL_VERSION,
|
||||
scriptc_package_version: "1.2.3",
|
||||
llvm_version: NATIVE_CODEGEN_LLVM_VERSION,
|
||||
default_target: helper.defaultTarget,
|
||||
data_layout: helper.defaultDataLayout,
|
||||
targets: [target.llvmBackend],
|
||||
supported_targets: [target.llvmTriple],
|
||||
host_triple: target.llvmTriple,
|
||||
}),
|
||||
);
|
||||
return {
|
||||
executable: join(packageRoot, "helper"),
|
||||
packageRoot,
|
||||
compilerVersion: "1.2.3",
|
||||
target,
|
||||
helper,
|
||||
};
|
||||
}
|
||||
|
||||
test("a helper the verification record knows is not asked for its version again", () => {
|
||||
const options = helperOptions();
|
||||
let known = false;
|
||||
const remember = vi.fn(() => {
|
||||
known = true;
|
||||
});
|
||||
const verified = { known: () => known, remember };
|
||||
verifyNativeHelper({ ...options, verified });
|
||||
expect(execFileSync).toHaveBeenCalledTimes(1);
|
||||
expect(remember).toHaveBeenCalledTimes(1);
|
||||
verifyNativeHelper({ ...options, verified });
|
||||
expect(execFileSync).toHaveBeenCalledTimes(1);
|
||||
expect(remember).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
test("a helper whose version answer is rejected is not recorded", () => {
|
||||
const options = helperOptions();
|
||||
vi.mocked(execFileSync).mockReturnValue(JSON.stringify({ ok: false }));
|
||||
const remember = vi.fn();
|
||||
expect(() =>
|
||||
verifyNativeHelper({ ...options, verified: { known: () => false, remember } }),
|
||||
).toThrow(/incompatible/);
|
||||
expect(remember).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
test("the package identity is checked even for a helper the record knows", () => {
|
||||
const options = helperOptions();
|
||||
const remember = vi.fn();
|
||||
expect(() =>
|
||||
verifyNativeHelper({
|
||||
...options,
|
||||
compilerVersion: "9.9.9",
|
||||
verified: { known: () => true, remember },
|
||||
}),
|
||||
).toThrow(/package mismatch/);
|
||||
expect(execFileSync).not.toHaveBeenCalled();
|
||||
});
|
||||
@@ -56,6 +56,7 @@ export function emitNativeObject(options: {
|
||||
partitionCache?: string;
|
||||
/** Profile-guided optimization: instrumentation or an indexed profile. */
|
||||
profile?: NativeProfileOptions;
|
||||
verified?: NativeHelperVerification;
|
||||
}): { outputs: string[]; profileMatch: NativeProfileMatch | null } {
|
||||
verifyNativeHelper(options);
|
||||
const outputs = nativePartitionPaths(options.outputPath, options.partitions ?? 1);
|
||||
@@ -111,13 +112,24 @@ export function mergeNativeProfiles(options: {
|
||||
]);
|
||||
}
|
||||
|
||||
/** Verify installed identity even when a caller can reuse a cached object. */
|
||||
/** Helpers that answered `version` as expected: `known` is true when the
|
||||
* helper file, as it is now, already did, and `remember` records that this
|
||||
* one just did. */
|
||||
export interface NativeHelperVerification {
|
||||
known(): boolean;
|
||||
remember(): void;
|
||||
}
|
||||
|
||||
/** Verify installed identity even when a caller can reuse a cached object.
|
||||
* With `verified`, the version probe runs only for a helper it does not
|
||||
* know yet. */
|
||||
export function verifyNativeHelper(options: {
|
||||
executable: string;
|
||||
packageRoot: string;
|
||||
compilerVersion: string;
|
||||
target: NativeTargetSpec;
|
||||
helper: NativeHelperSpec;
|
||||
verified?: NativeHelperVerification;
|
||||
}): void {
|
||||
const identity = JSON.parse(readFileSync(options.packageRoot + "/package.json", "utf8")) as {
|
||||
name: string;
|
||||
@@ -131,8 +143,10 @@ export function verifyNativeHelper(options: {
|
||||
`LLVM helper package mismatch: expected ${options.helper.packageName}@${options.compilerVersion}`,
|
||||
);
|
||||
}
|
||||
if (options.verified?.known() === true) return;
|
||||
const version = JSON.parse(
|
||||
runNativeTool(options.executable, ["version", "--format=json"]),
|
||||
) as Record<string, unknown>;
|
||||
validateNativeCodegenVersion(version, options.target, options.helper, options.compilerVersion);
|
||||
options.verified?.remember();
|
||||
}
|
||||
|
||||
@@ -178,6 +178,8 @@ export interface EarlyExecutableCacheOptions {
|
||||
checker?: string;
|
||||
/** Omitted unless SCRIPTC_YOUNG_NURSERY=1 (backend/young-nursery.ts). */
|
||||
youngNursery?: true;
|
||||
/** Omitted unless SCRIPTC_HEAP_CENSUS is set (backend/llvm/heap-census.ts). */
|
||||
heapCensus?: true;
|
||||
npmStatic: readonly string[] | "auto" | null;
|
||||
/** Omitted unless shape inference's overlay compiles the packages
|
||||
* (frontend/shape-inference). */
|
||||
@@ -334,6 +336,7 @@ function cacheKey(options: EarlyExecutableCacheOptions): string {
|
||||
...(options.functionLayout !== undefined ? [`layout:${options.functionLayout}`] : []),
|
||||
...(options.checker !== undefined ? [`checker:${options.checker}`] : []),
|
||||
...(options.youngNursery ? ["young-nursery"] : []),
|
||||
...(options.heapCensus ? ["heap-census"] : []),
|
||||
options.npmStatic === null
|
||||
? "<npm-static-off>"
|
||||
: options.npmStatic === "auto"
|
||||
@@ -380,6 +383,7 @@ function routeKey(options: Omit<EarlyExecutableRouteOptions, "nativeEnvironment"
|
||||
if (options.functionLayout !== undefined) hash.update(`layout:${options.functionLayout}\0`);
|
||||
if (options.checker !== undefined) hash.update(`checker:${options.checker}\0`);
|
||||
if (options.youngNursery) hash.update("young-nursery\0");
|
||||
if (options.heapCensus) hash.update("heap-census\0");
|
||||
hash
|
||||
.update(
|
||||
options.npmStatic === null
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import * as fs from "node:fs";
|
||||
import { chmod, mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
@@ -301,6 +302,36 @@ test("a settled file is checked by its identity, and any write invalidates it",
|
||||
expect(frontendInputsStillMatch(snapshot)).toBe(false);
|
||||
});
|
||||
|
||||
test("a settled file an earlier run read keeps that run's digest until it is written", async () => {
|
||||
const dir = await mkdtemp(join(tmpdir(), "scriptc-inputs-"));
|
||||
scratch.push(dir);
|
||||
const source = join(dir, "entry.ts");
|
||||
await writeFile(source, "export const answer = 1;\n");
|
||||
const settledRead = (tracker: FrontendInputTracker, later: number) => {
|
||||
vi.useFakeTimers({ toFake: ["Date"] });
|
||||
vi.setSystemTime(Date.now() + later);
|
||||
try {
|
||||
tracker.runSynchronous(() => trackedReadFile(source));
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
const probe = tracker.snapshot().probes.find((entry) => entry.op === "file");
|
||||
return probe?.op === "file" ? probe : undefined;
|
||||
};
|
||||
const first = settledRead(new FrontendInputTracker(), 5000)!;
|
||||
expect(first.identity).toBeDefined();
|
||||
// A digest only the previous snapshot holds shows it was not computed again.
|
||||
const marker = "f".repeat(64);
|
||||
const previous = { version: 1 as const, stable: true, probes: [{ ...first, digest: marker }] };
|
||||
expect(settledRead(new FrontendInputTracker(previous), 5000)?.digest).toBe(marker);
|
||||
await writeFile(source, "export const answer = 2;\n");
|
||||
const rewritten = settledRead(new FrontendInputTracker(previous), 10000);
|
||||
expect(rewritten?.identity).not.toBe(first.identity);
|
||||
expect(rewritten?.digest).toBe(
|
||||
createHash("sha256").update("export const answer = 2;\n").digest("hex"),
|
||||
);
|
||||
});
|
||||
|
||||
test("a file changed shortly before the read is hashed on every check", async () => {
|
||||
const dir = await mkdtemp(join(tmpdir(), "scriptc-inputs-"));
|
||||
scratch.push(dir);
|
||||
|
||||
@@ -127,6 +127,23 @@ let synchronousTracker: FrontendInputTracker | undefined;
|
||||
export class FrontendInputTracker {
|
||||
private readonly probes = new Map<string, FrontendInputProbe>();
|
||||
private stable = true;
|
||||
/** The digests `previous` recorded for files read with a settled identity. */
|
||||
private readonly known = new Map<string, { identity: string; digest: string }>();
|
||||
|
||||
/** `previous` is the snapshot of an earlier run whose cache entry is being
|
||||
* rebuilt: a file whose settled identity has not changed since then has the
|
||||
* content it hashed, so its digest is not computed again. */
|
||||
constructor(previous?: FrontendInputSnapshot) {
|
||||
for (const probe of previous?.probes ?? [])
|
||||
if (probe.op === "file" && probe.identity !== undefined)
|
||||
this.known.set(probe.path, { identity: probe.identity, digest: probe.digest });
|
||||
}
|
||||
|
||||
/** The digest an earlier run recorded for `path` with this identity. */
|
||||
knownDigest(path: string, identity: string): string | undefined {
|
||||
const known = this.known.get(path);
|
||||
return known?.identity === identity ? known.digest : undefined;
|
||||
}
|
||||
|
||||
run<T>(fn: () => T): T {
|
||||
const parent = activeTracker.getStore();
|
||||
@@ -254,7 +271,11 @@ export function trackedReadFile(path: string): string | null {
|
||||
if (virtual !== undefined) return virtual;
|
||||
try {
|
||||
const { text, identity } = readWithIdentity(path);
|
||||
const digest = frontendSourceDigest(text);
|
||||
const known =
|
||||
identity === null
|
||||
? undefined
|
||||
: (synchronousTracker ?? activeTracker.getStore())?.knownDigest(path, identity);
|
||||
const digest = known ?? frontendSourceDigest(text);
|
||||
record(
|
||||
identity === null ? { op: "file", path, digest } : { op: "file", path, digest, identity },
|
||||
);
|
||||
|
||||
@@ -165,6 +165,8 @@ import { clearResolveCaches } from "./frontend/resolve.js";
|
||||
import { loadFfiProfile, type FfiProfile } from "./ffi/ffi-manifest.js";
|
||||
import { executableLinkFeatures } from "./backend/executable-features.js";
|
||||
import { youngNurseryRequested } from "./backend/young-nursery.js";
|
||||
import { heapCensusRequested } from "./backend/llvm/heap-census.js";
|
||||
import { representationReportPath } from "./backend/llvm/repr-report.js";
|
||||
import {
|
||||
type FrontendInputSnapshot,
|
||||
FrontendInputTracker,
|
||||
@@ -1742,6 +1744,7 @@ async function compileTracked(
|
||||
...(functionLayoutDigest === undefined ? {} : { functionLayout: functionLayoutDigest }),
|
||||
...checkerField(),
|
||||
...(youngNurseryRequested() ? { youngNursery: true as const } : {}),
|
||||
...(heapCensusRequested() ? { heapCensus: true as const } : {}),
|
||||
npmStatic: opts.npmStatic ?? null,
|
||||
...(shapeOverlayEnabled() ? { shapeOverlay: true as const } : {}),
|
||||
ffiProfile:
|
||||
@@ -1785,8 +1788,9 @@ async function compileTracked(
|
||||
implementationDependencies: implementation.dependencies,
|
||||
};
|
||||
}
|
||||
// A cache hit emits no module, so it would write no representation report.
|
||||
const cachedHit =
|
||||
earlyCacheOptions === null
|
||||
earlyCacheOptions === null || representationReportPath() !== null
|
||||
? null
|
||||
: await readEarlyExecutableCache(cacheRoot, earlyCacheOptions);
|
||||
// A --libnode=beside build that calls into Node places its island modules
|
||||
|
||||
@@ -170,6 +170,7 @@ export class NativeCache {
|
||||
"kept",
|
||||
"linker-job",
|
||||
"runtime-pack",
|
||||
"helper",
|
||||
"binary",
|
||||
"dsym",
|
||||
"coverage",
|
||||
|
||||
@@ -28,6 +28,7 @@ import {
|
||||
requireNativeArtifact,
|
||||
runNativeTool,
|
||||
verifyNativeHelper,
|
||||
type NativeHelperVerification,
|
||||
} from "../backend/native-tools.js";
|
||||
import {
|
||||
type NativeRuntimeSelection,
|
||||
@@ -192,6 +193,44 @@ export class NativeCompiler {
|
||||
};
|
||||
}
|
||||
|
||||
/** The helpers whose `version` answer this cache recorded, by file identity:
|
||||
* the probe runs again only when the helper file changes. */
|
||||
private helperVerification(): { verified?: NativeHelperVerification } {
|
||||
const cache = this.cache;
|
||||
const toolchain = this.toolchain;
|
||||
if (cache === null) return {};
|
||||
let identity: string;
|
||||
let key: string;
|
||||
try {
|
||||
identity = nativeFileIdentity(toolchain.helperExecutable);
|
||||
key = contentDigest(
|
||||
JSON.stringify({
|
||||
schema: 1,
|
||||
helper: identity,
|
||||
compiler: nativeFileIdentity(process.execPath),
|
||||
version: toolchain.compilerVersion,
|
||||
target: toolchain.target.name,
|
||||
package: toolchain.helper.packageName,
|
||||
}),
|
||||
);
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
return {
|
||||
verified: {
|
||||
known: () => cache.read("helper", key) !== null,
|
||||
remember: () => {
|
||||
try {
|
||||
if (nativeFileIdentity(toolchain.helperExecutable) === identity)
|
||||
cache.write("helper", key, "verified\n");
|
||||
} catch {
|
||||
/* A failed write only means probing again next time. */
|
||||
}
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
private emitObject(
|
||||
input: string,
|
||||
output: string,
|
||||
@@ -211,6 +250,7 @@ export class NativeCompiler {
|
||||
compilerVersion: toolchain.compilerVersion,
|
||||
target: toolchain.target,
|
||||
helper: toolchain.helper,
|
||||
...this.helperVerification(),
|
||||
};
|
||||
let key: string | null = null;
|
||||
let identity: string | null = null;
|
||||
@@ -266,11 +306,7 @@ export class NativeCompiler {
|
||||
? null
|
||||
: this.cache.partitionDirectory(identity);
|
||||
const { profileMatch } = emitNativeObject({
|
||||
executable: toolchain.helperExecutable,
|
||||
packageRoot: toolchain.helperPackageRoot,
|
||||
compilerVersion: toolchain.compilerVersion,
|
||||
target: toolchain.target,
|
||||
helper: toolchain.helper,
|
||||
...helperOptions,
|
||||
inputPath: input,
|
||||
outputPath: output,
|
||||
sourcePath: source,
|
||||
|
||||
@@ -370,6 +370,8 @@ export function prepareNativeExecutable(
|
||||
const exclusions = { outputPaths, outputDirectories: [...directories] };
|
||||
const irInput = process.env[BOOTSTRAP_IR_INPUT] ?? "";
|
||||
let key: string | null = null;
|
||||
// The entry this build replaces: its probes give the digests of unchanged files.
|
||||
let previous: FrontendInputSnapshot | undefined;
|
||||
if (cache !== null && irInput === "" && provenanceSources() === null) {
|
||||
try {
|
||||
key = contentDigest(
|
||||
@@ -389,6 +391,7 @@ export function prepareNativeExecutable(
|
||||
timing("frontend-cache-hit");
|
||||
return cached.input;
|
||||
}
|
||||
if (cached !== null) previous = cached.stale.probes;
|
||||
const equivalent =
|
||||
cached === null || outputKind === "ir"
|
||||
? null
|
||||
@@ -403,7 +406,7 @@ export function prepareNativeExecutable(
|
||||
}
|
||||
}
|
||||
timing("frontend-cache-miss");
|
||||
const tracker = new FrontendInputTracker();
|
||||
const tracker = new FrontendInputTracker(previous);
|
||||
const prepared =
|
||||
irInput !== ""
|
||||
? preparedFromIr(irInput, entry, options)
|
||||
|
||||
@@ -327,6 +327,16 @@ describe.runIf(helper !== null)("LLVM program partitions", () => {
|
||||
const first = await readFile(parts[0]!);
|
||||
const [single] = await emit(dir, source, { outputs: 1, optLevel: "2", name: "whole" });
|
||||
expect(await readFile(single!)).toEqual(first);
|
||||
// The empty parts name the source path, so they do not depend on the
|
||||
// input's file name either.
|
||||
const renamed = await emit(dir, source, {
|
||||
outputs: 3,
|
||||
optLevel: "2",
|
||||
split: true,
|
||||
name: "renamed",
|
||||
});
|
||||
for (let index = 0; index < parts.length; index++)
|
||||
expect(await readFile(renamed[index]!)).toEqual(await readFile(parts[index]!));
|
||||
});
|
||||
|
||||
test("split code generation requires several optimized outputs and no cache", async () => {
|
||||
|
||||
@@ -225,7 +225,18 @@ static void scr_module_registry_cleanup(void) {
|
||||
* own cleanup only when the value is first materialized. That is equivalent
|
||||
* at process exit and retains the RC-audit cleanup guarantee for the values a
|
||||
* program actually observes. */
|
||||
#if defined(__linux__) && !defined(SCR_LIB)
|
||||
/* The heap census of a program built with SCRIPTC_HEAP_CENSUS
|
||||
* (heap-census.ts) writes its counters at exit, from here and from
|
||||
* scr_process_exit; every other program leaves the weak reference null.
|
||||
* Libraries have no census, and their archives declare no such symbol. */
|
||||
extern void scr_census_write(void) __attribute__((weak));
|
||||
#endif
|
||||
|
||||
static void scr_lib_cleanup(void) {
|
||||
#if defined(__linux__) && !defined(SCR_LIB)
|
||||
if (scr_census_write != NULL) scr_census_write();
|
||||
#endif
|
||||
scr_module_registry_cleanup();
|
||||
scr_arr_release(scr_argv_arr);
|
||||
scr_argv_arr = NULL;
|
||||
@@ -2137,6 +2148,9 @@ void scr_process_exit(double code) {
|
||||
fflush(stdout);
|
||||
#if defined(__linux__)
|
||||
if (__llvm_profile_write_file != NULL) (void)__llvm_profile_write_file();
|
||||
#ifndef SCR_LIB
|
||||
if (scr_census_write != NULL) scr_census_write();
|
||||
#endif
|
||||
scr_detach_teardown();
|
||||
#endif
|
||||
_Exit((int)code);
|
||||
|
||||
@@ -150,6 +150,10 @@ void scr_process_exit(double code);
|
||||
* helper process (Linux, `--detached-exit`). scr_lib.c defines a weak 0;
|
||||
* the program object of such an executable defines it as 1. */
|
||||
extern const unsigned char scr_detached_exit;
|
||||
/* The heap census writer of a program built with SCRIPTC_HEAP_CENSUS
|
||||
* (heap-census.ts): that program's object defines it, and scr_lib.c calls
|
||||
* it at exit through a weak reference (programs on Linux only). */
|
||||
void scr_census_write(void);
|
||||
void scr_process_set_exit_code(double code);
|
||||
double scr_process_exit_code_or_zero(void);
|
||||
/* The writer and the presence test of programs that read process.exitCode:
|
||||
|
||||
@@ -0,0 +1,113 @@
|
||||
/* The heap census (backend/llvm/heap-census.ts): a program built with
|
||||
* SCRIPTC_HEAP_CENSUS counts class instances and the stores, lengths and
|
||||
* accesses of array-typed fields, and writes them at exit, on a normal exit
|
||||
* and on process.exit alike. */
|
||||
import { execFile } from "node:child_process";
|
||||
import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { promisify } from "node:util";
|
||||
import { expect, test } from "vitest";
|
||||
import { compile } from "@scriptc/compiler";
|
||||
|
||||
const exec = promisify(execFile);
|
||||
const sanitize = process.env["SCRIPTC_SAN"] === "1";
|
||||
|
||||
const PROGRAM = `
|
||||
class Node {
|
||||
kids: Node[];
|
||||
tags: string[] | undefined;
|
||||
constructor(kids: Node[]) {
|
||||
this.kids = kids;
|
||||
if (kids.length > 1) this.tags = ["a", "b"];
|
||||
else this.tags = undefined;
|
||||
}
|
||||
}
|
||||
let total = 0;
|
||||
const roots: Node[] = [];
|
||||
for (let i = 0; i < 100; i++) {
|
||||
const leaves: Node[] = [];
|
||||
for (let j = 0; j < i % 6; j++) leaves.push(new Node([]));
|
||||
roots.push(new Node(leaves));
|
||||
}
|
||||
for (const r of roots) total += r.kids.length + (r.tags?.length ?? 0);
|
||||
console.log(total);
|
||||
`;
|
||||
|
||||
async function census(source: string): Promise<{ stdout: string; counts: Map<string, number> }> {
|
||||
const dir = await mkdtemp(join(tmpdir(), "scriptc-heap-census-"));
|
||||
const saved = process.env["SCRIPTC_HEAP_CENSUS"];
|
||||
try {
|
||||
const entry = join(dir, "main.ts");
|
||||
await writeFile(entry, source);
|
||||
process.env["SCRIPTC_HEAP_CENSUS"] = "1";
|
||||
const result = await compile(entry, {
|
||||
backend: "llvm",
|
||||
sanitize,
|
||||
outDir: dir,
|
||||
outPath: join(dir, "program"),
|
||||
});
|
||||
if (saved === undefined) delete process.env["SCRIPTC_HEAP_CENSUS"];
|
||||
else process.env["SCRIPTC_HEAP_CENSUS"] = saved;
|
||||
expect(result.ok, JSON.stringify(result.diagnostics)).toBe(true);
|
||||
if (!result.ok) throw new Error("compile failed");
|
||||
const out = join(dir, "census.tsv");
|
||||
const { stdout } = await exec(result.binaryPath, [], {
|
||||
env: { ...process.env, SCRIPTC_HEAP_CENSUS: out },
|
||||
timeout: 120_000,
|
||||
});
|
||||
const counts = new Map<string, number>();
|
||||
for (const line of (await readFile(out, "utf8")).trimEnd().split("\n")) {
|
||||
const [kind, cls, field, count] = line.split("\t");
|
||||
counts.set(`${kind} ${cls}${field ? "." + field : ""}`, Number(count));
|
||||
}
|
||||
return { stdout, counts };
|
||||
} finally {
|
||||
if (saved === undefined) delete process.env["SCRIPTC_HEAP_CENSUS"];
|
||||
else process.env["SCRIPTC_HEAP_CENSUS"] = saved;
|
||||
await rm(dir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
const EXPECTED: [string, number][] = [
|
||||
// 100 roots and 246 leaves (i % 6 children each).
|
||||
["new Node", 346],
|
||||
["store Node.kids", 346],
|
||||
["elements Node.kids", 246],
|
||||
["len0 Node.kids", 263],
|
||||
["len1 Node.kids", 17],
|
||||
["len2 Node.kids", 17],
|
||||
["len3-4 Node.kids", 33],
|
||||
["len5-8 Node.kids", 16],
|
||||
["len9-16 Node.kids", 0],
|
||||
// Literals count where the store itself writes one.
|
||||
["literal Node.kids", 0],
|
||||
["store Node.tags", 346],
|
||||
["literal Node.tags", 66],
|
||||
// An undefined tags field counts as length 0.
|
||||
["len0 Node.tags", 280],
|
||||
["len2 Node.tags", 66],
|
||||
["elements Node.tags", 132],
|
||||
];
|
||||
|
||||
test.skipIf(process.platform !== "linux")(
|
||||
"the census counts instances and array field stores at a normal exit",
|
||||
async () => {
|
||||
const { stdout, counts } = await census(PROGRAM);
|
||||
expect(stdout).toBe("378\n");
|
||||
for (const [key, count] of EXPECTED) expect([key, counts.get(key)]).toEqual([key, count]);
|
||||
// Every store also computes the field's address.
|
||||
expect(counts.get("access Node.kids")).toBeGreaterThanOrEqual(346 + 100);
|
||||
},
|
||||
240_000,
|
||||
);
|
||||
|
||||
test.skipIf(process.platform !== "linux")(
|
||||
"the census is written on process.exit too",
|
||||
async () => {
|
||||
const { stdout, counts } = await census(PROGRAM + "process.exit(0);\n");
|
||||
expect(stdout).toBe("378\n");
|
||||
expect(counts.get("new Node")).toBe(346);
|
||||
},
|
||||
240_000,
|
||||
);
|
||||
Reference in New Issue
Block a user