Free a typed teardown's blocks in the order it met them (#1849)

This commit is contained in:
Chris Tate
2026-10-23 12:09:37 -05:00
committed by GitHub
11 changed files with 732 additions and 68 deletions
+65 -12
View File
@@ -459,18 +459,72 @@ export type FreeObservers = "weak" | "weak-or-source";
* type's disposals), and the block — `scr_cyc_free` for collector-headered
* objects, `scr_rt_free` otherwise. Labels use the `sf.` prefix; the lines
* end in an open block.
*
* With `keep` (an alloca of ptr holding NULL; inline allocation paths only),
* a block that would go onto the calling thread's free list is stored there
* instead, for a teardown segment (emitSegmentFree); a block the system or
* another arena owns is still freed.
*/
export function emitObjectFree(
host: AllocHost,
cyc: boolean,
observers: FreeObservers = "weak",
keep: string | null = null,
): string[] {
const lines = threadLocalState(host, "sf", freeLines(host, cyc, observers));
const lines = threadLocalState(host, "sf", freeLines(host, cyc, observers, keep));
if (host.objectAudit === false) return lines.filter((line) => line !== FREE_NOTE);
host.declare(`declare void @scr_obj_free_note()`);
return lines;
}
/** Lines that put the blocks `first` to `last` (linked through their first
* words; one block when they are equal) of the class at reservation offset
* `off` on the calling thread's free list. */
function freeListPushLines(prefix: string, off: string, first: string, last: string): string[] {
return [
` %${prefix}.shp = getelementptr inbounds i8, ptr @scr_sa, i64 ${SA_SHIFT}`,
` %${prefix}.sh32 = load i32, ptr %${prefix}.shp`,
` %${prefix}.sh = zext i32 %${prefix}.sh32 to i64`,
` %${prefix}.c = lshr i64 ${off}, %${prefix}.sh`,
` %${prefix}.c8 = shl i64 %${prefix}.c, 3`,
` %${prefix}.fo = add i64 %${prefix}.c8, ${SA_FREE}`,
` %${prefix}.fp = getelementptr inbounds i8, ptr @scr_sa, i64 %${prefix}.fo`,
` %${prefix}.head = load ptr, ptr %${prefix}.fp`,
` store ptr %${prefix}.head, ptr ${last}`,
` store ptr ${first}, ptr %${prefix}.fp`,
];
}
/**
* Lines that free a teardown segment: the blocks `first` to `last` (ptr
* values, both NULL for an empty segment), linked through their first words,
* which emitObjectFree kept. Every block of a segment is in the calling
* thread's arena and of one size class, so the segment goes onto that
* class's free list in one step, and the next allocations of the class take
* its blocks in segment order. Labels use `prefix`; the lines end in the
* open block `${prefix}.done`.
*/
export function emitSegmentFree(
host: AllocHost,
prefix: string,
first: string,
last: string,
): string[] {
declareState(host);
const lines = [
` %${prefix}.none = icmp eq ptr ${first}, null`,
` br i1 %${prefix}.none, label %${prefix}.done, label %${prefix}.push`,
`${prefix}.push:`,
` %${prefix}.base = load i64, ptr @scr_sa`,
` %${prefix}.bi = ptrtoint ptr ${first} to i64`,
` %${prefix}.off = sub i64 %${prefix}.bi, %${prefix}.base`,
...freeListPushLines(prefix, `%${prefix}.off`, first, last),
` br label %${prefix}.done`,
`${prefix}.done:`,
];
return threadLocalState(host, prefix, lines);
}
/** Lines that leave in `%<prefix>.obs` whether the program can observe a
* disposal of this type (folds to a constant). */
function observedLines(prefix: string, observers: FreeObservers): string[] {
@@ -482,8 +536,14 @@ function observedLines(prefix: string, observers: FreeObservers): string[] {
];
}
function freeLines(host: AllocHost, cyc: boolean, observers: FreeObservers): string[] {
function freeLines(
host: AllocHost,
cyc: boolean,
observers: FreeObservers,
keep: string | null,
): string[] {
if (host.inlineAlloc !== true) {
if (keep !== null) throw new Error("a kept block needs the inline allocation paths");
if (cyc) {
host.declare(`declare void @scr_cyc_free(ptr)`);
return [` call void @scr_obj_free_note()`, ` call void @scr_cyc_free(ptr %o)`];
@@ -548,16 +608,9 @@ function freeLines(host: AllocHost, cyc: boolean, observers: FreeObservers): str
` %sf.inx = call i1 @llvm.expect.i1(i1 %sf.in, i1 true)`,
` br i1 %sf.inx, label %sf.push, label %sf.sys`,
`sf.push:`,
` %sf.shp = getelementptr inbounds i8, ptr @scr_sa, i64 ${SA_SHIFT}`,
` %sf.sh32 = load i32, ptr %sf.shp`,
` %sf.sh = zext i32 %sf.sh32 to i64`,
` %sf.c = lshr i64 %sf.off, %sf.sh`,
` %sf.c8 = shl i64 %sf.c, 3`,
` %sf.fo = add i64 %sf.c8, ${SA_FREE}`,
` %sf.fp = getelementptr inbounds i8, ptr @scr_sa, i64 %sf.fo`,
` %sf.head = load ptr, ptr %sf.fp`,
` store ptr %sf.head, ptr ${blk}`,
` store ptr ${blk}, ptr %sf.fp`,
...(keep !== null
? [` store ptr ${blk}, ptr ${keep}`]
: freeListPushLines("sf", "%sf.off", blk, blk)),
` br label %sf.done`,
`sf.sys: ; outside the reservation, or the allocator is compiled out`,
` call void @scr_obj_free_note()`,
@@ -77,6 +77,65 @@ test("leaf layouts keep direct teardown while recursive layouts share the depth
}
});
test("teardowns of types with children of their own type free their blocks as one segment", () => {
const linked = { kind: "object" as const, className: "Linked" };
const mod: IrModule = {
irVersion: 17,
sourceFile: loc.file,
entry: "main",
functions: [{ name: "main", params: [], returnType: VOID, locals: [], body: [], loc }],
classes: [
{
name: "Linked",
fields: [
{ name: "next", type: linked },
{ name: "other", type: linked },
],
methods: [],
loc,
},
],
records: [{ id: "branch", fields: [{ name: "next", type: linked }] }],
};
const define = (llvm: string, name: string): string =>
new RegExp(`^define internal [^@]*@${name}\\([^]*?^}`, "m").exec(llvm)?.[0] ?? "";
const linkedDestroy = `${mangleClassRelease("Linked")}_destroy`;
const branchDestroy = `${mangleRecordRelease("branch")}_destroy`;
for (const workers of [false, true]) {
const llvm = emitLlvmModule({ ...mod, workers }, { pointerBits: 64 });
// Linked's teardown keeps its block and its children's in a segment: it
// releases its own block first and places it by address against its
// first child, a child torn down directly appends to the segment, one
// past the direct chain appends through the runtime, and the chain's
// first teardown frees the segment onto the free list in one step.
const seg = define(llvm, `${linkedDestroy}_s`);
expect(seg.startsWith("define internal ptr"), "segment form").toBe(true);
expect(seg).toMatch(
/store ptr %(?:sf\.blk|o), ptr %sg\.own[^]*call ptr @sc_release_Linked_destroy_s\(/,
);
// The children's releases have a copy for each place of the own block.
for (const copy of ["ta", "tb"])
expect(seg).toContain(
`call ptr @scr_rc_destroy_seg(ptr %v0, ptr @${linkedDestroy}_sc, ptr @${linkedDestroy}, ptr %${copy}0.bt)`,
);
expect(seg).not.toContain("call void @scr_rc_destroy(");
expect(seg).toContain("icmp ult ptr %sg.o, %sg.r1");
const first = define(llvm, `${linkedDestroy}_d`);
expect(first).toContain(`call ptr @${linkedDestroy}_s(ptr %o, i32 %d, ptr %head)`);
expect(first).toContain("store ptr %first, ptr %sgf.fp");
expect(first.includes("llvm.threadlocal.address"), "thread-local state").toBe(workers);
expect(define(llvm, linkedDestroy)).toContain(`call void @${linkedDestroy}_d(ptr %o, i32 0)`);
// A record whose child is another type frees its own block as before.
expect(define(llvm, `${branchDestroy}_s`)).toBe("");
expect(define(llvm, branchDestroy)).toContain(`call void @${mangleClassRelease("Linked")}(`);
}
// Without the inline allocation paths (32-bit targets) a teardown frees
// through the runtime, block by block.
const llvm32 = emitLlvmModule(mod, { pointerBits: 32 });
expect(define(llvm32, `${linkedDestroy}_s`)).toBe("");
expect(define(llvm32, `${linkedDestroy}_d`)).toContain(`call void @${linkedDestroy}_d(`);
});
test("releases split into an inlinable fast path and an out-of-line remainder", () => {
// An array of the class itself makes the hierarchy cycle-capable.
const kids = { kind: "array" as const, elem: { kind: "object" as const, className: "Node" } };
+58 -15
View File
@@ -43,6 +43,9 @@ import {
destroyDefinitions,
directChildReleaseLines,
directDestroyTarget,
segmentPrologueLines,
segmentTeardownLines,
segmentedTeardown,
usedDirectTargets,
type DirectDestroy,
type RegionHooks,
@@ -1082,25 +1085,62 @@ export function emitClassShapes(
bounded &&
(directTargets.has(`object:${cls.name}`) ||
refFields.some((f) => directDestroyTarget(host, f.type, directTargets) !== null));
// The teardown's own symbol, and whether it keeps a segment (shapes.ts
// segmentedTeardown).
const destroy = `${meta.hierarchy ? mangleClassReleaseDirect(cls.name) : mangleClassRelease(cls.name)}_destroy`;
const segmented =
chained &&
!meta.compact &&
runtimePrefix === null &&
!isStreamRooted &&
!meta.threadSlot &&
segmentedTeardown(
host,
destroy,
refFields.map((f) => f.type),
directTargets,
);
// The field-releasing teardown body shared by both release shapes
// (the public one on standalone classes, the DIRECT one on hierarchy
// members). Runs at rc == 0.
const teardown = (lines: string[]): void => {
refFields.forEach((f, t) => {
lines.push(
` %f${t} = getelementptr inbounds %${struct}, ptr %o, i64 0, i32 ${f.index}`,
` %v${t} = load ptr, ptr %f${t}`,
);
const release = (prefix: string, f: (typeof refFields)[number], t: number): string[] => {
const direct = bounded ? directDestroyTarget(host, f.type, directTargets) : null;
if (direct)
lines.push(
...directChildReleaseLines(host, `td${t}`, `%v${t}`, direct, llvmCommentText(f.name)),
);
else
lines.push(
if (direct === null)
return [
` call void ${releaseSym(host, f.type)}(ptr %v${t}) ; ${llvmCommentText(f.name)}`,
);
});
];
const own = segmented && direct.destroy === destroy;
return directChildReleaseLines(
host,
`${prefix}${t}`,
`%v${t}`,
direct,
llvmCommentText(f.name),
own,
);
};
const loads = refFields.map((f, t) => [
` %f${t} = getelementptr inbounds %${struct}, ptr %o, i64 0, i32 ${f.index}`,
` %v${t} = load ptr, ptr %f${t}`,
]);
if (segmented) {
// A segmented teardown loads its fields, releases its own block, and
// then releases the fields (shapes.ts segmentTeardownLines).
const ownType = refFields.flatMap((f, t) =>
directDestroyTarget(host, f.type, directTargets)?.destroy === destroy ? [`%v${t}`] : [],
);
lines.push(
...segmentPrologueLines(),
...loads.flat(),
...emitObjectFree(host, traced, "weak", "%sg.own"),
...segmentTeardownLines(ownType, (prefix) =>
refFields.flatMap((f, t) => release(prefix, f, t)),
),
);
return;
}
refFields.forEach((f, t) => lines.push(...loads[t]!, ...release("td", f, t)));
if (runtimePrefix) lines.push(...regCall("td", runtimePrefix.drop, ""));
if (isStreamRooted) lines.push(...stCall("tds", "scr_stream_st_release", ""));
if (meta.threadSlot) lines.push(...slotForgetLines(host, struct, meta, "tdw"));
@@ -1182,10 +1222,12 @@ export function emitClassShapes(
];
if (traced) reld.push(...cycleDeadLines(host, "teardown"));
if (bounded) {
const destroy = `${mangleClassReleaseDirect(cls.name)}_destroy`;
const destroyBody: string[] = [];
teardown(destroyBody);
defs.push(...destroyDefinitions(destroy, destroyBody, chained), ``);
defs.push(
...destroyDefinitions(destroy, destroyBody, chained, segmented ? host : null),
``,
);
host.declare(`declare void @scr_rc_destroy(ptr, ptr)`);
reld.push(` call void @scr_rc_destroy(ptr %o, ptr @${destroy})`);
} else teardown(reld);
@@ -1207,6 +1249,7 @@ export function emitClassShapes(
bounded,
lifetime,
chained,
segmented,
),
``,
);
+218 -24
View File
@@ -34,6 +34,7 @@ import {
cycTypeDescriptor,
emitObjectAlloc,
emitObjectFree,
emitSegmentFree,
CYC_APPEND,
cycleAppendDecls,
cycleAppendHelper,
@@ -1405,17 +1406,23 @@ export interface DirectDestroy {
/** The release of child `value` (a loaded field, possibly NULL) inside a
* teardown whose chain length is `%d`: the child's own release for a shared
* or immortal reference, its teardown for the last one. Labels use `tag`;
* the lines end in the open block `${tag}.next`. */
* the lines end in the open block `${tag}.next`.
*
* With `segment` (the child has the teardown's own type, in a segmented
* teardown; segmentPrologueLines), the child's teardown appends its blocks
* to the teardown's segment. */
export function directChildReleaseLines(
host: ShapeHost,
tag: string,
value: string,
target: DirectDestroy,
comment: string,
segment = false,
): string[] {
const S = host.sizeType;
const C = target.count ?? S;
host.declare(`declare void @scr_rc_destroy(ptr, ptr)`);
if (segment) host.declare(`declare ptr @scr_rc_destroy_seg(ptr, ptr, ptr, ptr)`);
else host.declare(`declare void @scr_rc_destroy(ptr, ptr)`);
const lines = [
` %${tag}.nn = icmp ne ptr ${value}, null`,
` br i1 %${tag}.nn, label %${tag}.chk, label %${tag}.next`,
@@ -1447,16 +1454,139 @@ export function directChildReleaseLines(
` br i1 %${tag}.short, label %${tag}.direct, label %${tag}.bounded`,
`${tag}.direct:`,
` %${tag}.d = add i32 %d, 1`,
` call void @${target.destroy}_d(ptr ${value}, i32 %${tag}.d)`,
);
if (!segment) {
lines.push(
` call void @${target.destroy}_d(ptr ${value}, i32 %${tag}.d)`,
` br label %${tag}.next`,
`${tag}.bounded:`,
` call void @scr_rc_destroy(ptr ${value}, ptr @${target.destroy})`,
` br label %${tag}.next`,
`${tag}.next:`,
);
return lines;
}
lines.push(
` %${tag}.t = load ptr, ptr %sg.t`,
` %${tag}.dt = call ptr @${target.destroy}_s(ptr ${value}, i32 %${tag}.d, ptr %${tag}.t)`,
` store ptr %${tag}.dt, ptr %sg.t`,
` br label %${tag}.next`,
`${tag}.bounded:`,
` call void @scr_rc_destroy(ptr ${value}, ptr @${target.destroy})`,
` %${tag}.bt = load ptr, ptr %sg.t`,
` %${tag}.rt = call ptr @scr_rc_destroy_seg(ptr ${value}, ptr @${target.destroy}_sc, ptr @${target.destroy}, ptr %${tag}.bt)`,
` store ptr %${tag}.rt, ptr %sg.t`,
` br label %${tag}.next`,
`${tag}.next:`,
);
return lines;
}
/* Teardown segments. A typed teardown that frees each block onto its free
* list as it goes frees a tree in walk order, and the free lists, which are
* LIFO, hand the blocks to the next allocations in the reverse order: a tree
* built again in the same order as the last (each generation of a structure
* a loop rebuilds) is laid out mirrored, and every walk over it, the next
* teardown included, runs through memory backwards, which defeats the
* hardware prefetchers. The runtime frees large dead sets of the cycle
* collector in address order for the same reason (scr_mem_order_for_free).
*
* So the teardown of a type with children of its own type (the `_s` form,
* segmentedTeardown) keeps the blocks it frees in a segment, linked through
* their first words, which it builds in order through a tail pointer: the
* `_s` form takes the segment's last link (the first word of its last block,
* or the head slot of an empty segment) and returns the new one. A node
* first releases its own block (emitObjectFree, keeping it), then appends it
* ahead of its children's blocks when its address is the lower, behind them
* otherwise (segmentTeardownLines), and the children append theirs in field
* order, so a structure
* built parent first (object literals allocate before their fields) or
* children first (constructor arguments) comes back in ascending order. The
* chain's first teardown (`_d`) frees the whole segment onto the class's
* free list in one step (emitSegmentFree), and the walks over the next
* generation run forward. A teardown past DIRECT_DESTROY_CHAIN appends
* through scr_rc_destroy_seg, under the same depth budget as
* scr_rc_destroy. Blocks outside the calling thread's arena are freed as
* before; one type's blocks are all of one class. */
/** Whether the teardown of a type whose reference fields have `fieldTypes`
* (with direct targets `targets`) and whose own teardown is `destroy` keeps
* a segment: a child field has its own type, and the inline allocation paths
* are in use. */
export function segmentedTeardown(
host: ShapeHost,
destroy: string,
fieldTypes: readonly IrType[],
targets: ReadonlyMap<string, DirectDestroy>,
): boolean {
if (host.inlineAlloc !== true) return false;
return fieldTypes.some((t) => directDestroyTarget(host, t, targets)?.destroy === destroy);
}
/** The first lines of a segmented teardown, in its entry block: the
* segment's last link (`%sg.t`, from the `%tail` argument) and the kept own
* block (`%sg.own`, for emitObjectFree). */
export function segmentPrologueLines(): string[] {
return [
` %sg.t = alloca ptr`,
` %sg.own = alloca ptr`,
` store ptr %tail, ptr %sg.t`,
` store ptr null, ptr %sg.own`,
];
}
/** The rest of a segmented teardown after emitObjectFree kept (or freed)
* its own block: the block goes ahead of its children's when its address is
* below the first non-NULL one of `children` (loaded fields of the
* teardown's own type, which stand for their blocks) or there is none, and
* behind them otherwise; then the children's releases (`releases`, given a
* label prefix), and the segment's last link is returned. Each order has its
* own copy of the releases, so neither keeps the decision, or the block once
* it is appended, across the children's teardowns. */
export function segmentTeardownLines(
children: readonly string[],
releases: (prefix: string) => string[],
): string[] {
const lines = [` %sg.o = load ptr, ptr %sg.own`, ` %sg.kept = icmp ne ptr %sg.o, null`];
let ref = "null";
[...children].reverse().forEach((child, i) => {
lines.push(
` %sg.nn${i} = icmp ne ptr ${child}, null`,
` %sg.r${i} = select i1 %sg.nn${i}, ptr ${child}, ptr ${ref}`,
);
ref = `%sg.r${i}`;
});
lines.push(
` %sg.leaf = icmp eq ptr ${ref}, null`,
` %sg.lower = icmp ult ptr %sg.o, ${ref}`,
` %sg.first = or i1 %sg.leaf, %sg.lower`,
` %sg.ahead = and i1 %sg.kept, %sg.first`,
` br i1 %sg.ahead, label %sg.front, label %sg.back`,
`sg.front:`,
...segmentAppendLines("a"),
...releases("ta"),
` br label %sg.ret`,
`sg.back:`,
...releases("tb"),
` %sg.keep = icmp ne ptr %sg.o, null`,
` br i1 %sg.keep, label %sg.last, label %sg.ret`,
`sg.last:`,
...segmentAppendLines("l"),
` br label %sg.ret`,
`sg.ret:`,
` %sg.tr = load ptr, ptr %sg.t`,
` ret ptr %sg.tr`,
);
return lines;
}
function segmentAppendLines(tag: string): string[] {
return [
` %sg.t${tag} = load ptr, ptr %sg.t`,
` store ptr %sg.o, ptr %sg.t${tag}`,
` store ptr %sg.o, ptr %sg.t`,
];
}
/** The direct-teardown target of a reference of `type`, if `targets` (keyed
* `record:<id>` / `object:<class>`) has one. Nullable unions hold the arm's
* pointer, NULL or an immortal sentinel, which the child release handles. */
@@ -1499,12 +1629,42 @@ export function usedDirectTargets(
/** The teardown definitions for `destroy`: with `chained`, the body takes
* the chain length `%d` (directChildReleaseLines) and the runtime's
* one-argument entry starts a chain. */
* one-argument entry starts a chain. With `segment` (the host, for a
* segmented teardown), the body is a segment teardown's
* (segmentPrologueLines to segmentTeardownLines), defined as `_s`, with
* `_sc` for scr_rc_destroy_seg, and `_d` starts a segment and frees it. */
export function destroyDefinitions(
destroy: string,
body: readonly string[],
chained: boolean,
segment: ShapeHost | null = null,
): string[] {
if (segment !== null)
return [
`define internal ptr @${destroy}_s(ptr %o, i32 %d, ptr %tail) ${FN_ATTRS} {`,
`entry:`,
...body,
`}`,
`define internal ptr @${destroy}_sc(ptr %o, ptr %tail) ${FN_ATTRS} {`,
`entry:`,
` %t = call ptr @${destroy}_s(ptr %o, i32 0, ptr %tail)`,
` ret ptr %t`,
`}`,
`define internal void @${destroy}_d(ptr %o, i32 %d) ${FN_ATTRS} {`,
`entry:`,
` %head = alloca ptr`,
` store ptr null, ptr %head`,
` %last = call ptr @${destroy}_s(ptr %o, i32 %d, ptr %head)`,
` %first = load ptr, ptr %head`,
...emitSegmentFree(segment, "sgf", "%first", "%last"),
` ret void`,
`}`,
`define internal void @${destroy}(ptr %o) ${FN_ATTRS} {`,
`entry:`,
` call void @${destroy}_d(ptr %o, i32 0)`,
` ret void`,
`}`,
];
if (!chained)
return [
`define internal void @${destroy}(ptr %o) ${FN_ATTRS} {`,
@@ -1543,12 +1703,13 @@ export function releaseBody(
bounded = true,
immortal = false,
chained = false,
segmented = false,
): string[] {
const S = host.sizeType;
if (bounded) host.declare(`declare void @scr_rc_destroy(ptr, ptr)`);
const destroy = `${fnName}_destroy`;
const lines = [
...(bounded ? destroyDefinitions(destroy, freeBody, chained) : []),
...(bounded ? destroyDefinitions(destroy, freeBody, chained, segmented ? host : null) : []),
...releaseFastPath(host, fnName, `${fnName}_slow`, traced, comment, immortal),
`define internal void @${fnName}_slow(ptr %o) noinline ${FN_ATTRS} {${comment ? ` ; ${comment}` : ""}`,
`entry:`,
@@ -1780,26 +1941,58 @@ export function emitRecordShapes(
const chained =
directTargets.has(`record:${shape.id}`) ||
refMembers.some((m) => directDestroyTarget(host, m.type, directTargets) !== null);
for (const m of refMembers) {
freeBody.push(
` %f${t} = getelementptr inbounds %${struct}, ptr %o, i64 0, i32 ${m.index}`,
` %v${t} = load ptr, ptr %f${t}`,
const destroy = `${mangleRecordRelease(shape.id)}_destroy`;
const bounded = refMembers.some((member) => needsBoundedRelease(member.type));
const segmented =
bounded &&
segmentedTeardown(
host,
destroy,
refMembers.map((m) => m.type),
directTargets,
);
const direct = directDestroyTarget(host, m.type, directTargets);
if (direct) {
freeBody.push(
...directChildReleaseLines(host, `td${t}`, `%v${t}`, direct, llvmCommentText(m.name)),
);
} else {
freeBody.push(
` call void ${releaseSym(host, m.type)}(ptr %v${t}) ; ${llvmCommentText(m.name)}`,
);
}
t++;
}
// Closed layouts can carry a kept record source (record-sources.ts).
const observers = recordMayKeepSource(shape) ? "weak-or-source" : "weak";
freeBody.push(...emitObjectFree(host, traced, observers));
// A segmented teardown loads its fields, releases its own block, and
// then releases the fields (segmentTeardownLines).
const loads: string[] = [];
const ownType: string[] = [];
const release = (prefix: string, m: (typeof refMembers)[number], i: number): string[] => {
const direct = directDestroyTarget(host, m.type, directTargets);
if (direct === null)
return [` call void ${releaseSym(host, m.type)}(ptr %v${i}) ; ${llvmCommentText(m.name)}`];
const own = segmented && direct.destroy === destroy;
return directChildReleaseLines(
host,
`${prefix}${i}`,
`%v${i}`,
direct,
llvmCommentText(m.name),
own,
);
};
for (const m of refMembers) {
const fieldLines = [
` %f${t} = getelementptr inbounds %${struct}, ptr %o, i64 0, i32 ${m.index}`,
` %v${t} = load ptr, ptr %f${t}`,
];
if (segmented) {
loads.push(...fieldLines);
if (directDestroyTarget(host, m.type, directTargets)?.destroy === destroy)
ownType.push(`%v${t}`);
} else freeBody.push(...fieldLines, ...release("td", m, t));
t++;
}
if (segmented)
freeBody.push(
...segmentPrologueLines(),
...loads,
...emitObjectFree(host, traced, observers, "%sg.own"),
...segmentTeardownLines(ownType, (prefix) =>
refMembers.flatMap((m, i) => release(prefix, m, i)),
),
);
else freeBody.push(...emitObjectFree(host, traced, observers));
defs.push(
...releaseBody(
host,
@@ -1807,9 +2000,10 @@ export function emitRecordShapes(
traced,
freeBody,
"",
refMembers.some((member) => needsBoundedRelease(member.type)),
bounded,
false,
chained,
segmented,
),
``,
);
@@ -0,0 +1,6 @@
{
"order": [
"<repo>/tests/corpus/7260-teardown-segments.ts"
],
"diags": []
}
+39 -17
View File
@@ -293,24 +293,22 @@ static SCR_TL ScrDestroyEntry *scr_destroy_pending;
static SCR_TL size_t scr_destroy_count, scr_destroy_capacity;
#endif
void scr_rc_destroy(void *obj, void (*destroy)(void *)) {
if (scr_destroy_depth == SCR_DESTROY_DEPTH) {
if (scr_destroy_count == scr_destroy_capacity) {
size_t capacity = scr_destroy_capacity ? scr_destroy_capacity * 2 : 64;
if (capacity < scr_destroy_capacity || capacity > SIZE_MAX / sizeof(ScrDestroyEntry))
scr_cyc_oom();
ScrDestroyEntry *entries = realloc(scr_destroy_pending, capacity * sizeof(*entries));
if (!entries) scr_cyc_oom();
scr_destroy_pending = entries;
scr_destroy_capacity = capacity;
}
scr_destroy_pending[scr_destroy_count++] = (ScrDestroyEntry){ obj, destroy };
return;
/* Queues a disposal past the depth budget; the outermost disposal runs it. */
static void scr_rc_destroy_defer(void *obj, void (*destroy)(void *)) {
if (scr_destroy_count == scr_destroy_capacity) {
size_t capacity = scr_destroy_capacity ? scr_destroy_capacity * 2 : 64;
if (capacity < scr_destroy_capacity || capacity > SIZE_MAX / sizeof(ScrDestroyEntry))
scr_cyc_oom();
ScrDestroyEntry *entries = realloc(scr_destroy_pending, capacity * sizeof(*entries));
if (!entries) scr_cyc_oom();
scr_destroy_pending = entries;
scr_destroy_capacity = capacity;
}
scr_destroy_depth++;
destroy(obj);
scr_destroy_depth--;
if (scr_destroy_depth) return;
scr_destroy_pending[scr_destroy_count++] = (ScrDestroyEntry){ obj, destroy };
}
/* Runs the queued disposals once the outermost one has returned. */
static void scr_rc_destroy_drain(void) {
while (scr_destroy_count) {
ScrDestroyEntry entry = scr_destroy_pending[--scr_destroy_count];
scr_destroy_depth = 1;
@@ -326,6 +324,30 @@ void scr_rc_destroy(void *obj, void (*destroy)(void *)) {
}
}
void scr_rc_destroy(void *obj, void (*destroy)(void *)) {
if (scr_destroy_depth == SCR_DESTROY_DEPTH) {
scr_rc_destroy_defer(obj, destroy);
return;
}
scr_destroy_depth++;
destroy(obj);
scr_destroy_depth--;
if (scr_destroy_depth == 0) scr_rc_destroy_drain();
}
void *scr_rc_destroy_seg(void *obj, void *(*destroy_seg)(void *, void *),
void (*destroy)(void *), void *tail) {
if (scr_destroy_depth == SCR_DESTROY_DEPTH) {
scr_rc_destroy_defer(obj, destroy);
return tail;
}
scr_destroy_depth++;
tail = destroy_seg(obj, tail);
scr_destroy_depth--;
if (scr_destroy_depth == 0) scr_rc_destroy_drain();
return tail;
}
/* Live cycle-headered objects and the running count of OLD objects freed.
* The inline allocation/free paths (scr_runtime.h, llvm/alloc.ts) maintain
* both counters so full passes can measure old-generation reclamation. */
+8
View File
@@ -552,6 +552,14 @@ _Static_assert(sizeof(((ScrCycHdr *)0)->gen) == 1 && SCR_CYC_OLD == 2,
* depth; deeper objects wait until the outer disposal drains the worklist.
* The callback owns the object and must release its children and free it. */
void scr_rc_destroy(void *obj, void (*destroy)(void *));
/* scr_rc_destroy for an emitted teardown that keeps the blocks it frees in
* a segment, linked through their first words, instead of freeing them onto
* the free list (llvm/shapes.ts): destroy_seg tears obj down, appending its
* blocks after the segment's last link `tail`, and returns the new last
* link, which this returns. A disposal past the depth budget is queued with
* `destroy`, which frees its own blocks, and leaves `tail` as it was. */
void *scr_rc_destroy_seg(void *obj, void *(*destroy_seg)(void *, void *),
void (*destroy)(void *), void *tail);
/* RC hooks for cycle-headered types. on_release: rc was decremented and
* stayed above zero — buffer the object as a possible cycle root (may run
+2
View File
@@ -409,6 +409,7 @@
"tests/harness/stdlib-surface.test.ts": 48,
"tests/harness/strip-vt-linear.test.ts": 1795,
"tests/harness/surface-manifest.test.ts": 5309,
"tests/harness/teardown-segments.test.ts": 2600,
"tests/harness/terminal-geometry.test.ts": 2730,
"tests/harness/test262-runner.test.ts": 207,
"tests/harness/three-library.test.ts": 251122,
@@ -828,6 +829,7 @@
"tests/harness/stdlib-surface.test.ts": 46,
"tests/harness/strip-vt-linear.test.ts": 2546,
"tests/harness/surface-manifest.test.ts": 5951,
"tests/harness/teardown-segments.test.ts": 5,
"tests/harness/terminal-geometry.test.ts": 3270,
"tests/harness/test262-runner.test.ts": 230,
"tests/harness/three-library.test.ts": 361470,
+179
View File
@@ -0,0 +1,179 @@
// Teardowns that hand their blocks back in order.
//
// A typed teardown of a type with children of its own type keeps the blocks
// it frees in one segment and frees the segment when the chain's first
// teardown returns, so the next structure built of that type reuses them in
// order. This program rebuilds structures of such types for many
// generations and reads every value of each generation before dropping it:
// a block handed out twice, or one freed while still in use, shows up as a
// wrong checksum. The shapes cover the ways a segment is put together:
// record literals (allocated before their fields) and constructors whose
// arguments build the children first, lists built by prepending and by
// appending, trees far deeper than one direct teardown chain and lists
// longer than the runtime's whole depth budget (whose tails wait in its
// queue), subtrees still owned elsewhere when their parent goes, records of
// one type holding records of another, and nodes of a cycle-capable class
// that are still candidates of the cycle collector when the last reference
// goes, and rings of them that only the collector frees.
interface Node {
left: Node | null;
right: Node | null;
value: number;
}
function literalTree(depth: number, value: number): Node {
return depth === 0
? { left: null, right: null, value }
: { left: literalTree(depth - 1, value * 2), right: literalTree(depth - 1, value * 2 + 1), value };
}
function sum(node: Node | null): number {
return node === null ? 0 : node.value + sum(node.left) + sum(node.right);
}
class Branch {
readonly left: Branch | null;
readonly right: Branch | null;
readonly label: string;
constructor(left: Branch | null, right: Branch | null, label: string) {
this.left = left;
this.right = right;
this.label = label;
}
}
function classTree(depth: number, label: string): Branch {
return depth === 0
? new Branch(null, null, label)
: new Branch(classTree(depth - 1, label + "l"), classTree(depth - 1, label + "r"), label);
}
function labels(branch: Branch | null): number {
return branch === null ? 0 : branch.label.length + labels(branch.left) + labels(branch.right);
}
interface Cell {
next: Cell | null;
n: number;
}
function prepended(count: number, base: number): Cell | null {
let head: Cell | null = null;
for (let i = 0; i < count; i++) head = { next: head, n: base + i };
return head;
}
function appended(count: number, base: number): Cell | null {
if (count === 0) return null;
const head: Cell = { next: null, n: base };
let tail = head;
for (let i = 1; i < count; i++) {
const cell: Cell = { next: null, n: base + i };
tail.next = cell;
tail = cell;
}
return head;
}
function cells(head: Cell | null): [number, number] {
let count = 0;
let total = 0;
for (let cell = head; cell !== null; cell = cell.next) {
count++;
total += cell.n;
}
return [count, total];
}
// Generations of literal trees: shallow, one chain deep, and several chains
// deep. Each generation is checked against the sum the shape must have.
for (const depth of [0, 1, 5, 16, 17, 18]) {
let ok = true;
for (let generation = 0; generation < 6; generation++) {
const tree = literalTree(depth, 1);
const expected = ((2 ** (depth + 1)) * (2 ** (depth + 1) - 1)) / 2;
if (sum(tree) !== expected) ok = false;
}
console.log(`literal trees of depth ${depth}: ${ok}`);
}
// Generations of class trees built children first.
for (const depth of [3, 12, 17]) {
let total = 0;
for (let generation = 0; generation < 4; generation++) total += labels(classTree(depth, "t"));
console.log(`class trees of depth ${depth}: ${total}`);
}
// Lists both ways, alternating, so each reuses the other's blocks; the long
// ones outlast the runtime's depth budget.
for (const count of [1, 15, 16, 17, 1000, 3000, 40000]) {
const seen: string[] = [];
for (let generation = 0; generation < 4; generation++) {
const list = generation % 2 === 0 ? prepended(count, generation) : appended(count, generation);
const [length, total] = cells(list);
seen.push(`${length}/${total}`);
}
console.log(`lists of ${count}: ${seen.join(" ")}`);
}
// Subtrees owned elsewhere survive their parent's teardown, and the blocks
// freed around them are not handed out while they live.
const kept: Node[] = [];
for (let generation = 0; generation < 5; generation++) {
const tree = literalTree(10, generation + 1);
if (tree.left !== null && tree.left.right !== null) kept.push(tree.left.right);
if (tree.right !== null) kept.push(tree.right);
// Building the next tree reuses the dropped blocks.
literalTree(10, 7);
}
console.log(`kept subtrees: ${kept.length} ${kept.map(sum).join(",")}`);
// A record of one type holding trees of another: the outer teardown frees
// its own blocks as before and tears each inner tree down as a segment.
interface Forest {
first: Node;
second: Node | null;
rest: Forest | null;
}
function forest(count: number): Forest | null {
let out: Forest | null = null;
for (let i = 0; i < count; i++) out = { first: literalTree(4, i), second: i % 3 === 0 ? null : literalTree(2, i), rest: out };
return out;
}
let forestTotal = 0;
for (let generation = 0; generation < 3; generation++) {
for (let f = forest(500); f !== null; f = f.rest) forestTotal += sum(f.first) + sum(f.second);
}
console.log(`forests: ${forestTotal}`);
// A cycle-capable class: releases of shared links make nodes candidates of
// the cycle collector, which are then torn down while still buffered, and
// the peer links close some rings, which the collector frees.
class Ring {
next: Ring | null = null;
peer: Ring | null = null;
id: number;
constructor(id: number) {
this.id = id;
}
}
let ringTotal = 0;
for (let generation = 0; generation < 3; generation++) {
let head: Ring | null = null;
const peers: Ring[] = [];
for (let i = 0; i < 2000; i++) {
const ring: Ring = new Ring(i);
ring.next = head;
if (i % 7 === 0) peers.push(ring);
head = ring;
}
for (let r = head; r !== null; r = r.next) if (r.id % 5 === 0) r.peer = peers[r.id % peers.length] ?? null;
for (const peer of peers) ringTotal += peer.id;
peers.length = 0;
for (let r = head; r !== null; r = r.next) ringTotal += r.peer === null ? 0 : r.peer.id;
}
console.log(`rings: ${ringTotal}`);
@@ -0,0 +1 @@
{ "plain": 5343, "san": 11960 }
+97
View File
@@ -0,0 +1,97 @@
/* Teardown segments (llvm/shapes.ts): a typed teardown of a type with
* children of its own type keeps the blocks it frees in a segment and frees
* the segment onto the free list when the chain's first teardown returns.
* Corpus program 7260 pins what such programs print; this suite checks the
* memory side natively, which Node parity cannot: a program that rebuilds
* large trees and lists for a hundred generations reuses every block of the
* last generation, so its resident set stays at about one generation's
* worth. A segment that lost blocks would grow it by a generation each time.
* The sanitized lane frees through the system allocator (no segments) and
* keeps freed memory in quarantine, so the bound is checked in the plain
* lane only. */
import { execFile } from "node:child_process";
import { mkdtemp, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { promisify } from "node:util";
import { expect, test } from "vitest";
import { compile } from "@scriptc/compiler";
const exec = promisify(execFile);
const sanitize = process.env["SCRIPTC_SAN"] === "1";
const PROGRAM = `
interface Node {
left: Node | null;
right: Node | null;
value: number;
}
function tree(depth: number): Node {
return depth === 0
? { left: null, right: null, value: 1 }
: { left: tree(depth - 1), right: tree(depth - 1), value: depth };
}
function size(node: Node | null): number {
return node === null ? 0 : 1 + size(node.left) + size(node.right);
}
class Branch {
readonly left: Branch | null;
readonly right: Branch | null;
constructor(left: Branch | null, right: Branch | null) {
this.left = left;
this.right = right;
}
}
function branches(depth: number): Branch {
return depth === 0 ? new Branch(null, null) : new Branch(branches(depth - 1), branches(depth - 1));
}
function count(branch: Branch | null): number {
return branch === null ? 0 : 1 + count(branch.left) + count(branch.right);
}
interface Cell {
next: Cell | null;
n: number;
}
function cells(n: number): Cell | null {
let head: Cell | null = null;
for (let i = 0; i < n; i++) head = { next: head, n: i };
return head;
}
let total = 0;
let settled = 0;
for (let generation = 0; generation < 100; generation++) {
total += size(tree(16)) + count(branches(15));
for (let c = cells(100000); c !== null; c = c.next) total += 1;
if (generation === 2) settled = process.memoryUsage.rss();
}
const growth = Math.round((process.memoryUsage.rss() - settled) / (1024 * 1024));
console.log(total, growth);
`;
test.skipIf(sanitize)(
"rebuilding large trees and lists reuses every block their teardowns free",
{ timeout: 300_000 },
async () => {
const dir = await mkdtemp(join(tmpdir(), "scriptc-segments-"));
try {
const source = join(dir, "churn.ts");
await writeFile(source, PROGRAM);
const result = await compile(source, {
backend: "llvm",
outDir: dir,
outPath: join(dir, "churn"),
});
expect(result.ok, JSON.stringify(result.diagnostics)).toBe(true);
if (!result.ok) return;
const { stdout } = await exec(result.binaryPath, [], { timeout: 120_000 });
const [total, growth] = stdout.trim().split(" ").map(Number);
// 100 generations of 131071 + 65535 nodes and 100000 cells.
expect(total).toBe(100 * (131071 + 65535 + 100000));
// One generation is about 10 MiB; a lost generation per round would be
// hundreds.
expect(growth).toBeLessThan(8);
} finally {
await rm(dir, { recursive: true, force: true });
}
},
);