From 65a68470cf0e5dba6e51e473faebeaf9f0f2e7b0 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 22 Oct 2026 11:12:23 -0500 Subject: [PATCH] Free a typed teardown's blocks in the order it met them A typed teardown freed each block onto its free list as it went, and the free lists, which are LIFO, handed a torn-down structure's blocks to the next allocations in the reverse order. A structure that a loop rebuilds the same way each time was therefore laid out mirrored every other generation, and every walk over it, its teardown included, ran through memory backwards, which the hardware prefetchers handle three to four times worse. A program that rebuilds a tree of 2^21 - 1 object literals 12 times, summing each generation before dropping it, took 26 to 28 ns per node to build, 24 to sum and 16 to drop in those generations, against 6.8, 6.3 and 5.3 in the others. With this change, every generation runs at the second set of times, and the program takes 0.50 s instead of 1.18 s (Node: 0.91 s). The teardown of a class or record type with children of its own type now keeps the blocks it frees in a segment, linked through their first words and built in order through a tail pointer. A node releases its own block first, places it ahead of its children's when its address is the lower (object literals allocate before their fields) and behind them otherwise (constructor arguments build the children first), and the chain's first teardown frees the whole segment onto the free list in one step. A teardown past DIRECT_DESTROY_CHAIN appends through scr_rc_destroy_seg, which keeps scr_rc_destroy's depth budget and queue. Blocks outside the calling thread's arena are freed as before, and targets without the inline allocation paths keep the old teardown. --- packages/compiler/src/backend/llvm/alloc.ts | 77 +++++- .../compiler/src/backend/llvm/classes.test.ts | 59 +++++ packages/compiler/src/backend/llvm/classes.ts | 73 ++++-- packages/compiler/src/backend/llvm/shapes.ts | 242 ++++++++++++++++-- .../corpus/7260-teardown-segments.ts.json | 6 + packages/runtime/src/scr_cycle.c | 56 ++-- packages/runtime/src/scr_rc.h | 8 + scripts/ci-test-costs.json | 2 + tests/corpus/7260-teardown-segments.ts | 179 +++++++++++++ .../7260-teardown-segments.ts.json | 1 + tests/harness/teardown-segments.test.ts | 97 +++++++ 11 files changed, 732 insertions(+), 68 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7260-teardown-segments.ts.json create mode 100644 tests/corpus/7260-teardown-segments.ts create mode 100644 tests/harness/corpus-costs/7260-teardown-segments.ts.json create mode 100644 tests/harness/teardown-segments.test.ts diff --git a/packages/compiler/src/backend/llvm/alloc.ts b/packages/compiler/src/backend/llvm/alloc.ts index aa6345dab..2efeb96ac 100644 --- a/packages/compiler/src/backend/llvm/alloc.ts +++ b/packages/compiler/src/backend/llvm/alloc.ts @@ -459,18 +459,72 @@ export type FreeObservers = "weak" | "weak-or-source"; * type's disposals), and the block — `scr_cyc_free` for collector-headered * objects, `scr_rt_free` otherwise. Labels use the `sf.` prefix; the lines * end in an open block. + * + * With `keep` (an alloca of ptr holding NULL; inline allocation paths only), + * a block that would go onto the calling thread's free list is stored there + * instead, for a teardown segment (emitSegmentFree); a block the system or + * another arena owns is still freed. */ export function emitObjectFree( host: AllocHost, cyc: boolean, observers: FreeObservers = "weak", + keep: string | null = null, ): string[] { - const lines = threadLocalState(host, "sf", freeLines(host, cyc, observers)); + const lines = threadLocalState(host, "sf", freeLines(host, cyc, observers, keep)); if (host.objectAudit === false) return lines.filter((line) => line !== FREE_NOTE); host.declare(`declare void @scr_obj_free_note()`); return lines; } +/** Lines that put the blocks `first` to `last` (linked through their first + * words; one block when they are equal) of the class at reservation offset + * `off` on the calling thread's free list. */ +function freeListPushLines(prefix: string, off: string, first: string, last: string): string[] { + return [ + ` %${prefix}.shp = getelementptr inbounds i8, ptr @scr_sa, i64 ${SA_SHIFT}`, + ` %${prefix}.sh32 = load i32, ptr %${prefix}.shp`, + ` %${prefix}.sh = zext i32 %${prefix}.sh32 to i64`, + ` %${prefix}.c = lshr i64 ${off}, %${prefix}.sh`, + ` %${prefix}.c8 = shl i64 %${prefix}.c, 3`, + ` %${prefix}.fo = add i64 %${prefix}.c8, ${SA_FREE}`, + ` %${prefix}.fp = getelementptr inbounds i8, ptr @scr_sa, i64 %${prefix}.fo`, + ` %${prefix}.head = load ptr, ptr %${prefix}.fp`, + ` store ptr %${prefix}.head, ptr ${last}`, + ` store ptr ${first}, ptr %${prefix}.fp`, + ]; +} + +/** + * Lines that free a teardown segment: the blocks `first` to `last` (ptr + * values, both NULL for an empty segment), linked through their first words, + * which emitObjectFree kept. Every block of a segment is in the calling + * thread's arena and of one size class, so the segment goes onto that + * class's free list in one step, and the next allocations of the class take + * its blocks in segment order. Labels use `prefix`; the lines end in the + * open block `${prefix}.done`. + */ +export function emitSegmentFree( + host: AllocHost, + prefix: string, + first: string, + last: string, +): string[] { + declareState(host); + const lines = [ + ` %${prefix}.none = icmp eq ptr ${first}, null`, + ` br i1 %${prefix}.none, label %${prefix}.done, label %${prefix}.push`, + `${prefix}.push:`, + ` %${prefix}.base = load i64, ptr @scr_sa`, + ` %${prefix}.bi = ptrtoint ptr ${first} to i64`, + ` %${prefix}.off = sub i64 %${prefix}.bi, %${prefix}.base`, + ...freeListPushLines(prefix, `%${prefix}.off`, first, last), + ` br label %${prefix}.done`, + `${prefix}.done:`, + ]; + return threadLocalState(host, prefix, lines); +} + /** Lines that leave in `%.obs` whether the program can observe a * disposal of this type (folds to a constant). */ function observedLines(prefix: string, observers: FreeObservers): string[] { @@ -482,8 +536,14 @@ function observedLines(prefix: string, observers: FreeObservers): string[] { ]; } -function freeLines(host: AllocHost, cyc: boolean, observers: FreeObservers): string[] { +function freeLines( + host: AllocHost, + cyc: boolean, + observers: FreeObservers, + keep: string | null, +): string[] { if (host.inlineAlloc !== true) { + if (keep !== null) throw new Error("a kept block needs the inline allocation paths"); if (cyc) { host.declare(`declare void @scr_cyc_free(ptr)`); return [` call void @scr_obj_free_note()`, ` call void @scr_cyc_free(ptr %o)`]; @@ -548,16 +608,9 @@ function freeLines(host: AllocHost, cyc: boolean, observers: FreeObservers): str ` %sf.inx = call i1 @llvm.expect.i1(i1 %sf.in, i1 true)`, ` br i1 %sf.inx, label %sf.push, label %sf.sys`, `sf.push:`, - ` %sf.shp = getelementptr inbounds i8, ptr @scr_sa, i64 ${SA_SHIFT}`, - ` %sf.sh32 = load i32, ptr %sf.shp`, - ` %sf.sh = zext i32 %sf.sh32 to i64`, - ` %sf.c = lshr i64 %sf.off, %sf.sh`, - ` %sf.c8 = shl i64 %sf.c, 3`, - ` %sf.fo = add i64 %sf.c8, ${SA_FREE}`, - ` %sf.fp = getelementptr inbounds i8, ptr @scr_sa, i64 %sf.fo`, - ` %sf.head = load ptr, ptr %sf.fp`, - ` store ptr %sf.head, ptr ${blk}`, - ` store ptr ${blk}, ptr %sf.fp`, + ...(keep !== null + ? [` store ptr ${blk}, ptr ${keep}`] + : freeListPushLines("sf", "%sf.off", blk, blk)), ` br label %sf.done`, `sf.sys: ; outside the reservation, or the allocator is compiled out`, ` call void @scr_obj_free_note()`, diff --git a/packages/compiler/src/backend/llvm/classes.test.ts b/packages/compiler/src/backend/llvm/classes.test.ts index 2d95e942c..0c0b73aeb 100644 --- a/packages/compiler/src/backend/llvm/classes.test.ts +++ b/packages/compiler/src/backend/llvm/classes.test.ts @@ -68,6 +68,65 @@ test("leaf layouts keep direct teardown while recursive layouts share the depth } }); +test("teardowns of types with children of their own type free their blocks as one segment", () => { + const linked = { kind: "object" as const, className: "Linked" }; + const mod: IrModule = { + irVersion: 17, + sourceFile: loc.file, + entry: "main", + functions: [{ name: "main", params: [], returnType: VOID, locals: [], body: [], loc }], + classes: [ + { + name: "Linked", + fields: [ + { name: "next", type: linked }, + { name: "other", type: linked }, + ], + methods: [], + loc, + }, + ], + records: [{ id: "branch", fields: [{ name: "next", type: linked }] }], + }; + const define = (llvm: string, name: string): string => + new RegExp(`^define internal [^@]*@${name}\\([^]*?^}`, "m").exec(llvm)?.[0] ?? ""; + const linkedDestroy = `${mangleClassRelease("Linked")}_destroy`; + const branchDestroy = `${mangleRecordRelease("branch")}_destroy`; + for (const workers of [false, true]) { + const llvm = emitLlvmModule({ ...mod, workers }, { pointerBits: 64 }); + // Linked's teardown keeps its block and its children's in a segment: it + // releases its own block first and places it by address against its + // first child, a child torn down directly appends to the segment, one + // past the direct chain appends through the runtime, and the chain's + // first teardown frees the segment onto the free list in one step. + const seg = define(llvm, `${linkedDestroy}_s`); + expect(seg.startsWith("define internal ptr"), "segment form").toBe(true); + expect(seg).toMatch( + /store ptr %(?:sf\.blk|o), ptr %sg\.own[^]*call ptr @sc_release_Linked_destroy_s\(/, + ); + // The children's releases have a copy for each place of the own block. + for (const copy of ["ta", "tb"]) + expect(seg).toContain( + `call ptr @scr_rc_destroy_seg(ptr %v0, ptr @${linkedDestroy}_sc, ptr @${linkedDestroy}, ptr %${copy}0.bt)`, + ); + expect(seg).not.toContain("call void @scr_rc_destroy("); + expect(seg).toContain("icmp ult ptr %sg.o, %sg.r1"); + const first = define(llvm, `${linkedDestroy}_d`); + expect(first).toContain(`call ptr @${linkedDestroy}_s(ptr %o, i32 %d, ptr %head)`); + expect(first).toContain("store ptr %first, ptr %sgf.fp"); + expect(first.includes("llvm.threadlocal.address"), "thread-local state").toBe(workers); + expect(define(llvm, linkedDestroy)).toContain(`call void @${linkedDestroy}_d(ptr %o, i32 0)`); + // A record whose child is another type frees its own block as before. + expect(define(llvm, `${branchDestroy}_s`)).toBe(""); + expect(define(llvm, branchDestroy)).toContain(`call void @${mangleClassRelease("Linked")}(`); + } + // Without the inline allocation paths (32-bit targets) a teardown frees + // through the runtime, block by block. + const llvm32 = emitLlvmModule(mod, { pointerBits: 32 }); + expect(define(llvm32, `${linkedDestroy}_s`)).toBe(""); + expect(define(llvm32, `${linkedDestroy}_d`)).toContain(`call void @${linkedDestroy}_d(`); +}); + test("releases split into an inlinable fast path and an out-of-line remainder", () => { // An array of the class itself makes the hierarchy cycle-capable. const kids = { kind: "array" as const, elem: { kind: "object" as const, className: "Node" } }; diff --git a/packages/compiler/src/backend/llvm/classes.ts b/packages/compiler/src/backend/llvm/classes.ts index 3073bd1bd..3d67fbfd3 100644 --- a/packages/compiler/src/backend/llvm/classes.ts +++ b/packages/compiler/src/backend/llvm/classes.ts @@ -43,6 +43,9 @@ import { destroyDefinitions, directChildReleaseLines, directDestroyTarget, + segmentPrologueLines, + segmentTeardownLines, + segmentedTeardown, usedDirectTargets, type DirectDestroy, type RegionHooks, @@ -1078,25 +1081,62 @@ export function emitClassShapes( bounded && (directTargets.has(`object:${cls.name}`) || refFields.some((f) => directDestroyTarget(host, f.type, directTargets) !== null)); + // The teardown's own symbol, and whether it keeps a segment (shapes.ts + // segmentedTeardown). + const destroy = `${meta.hierarchy ? mangleClassReleaseDirect(cls.name) : mangleClassRelease(cls.name)}_destroy`; + const segmented = + chained && + !meta.compact && + runtimePrefix === null && + !isStreamRooted && + !meta.threadSlot && + segmentedTeardown( + host, + destroy, + refFields.map((f) => f.type), + directTargets, + ); // The field-releasing teardown body shared by both release shapes // (the public one on standalone classes, the DIRECT one on hierarchy // members). Runs at rc == 0. const teardown = (lines: string[]): void => { - refFields.forEach((f, t) => { - lines.push( - ` %f${t} = getelementptr inbounds %${struct}, ptr %o, i64 0, i32 ${f.index}`, - ` %v${t} = load ptr, ptr %f${t}`, - ); + const release = (prefix: string, f: (typeof refFields)[number], t: number): string[] => { const direct = bounded ? directDestroyTarget(host, f.type, directTargets) : null; - if (direct) - lines.push( - ...directChildReleaseLines(host, `td${t}`, `%v${t}`, direct, llvmCommentText(f.name)), - ); - else - lines.push( + if (direct === null) + return [ ` call void ${releaseSym(host, f.type)}(ptr %v${t}) ; ${llvmCommentText(f.name)}`, - ); - }); + ]; + const own = segmented && direct.destroy === destroy; + return directChildReleaseLines( + host, + `${prefix}${t}`, + `%v${t}`, + direct, + llvmCommentText(f.name), + own, + ); + }; + const loads = refFields.map((f, t) => [ + ` %f${t} = getelementptr inbounds %${struct}, ptr %o, i64 0, i32 ${f.index}`, + ` %v${t} = load ptr, ptr %f${t}`, + ]); + if (segmented) { + // A segmented teardown loads its fields, releases its own block, and + // then releases the fields (shapes.ts segmentTeardownLines). + const ownType = refFields.flatMap((f, t) => + directDestroyTarget(host, f.type, directTargets)?.destroy === destroy ? [`%v${t}`] : [], + ); + lines.push( + ...segmentPrologueLines(), + ...loads.flat(), + ...emitObjectFree(host, traced, "weak", "%sg.own"), + ...segmentTeardownLines(ownType, (prefix) => + refFields.flatMap((f, t) => release(prefix, f, t)), + ), + ); + return; + } + refFields.forEach((f, t) => lines.push(...loads[t]!, ...release("td", f, t))); if (runtimePrefix) lines.push(...regCall("td", runtimePrefix.drop, "")); if (isStreamRooted) lines.push(...stCall("tds", "scr_stream_st_release", "")); if (meta.threadSlot) lines.push(...slotForgetLines(host, struct, meta, "tdw")); @@ -1178,10 +1218,12 @@ export function emitClassShapes( ]; if (traced) reld.push(...cycleDeadLines(host, "teardown")); if (bounded) { - const destroy = `${mangleClassReleaseDirect(cls.name)}_destroy`; const destroyBody: string[] = []; teardown(destroyBody); - defs.push(...destroyDefinitions(destroy, destroyBody, chained), ``); + defs.push( + ...destroyDefinitions(destroy, destroyBody, chained, segmented ? host : null), + ``, + ); host.declare(`declare void @scr_rc_destroy(ptr, ptr)`); reld.push(` call void @scr_rc_destroy(ptr %o, ptr @${destroy})`); } else teardown(reld); @@ -1203,6 +1245,7 @@ export function emitClassShapes( bounded, lifetime, chained, + segmented, ), ``, ); diff --git a/packages/compiler/src/backend/llvm/shapes.ts b/packages/compiler/src/backend/llvm/shapes.ts index 6de92f667..ae217a76c 100644 --- a/packages/compiler/src/backend/llvm/shapes.ts +++ b/packages/compiler/src/backend/llvm/shapes.ts @@ -34,6 +34,7 @@ import { cycTypeDescriptor, emitObjectAlloc, emitObjectFree, + emitSegmentFree, CYC_APPEND, cycleAppendDecls, cycleAppendHelper, @@ -1340,17 +1341,23 @@ export interface DirectDestroy { /** The release of child `value` (a loaded field, possibly NULL) inside a * teardown whose chain length is `%d`: the child's own release for a shared * or immortal reference, its teardown for the last one. Labels use `tag`; - * the lines end in the open block `${tag}.next`. */ + * the lines end in the open block `${tag}.next`. + * + * With `segment` (the child has the teardown's own type, in a segmented + * teardown; segmentPrologueLines), the child's teardown appends its blocks + * to the teardown's segment. */ export function directChildReleaseLines( host: ShapeHost, tag: string, value: string, target: DirectDestroy, comment: string, + segment = false, ): string[] { const S = host.sizeType; const C = target.count ?? S; - host.declare(`declare void @scr_rc_destroy(ptr, ptr)`); + if (segment) host.declare(`declare ptr @scr_rc_destroy_seg(ptr, ptr, ptr, ptr)`); + else host.declare(`declare void @scr_rc_destroy(ptr, ptr)`); const lines = [ ` %${tag}.nn = icmp ne ptr ${value}, null`, ` br i1 %${tag}.nn, label %${tag}.chk, label %${tag}.next`, @@ -1382,16 +1389,139 @@ export function directChildReleaseLines( ` br i1 %${tag}.short, label %${tag}.direct, label %${tag}.bounded`, `${tag}.direct:`, ` %${tag}.d = add i32 %d, 1`, - ` call void @${target.destroy}_d(ptr ${value}, i32 %${tag}.d)`, + ); + if (!segment) { + lines.push( + ` call void @${target.destroy}_d(ptr ${value}, i32 %${tag}.d)`, + ` br label %${tag}.next`, + `${tag}.bounded:`, + ` call void @scr_rc_destroy(ptr ${value}, ptr @${target.destroy})`, + ` br label %${tag}.next`, + `${tag}.next:`, + ); + return lines; + } + lines.push( + ` %${tag}.t = load ptr, ptr %sg.t`, + ` %${tag}.dt = call ptr @${target.destroy}_s(ptr ${value}, i32 %${tag}.d, ptr %${tag}.t)`, + ` store ptr %${tag}.dt, ptr %sg.t`, ` br label %${tag}.next`, `${tag}.bounded:`, - ` call void @scr_rc_destroy(ptr ${value}, ptr @${target.destroy})`, + ` %${tag}.bt = load ptr, ptr %sg.t`, + ` %${tag}.rt = call ptr @scr_rc_destroy_seg(ptr ${value}, ptr @${target.destroy}_sc, ptr @${target.destroy}, ptr %${tag}.bt)`, + ` store ptr %${tag}.rt, ptr %sg.t`, ` br label %${tag}.next`, `${tag}.next:`, ); return lines; } +/* Teardown segments. A typed teardown that frees each block onto its free + * list as it goes frees a tree in walk order, and the free lists, which are + * LIFO, hand the blocks to the next allocations in the reverse order: a tree + * built again in the same order as the last (each generation of a structure + * a loop rebuilds) is laid out mirrored, and every walk over it, the next + * teardown included, runs through memory backwards, which defeats the + * hardware prefetchers. The runtime frees large dead sets of the cycle + * collector in address order for the same reason (scr_mem_order_for_free). + * + * So the teardown of a type with children of its own type (the `_s` form, + * segmentedTeardown) keeps the blocks it frees in a segment, linked through + * their first words, which it builds in order through a tail pointer: the + * `_s` form takes the segment's last link (the first word of its last block, + * or the head slot of an empty segment) and returns the new one. A node + * first releases its own block (emitObjectFree, keeping it), then appends it + * ahead of its children's blocks when its address is the lower, behind them + * otherwise (segmentTeardownLines), and the children append theirs in field + * order, so a structure + * built parent first (object literals allocate before their fields) or + * children first (constructor arguments) comes back in ascending order. The + * chain's first teardown (`_d`) frees the whole segment onto the class's + * free list in one step (emitSegmentFree), and the walks over the next + * generation run forward. A teardown past DIRECT_DESTROY_CHAIN appends + * through scr_rc_destroy_seg, under the same depth budget as + * scr_rc_destroy. Blocks outside the calling thread's arena are freed as + * before; one type's blocks are all of one class. */ + +/** Whether the teardown of a type whose reference fields have `fieldTypes` + * (with direct targets `targets`) and whose own teardown is `destroy` keeps + * a segment: a child field has its own type, and the inline allocation paths + * are in use. */ +export function segmentedTeardown( + host: ShapeHost, + destroy: string, + fieldTypes: readonly IrType[], + targets: ReadonlyMap, +): boolean { + if (host.inlineAlloc !== true) return false; + return fieldTypes.some((t) => directDestroyTarget(host, t, targets)?.destroy === destroy); +} + +/** The first lines of a segmented teardown, in its entry block: the + * segment's last link (`%sg.t`, from the `%tail` argument) and the kept own + * block (`%sg.own`, for emitObjectFree). */ +export function segmentPrologueLines(): string[] { + return [ + ` %sg.t = alloca ptr`, + ` %sg.own = alloca ptr`, + ` store ptr %tail, ptr %sg.t`, + ` store ptr null, ptr %sg.own`, + ]; +} + +/** The rest of a segmented teardown after emitObjectFree kept (or freed) + * its own block: the block goes ahead of its children's when its address is + * below the first non-NULL one of `children` (loaded fields of the + * teardown's own type, which stand for their blocks) or there is none, and + * behind them otherwise; then the children's releases (`releases`, given a + * label prefix), and the segment's last link is returned. Each order has its + * own copy of the releases, so neither keeps the decision, or the block once + * it is appended, across the children's teardowns. */ +export function segmentTeardownLines( + children: readonly string[], + releases: (prefix: string) => string[], +): string[] { + const lines = [` %sg.o = load ptr, ptr %sg.own`, ` %sg.kept = icmp ne ptr %sg.o, null`]; + let ref = "null"; + [...children].reverse().forEach((child, i) => { + lines.push( + ` %sg.nn${i} = icmp ne ptr ${child}, null`, + ` %sg.r${i} = select i1 %sg.nn${i}, ptr ${child}, ptr ${ref}`, + ); + ref = `%sg.r${i}`; + }); + lines.push( + ` %sg.leaf = icmp eq ptr ${ref}, null`, + ` %sg.lower = icmp ult ptr %sg.o, ${ref}`, + ` %sg.first = or i1 %sg.leaf, %sg.lower`, + ` %sg.ahead = and i1 %sg.kept, %sg.first`, + ` br i1 %sg.ahead, label %sg.front, label %sg.back`, + `sg.front:`, + ...segmentAppendLines("a"), + ...releases("ta"), + ` br label %sg.ret`, + `sg.back:`, + ...releases("tb"), + ` %sg.keep = icmp ne ptr %sg.o, null`, + ` br i1 %sg.keep, label %sg.last, label %sg.ret`, + `sg.last:`, + ...segmentAppendLines("l"), + ` br label %sg.ret`, + `sg.ret:`, + ` %sg.tr = load ptr, ptr %sg.t`, + ` ret ptr %sg.tr`, + ); + return lines; +} + +function segmentAppendLines(tag: string): string[] { + return [ + ` %sg.t${tag} = load ptr, ptr %sg.t`, + ` store ptr %sg.o, ptr %sg.t${tag}`, + ` store ptr %sg.o, ptr %sg.t`, + ]; +} + /** The direct-teardown target of a reference of `type`, if `targets` (keyed * `record:` / `object:`) has one. Nullable unions hold the arm's * pointer, NULL or an immortal sentinel, which the child release handles. */ @@ -1434,12 +1564,42 @@ export function usedDirectTargets( /** The teardown definitions for `destroy`: with `chained`, the body takes * the chain length `%d` (directChildReleaseLines) and the runtime's - * one-argument entry starts a chain. */ + * one-argument entry starts a chain. With `segment` (the host, for a + * segmented teardown), the body is a segment teardown's + * (segmentPrologueLines to segmentTeardownLines), defined as `_s`, with + * `_sc` for scr_rc_destroy_seg, and `_d` starts a segment and frees it. */ export function destroyDefinitions( destroy: string, body: readonly string[], chained: boolean, + segment: ShapeHost | null = null, ): string[] { + if (segment !== null) + return [ + `define internal ptr @${destroy}_s(ptr %o, i32 %d, ptr %tail) ${FN_ATTRS} {`, + `entry:`, + ...body, + `}`, + `define internal ptr @${destroy}_sc(ptr %o, ptr %tail) ${FN_ATTRS} {`, + `entry:`, + ` %t = call ptr @${destroy}_s(ptr %o, i32 0, ptr %tail)`, + ` ret ptr %t`, + `}`, + `define internal void @${destroy}_d(ptr %o, i32 %d) ${FN_ATTRS} {`, + `entry:`, + ` %head = alloca ptr`, + ` store ptr null, ptr %head`, + ` %last = call ptr @${destroy}_s(ptr %o, i32 %d, ptr %head)`, + ` %first = load ptr, ptr %head`, + ...emitSegmentFree(segment, "sgf", "%first", "%last"), + ` ret void`, + `}`, + `define internal void @${destroy}(ptr %o) ${FN_ATTRS} {`, + `entry:`, + ` call void @${destroy}_d(ptr %o, i32 0)`, + ` ret void`, + `}`, + ]; if (!chained) return [ `define internal void @${destroy}(ptr %o) ${FN_ATTRS} {`, @@ -1478,12 +1638,13 @@ export function releaseBody( bounded = true, immortal = false, chained = false, + segmented = false, ): string[] { const S = host.sizeType; if (bounded) host.declare(`declare void @scr_rc_destroy(ptr, ptr)`); const destroy = `${fnName}_destroy`; const lines = [ - ...(bounded ? destroyDefinitions(destroy, freeBody, chained) : []), + ...(bounded ? destroyDefinitions(destroy, freeBody, chained, segmented ? host : null) : []), ...releaseFastPath(host, fnName, `${fnName}_slow`, traced, comment, immortal), `define internal void @${fnName}_slow(ptr %o) noinline ${FN_ATTRS} {${comment ? ` ; ${comment}` : ""}`, `entry:`, @@ -1715,26 +1876,58 @@ export function emitRecordShapes( const chained = directTargets.has(`record:${shape.id}`) || refMembers.some((m) => directDestroyTarget(host, m.type, directTargets) !== null); - for (const m of refMembers) { - freeBody.push( - ` %f${t} = getelementptr inbounds %${struct}, ptr %o, i64 0, i32 ${m.index}`, - ` %v${t} = load ptr, ptr %f${t}`, + const destroy = `${mangleRecordRelease(shape.id)}_destroy`; + const bounded = refMembers.some((member) => needsBoundedRelease(member.type)); + const segmented = + bounded && + segmentedTeardown( + host, + destroy, + refMembers.map((m) => m.type), + directTargets, ); - const direct = directDestroyTarget(host, m.type, directTargets); - if (direct) { - freeBody.push( - ...directChildReleaseLines(host, `td${t}`, `%v${t}`, direct, llvmCommentText(m.name)), - ); - } else { - freeBody.push( - ` call void ${releaseSym(host, m.type)}(ptr %v${t}) ; ${llvmCommentText(m.name)}`, - ); - } - t++; - } // Closed layouts can carry a kept record source (record-sources.ts). const observers = recordMayKeepSource(shape) ? "weak-or-source" : "weak"; - freeBody.push(...emitObjectFree(host, traced, observers)); + // A segmented teardown loads its fields, releases its own block, and + // then releases the fields (segmentTeardownLines). + const loads: string[] = []; + const ownType: string[] = []; + const release = (prefix: string, m: (typeof refMembers)[number], i: number): string[] => { + const direct = directDestroyTarget(host, m.type, directTargets); + if (direct === null) + return [` call void ${releaseSym(host, m.type)}(ptr %v${i}) ; ${llvmCommentText(m.name)}`]; + const own = segmented && direct.destroy === destroy; + return directChildReleaseLines( + host, + `${prefix}${i}`, + `%v${i}`, + direct, + llvmCommentText(m.name), + own, + ); + }; + for (const m of refMembers) { + const fieldLines = [ + ` %f${t} = getelementptr inbounds %${struct}, ptr %o, i64 0, i32 ${m.index}`, + ` %v${t} = load ptr, ptr %f${t}`, + ]; + if (segmented) { + loads.push(...fieldLines); + if (directDestroyTarget(host, m.type, directTargets)?.destroy === destroy) + ownType.push(`%v${t}`); + } else freeBody.push(...fieldLines, ...release("td", m, t)); + t++; + } + if (segmented) + freeBody.push( + ...segmentPrologueLines(), + ...loads, + ...emitObjectFree(host, traced, observers, "%sg.own"), + ...segmentTeardownLines(ownType, (prefix) => + refMembers.flatMap((m, i) => release(prefix, m, i)), + ), + ); + else freeBody.push(...emitObjectFree(host, traced, observers)); defs.push( ...releaseBody( host, @@ -1742,9 +1935,10 @@ export function emitRecordShapes( traced, freeBody, "", - refMembers.some((member) => needsBoundedRelease(member.type)), + bounded, false, chained, + segmented, ), ``, ); diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7260-teardown-segments.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7260-teardown-segments.ts.json new file mode 100644 index 000000000..a7eac099f --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7260-teardown-segments.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7260-teardown-segments.ts" + ], + "diags": [] +} diff --git a/packages/runtime/src/scr_cycle.c b/packages/runtime/src/scr_cycle.c index 5e5029e14..1bd0ff7ac 100644 --- a/packages/runtime/src/scr_cycle.c +++ b/packages/runtime/src/scr_cycle.c @@ -293,24 +293,22 @@ static SCR_TL ScrDestroyEntry *scr_destroy_pending; static SCR_TL size_t scr_destroy_count, scr_destroy_capacity; #endif -void scr_rc_destroy(void *obj, void (*destroy)(void *)) { - if (scr_destroy_depth == SCR_DESTROY_DEPTH) { - if (scr_destroy_count == scr_destroy_capacity) { - size_t capacity = scr_destroy_capacity ? scr_destroy_capacity * 2 : 64; - if (capacity < scr_destroy_capacity || capacity > SIZE_MAX / sizeof(ScrDestroyEntry)) - scr_cyc_oom(); - ScrDestroyEntry *entries = realloc(scr_destroy_pending, capacity * sizeof(*entries)); - if (!entries) scr_cyc_oom(); - scr_destroy_pending = entries; - scr_destroy_capacity = capacity; - } - scr_destroy_pending[scr_destroy_count++] = (ScrDestroyEntry){ obj, destroy }; - return; +/* Queues a disposal past the depth budget; the outermost disposal runs it. */ +static void scr_rc_destroy_defer(void *obj, void (*destroy)(void *)) { + if (scr_destroy_count == scr_destroy_capacity) { + size_t capacity = scr_destroy_capacity ? scr_destroy_capacity * 2 : 64; + if (capacity < scr_destroy_capacity || capacity > SIZE_MAX / sizeof(ScrDestroyEntry)) + scr_cyc_oom(); + ScrDestroyEntry *entries = realloc(scr_destroy_pending, capacity * sizeof(*entries)); + if (!entries) scr_cyc_oom(); + scr_destroy_pending = entries; + scr_destroy_capacity = capacity; } - scr_destroy_depth++; - destroy(obj); - scr_destroy_depth--; - if (scr_destroy_depth) return; + scr_destroy_pending[scr_destroy_count++] = (ScrDestroyEntry){ obj, destroy }; +} + +/* Runs the queued disposals once the outermost one has returned. */ +static void scr_rc_destroy_drain(void) { while (scr_destroy_count) { ScrDestroyEntry entry = scr_destroy_pending[--scr_destroy_count]; scr_destroy_depth = 1; @@ -326,6 +324,30 @@ void scr_rc_destroy(void *obj, void (*destroy)(void *)) { } } +void scr_rc_destroy(void *obj, void (*destroy)(void *)) { + if (scr_destroy_depth == SCR_DESTROY_DEPTH) { + scr_rc_destroy_defer(obj, destroy); + return; + } + scr_destroy_depth++; + destroy(obj); + scr_destroy_depth--; + if (scr_destroy_depth == 0) scr_rc_destroy_drain(); +} + +void *scr_rc_destroy_seg(void *obj, void *(*destroy_seg)(void *, void *), + void (*destroy)(void *), void *tail) { + if (scr_destroy_depth == SCR_DESTROY_DEPTH) { + scr_rc_destroy_defer(obj, destroy); + return tail; + } + scr_destroy_depth++; + tail = destroy_seg(obj, tail); + scr_destroy_depth--; + if (scr_destroy_depth == 0) scr_rc_destroy_drain(); + return tail; +} + /* Live cycle-headered objects and the running count of OLD objects freed. * The inline allocation/free paths (scr_runtime.h, llvm/alloc.ts) maintain * both counters so full passes can measure old-generation reclamation. */ diff --git a/packages/runtime/src/scr_rc.h b/packages/runtime/src/scr_rc.h index 7a921b451..83d0589c1 100644 --- a/packages/runtime/src/scr_rc.h +++ b/packages/runtime/src/scr_rc.h @@ -552,6 +552,14 @@ _Static_assert(sizeof(((ScrCycHdr *)0)->gen) == 1 && SCR_CYC_OLD == 2, * depth; deeper objects wait until the outer disposal drains the worklist. * The callback owns the object and must release its children and free it. */ void scr_rc_destroy(void *obj, void (*destroy)(void *)); +/* scr_rc_destroy for an emitted teardown that keeps the blocks it frees in + * a segment, linked through their first words, instead of freeing them onto + * the free list (llvm/shapes.ts): destroy_seg tears obj down, appending its + * blocks after the segment's last link `tail`, and returns the new last + * link, which this returns. A disposal past the depth budget is queued with + * `destroy`, which frees its own blocks, and leaves `tail` as it was. */ +void *scr_rc_destroy_seg(void *obj, void *(*destroy_seg)(void *, void *), + void (*destroy)(void *), void *tail); /* RC hooks for cycle-headered types. on_release: rc was decremented and * stayed above zero — buffer the object as a possible cycle root (may run diff --git a/scripts/ci-test-costs.json b/scripts/ci-test-costs.json index 4c4964579..99d47e972 100644 --- a/scripts/ci-test-costs.json +++ b/scripts/ci-test-costs.json @@ -409,6 +409,7 @@ "tests/harness/stdlib-surface.test.ts": 48, "tests/harness/strip-vt-linear.test.ts": 1795, "tests/harness/surface-manifest.test.ts": 5309, + "tests/harness/teardown-segments.test.ts": 2600, "tests/harness/terminal-geometry.test.ts": 2730, "tests/harness/test262-runner.test.ts": 207, "tests/harness/three-library.test.ts": 251122, @@ -828,6 +829,7 @@ "tests/harness/stdlib-surface.test.ts": 46, "tests/harness/strip-vt-linear.test.ts": 2546, "tests/harness/surface-manifest.test.ts": 5951, + "tests/harness/teardown-segments.test.ts": 5, "tests/harness/terminal-geometry.test.ts": 3270, "tests/harness/test262-runner.test.ts": 230, "tests/harness/three-library.test.ts": 361470, diff --git a/tests/corpus/7260-teardown-segments.ts b/tests/corpus/7260-teardown-segments.ts new file mode 100644 index 000000000..cbd2b144d --- /dev/null +++ b/tests/corpus/7260-teardown-segments.ts @@ -0,0 +1,179 @@ +// Teardowns that hand their blocks back in order. +// +// A typed teardown of a type with children of its own type keeps the blocks +// it frees in one segment and frees the segment when the chain's first +// teardown returns, so the next structure built of that type reuses them in +// order. This program rebuilds structures of such types for many +// generations and reads every value of each generation before dropping it: +// a block handed out twice, or one freed while still in use, shows up as a +// wrong checksum. The shapes cover the ways a segment is put together: +// record literals (allocated before their fields) and constructors whose +// arguments build the children first, lists built by prepending and by +// appending, trees far deeper than one direct teardown chain and lists +// longer than the runtime's whole depth budget (whose tails wait in its +// queue), subtrees still owned elsewhere when their parent goes, records of +// one type holding records of another, and nodes of a cycle-capable class +// that are still candidates of the cycle collector when the last reference +// goes, and rings of them that only the collector frees. + +interface Node { + left: Node | null; + right: Node | null; + value: number; +} + +function literalTree(depth: number, value: number): Node { + return depth === 0 + ? { left: null, right: null, value } + : { left: literalTree(depth - 1, value * 2), right: literalTree(depth - 1, value * 2 + 1), value }; +} + +function sum(node: Node | null): number { + return node === null ? 0 : node.value + sum(node.left) + sum(node.right); +} + +class Branch { + readonly left: Branch | null; + readonly right: Branch | null; + readonly label: string; + constructor(left: Branch | null, right: Branch | null, label: string) { + this.left = left; + this.right = right; + this.label = label; + } +} + +function classTree(depth: number, label: string): Branch { + return depth === 0 + ? new Branch(null, null, label) + : new Branch(classTree(depth - 1, label + "l"), classTree(depth - 1, label + "r"), label); +} + +function labels(branch: Branch | null): number { + return branch === null ? 0 : branch.label.length + labels(branch.left) + labels(branch.right); +} + +interface Cell { + next: Cell | null; + n: number; +} + +function prepended(count: number, base: number): Cell | null { + let head: Cell | null = null; + for (let i = 0; i < count; i++) head = { next: head, n: base + i }; + return head; +} + +function appended(count: number, base: number): Cell | null { + if (count === 0) return null; + const head: Cell = { next: null, n: base }; + let tail = head; + for (let i = 1; i < count; i++) { + const cell: Cell = { next: null, n: base + i }; + tail.next = cell; + tail = cell; + } + return head; +} + +function cells(head: Cell | null): [number, number] { + let count = 0; + let total = 0; + for (let cell = head; cell !== null; cell = cell.next) { + count++; + total += cell.n; + } + return [count, total]; +} + +// Generations of literal trees: shallow, one chain deep, and several chains +// deep. Each generation is checked against the sum the shape must have. +for (const depth of [0, 1, 5, 16, 17, 18]) { + let ok = true; + for (let generation = 0; generation < 6; generation++) { + const tree = literalTree(depth, 1); + const expected = ((2 ** (depth + 1)) * (2 ** (depth + 1) - 1)) / 2; + if (sum(tree) !== expected) ok = false; + } + console.log(`literal trees of depth ${depth}: ${ok}`); +} + +// Generations of class trees built children first. +for (const depth of [3, 12, 17]) { + let total = 0; + for (let generation = 0; generation < 4; generation++) total += labels(classTree(depth, "t")); + console.log(`class trees of depth ${depth}: ${total}`); +} + +// Lists both ways, alternating, so each reuses the other's blocks; the long +// ones outlast the runtime's depth budget. +for (const count of [1, 15, 16, 17, 1000, 3000, 40000]) { + const seen: string[] = []; + for (let generation = 0; generation < 4; generation++) { + const list = generation % 2 === 0 ? prepended(count, generation) : appended(count, generation); + const [length, total] = cells(list); + seen.push(`${length}/${total}`); + } + console.log(`lists of ${count}: ${seen.join(" ")}`); +} + +// Subtrees owned elsewhere survive their parent's teardown, and the blocks +// freed around them are not handed out while they live. +const kept: Node[] = []; +for (let generation = 0; generation < 5; generation++) { + const tree = literalTree(10, generation + 1); + if (tree.left !== null && tree.left.right !== null) kept.push(tree.left.right); + if (tree.right !== null) kept.push(tree.right); + // Building the next tree reuses the dropped blocks. + literalTree(10, 7); +} +console.log(`kept subtrees: ${kept.length} ${kept.map(sum).join(",")}`); + +// A record of one type holding trees of another: the outer teardown frees +// its own blocks as before and tears each inner tree down as a segment. +interface Forest { + first: Node; + second: Node | null; + rest: Forest | null; +} + +function forest(count: number): Forest | null { + let out: Forest | null = null; + for (let i = 0; i < count; i++) out = { first: literalTree(4, i), second: i % 3 === 0 ? null : literalTree(2, i), rest: out }; + return out; +} + +let forestTotal = 0; +for (let generation = 0; generation < 3; generation++) { + for (let f = forest(500); f !== null; f = f.rest) forestTotal += sum(f.first) + sum(f.second); +} +console.log(`forests: ${forestTotal}`); + +// A cycle-capable class: releases of shared links make nodes candidates of +// the cycle collector, which are then torn down while still buffered, and +// the peer links close some rings, which the collector frees. +class Ring { + next: Ring | null = null; + peer: Ring | null = null; + id: number; + constructor(id: number) { + this.id = id; + } +} + +let ringTotal = 0; +for (let generation = 0; generation < 3; generation++) { + let head: Ring | null = null; + const peers: Ring[] = []; + for (let i = 0; i < 2000; i++) { + const ring: Ring = new Ring(i); + ring.next = head; + if (i % 7 === 0) peers.push(ring); + head = ring; + } + for (let r = head; r !== null; r = r.next) if (r.id % 5 === 0) r.peer = peers[r.id % peers.length] ?? null; + for (const peer of peers) ringTotal += peer.id; + peers.length = 0; + for (let r = head; r !== null; r = r.next) ringTotal += r.peer === null ? 0 : r.peer.id; +} +console.log(`rings: ${ringTotal}`); diff --git a/tests/harness/corpus-costs/7260-teardown-segments.ts.json b/tests/harness/corpus-costs/7260-teardown-segments.ts.json new file mode 100644 index 000000000..7f7ce1c2b --- /dev/null +++ b/tests/harness/corpus-costs/7260-teardown-segments.ts.json @@ -0,0 +1 @@ +{ "plain": 5343, "san": 11960 } diff --git a/tests/harness/teardown-segments.test.ts b/tests/harness/teardown-segments.test.ts new file mode 100644 index 000000000..95a2d2795 --- /dev/null +++ b/tests/harness/teardown-segments.test.ts @@ -0,0 +1,97 @@ +/* Teardown segments (llvm/shapes.ts): a typed teardown of a type with + * children of its own type keeps the blocks it frees in a segment and frees + * the segment onto the free list when the chain's first teardown returns. + * Corpus program 7260 pins what such programs print; this suite checks the + * memory side natively, which Node parity cannot: a program that rebuilds + * large trees and lists for a hundred generations reuses every block of the + * last generation, so its resident set stays at about one generation's + * worth. A segment that lost blocks would grow it by a generation each time. + * The sanitized lane frees through the system allocator (no segments) and + * keeps freed memory in quarantine, so the bound is checked in the plain + * lane only. */ +import { execFile } from "node:child_process"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { promisify } from "node:util"; +import { expect, test } from "vitest"; +import { compile } from "@scriptc/compiler"; + +const exec = promisify(execFile); +const sanitize = process.env["SCRIPTC_SAN"] === "1"; + +const PROGRAM = ` +interface Node { + left: Node | null; + right: Node | null; + value: number; +} +function tree(depth: number): Node { + return depth === 0 + ? { left: null, right: null, value: 1 } + : { left: tree(depth - 1), right: tree(depth - 1), value: depth }; +} +function size(node: Node | null): number { + return node === null ? 0 : 1 + size(node.left) + size(node.right); +} +class Branch { + readonly left: Branch | null; + readonly right: Branch | null; + constructor(left: Branch | null, right: Branch | null) { + this.left = left; + this.right = right; + } +} +function branches(depth: number): Branch { + return depth === 0 ? new Branch(null, null) : new Branch(branches(depth - 1), branches(depth - 1)); +} +function count(branch: Branch | null): number { + return branch === null ? 0 : 1 + count(branch.left) + count(branch.right); +} +interface Cell { + next: Cell | null; + n: number; +} +function cells(n: number): Cell | null { + let head: Cell | null = null; + for (let i = 0; i < n; i++) head = { next: head, n: i }; + return head; +} +let total = 0; +let settled = 0; +for (let generation = 0; generation < 100; generation++) { + total += size(tree(16)) + count(branches(15)); + for (let c = cells(100000); c !== null; c = c.next) total += 1; + if (generation === 2) settled = process.memoryUsage.rss(); +} +const growth = Math.round((process.memoryUsage.rss() - settled) / (1024 * 1024)); +console.log(total, growth); +`; + +test.skipIf(sanitize)( + "rebuilding large trees and lists reuses every block their teardowns free", + { timeout: 300_000 }, + async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-segments-")); + try { + const source = join(dir, "churn.ts"); + await writeFile(source, PROGRAM); + const result = await compile(source, { + backend: "llvm", + outDir: dir, + outPath: join(dir, "churn"), + }); + expect(result.ok, JSON.stringify(result.diagnostics)).toBe(true); + if (!result.ok) return; + const { stdout } = await exec(result.binaryPath, [], { timeout: 120_000 }); + const [total, growth] = stdout.trim().split(" ").map(Number); + // 100 generations of 131071 + 65535 nodes and 100000 cells. + expect(total).toBe(100 * (131071 + 65535 + 100000)); + // One generation is about 10 MiB; a lost generation per round would be + // hundreds. + expect(growth).toBeLessThan(8); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }, +);