From 989d0d6bc2d03ad2202c974143533b3d033c8bb6 Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Sun, 18 Oct 2026 09:12:14 -0700 Subject: [PATCH] Buffer nursery cycle candidates on the release fast paths Every release that left a headered object alive and unbuffered called scr_cyc_on_release, which looked up the type's bypass statistics, pushed the object, kept a running total and compared it with two triggers. The nursery case now has a fast path that appends the object to the nursery buffer itself, while the runtime-maintained window says the append can neither make a collection due nor grow the buffer, and the live count is below the allocation trigger: - scr_rc.h exports the candidate buffers (scr_cyc_buffers: the three buffers, the window and the allocation trigger) where the collector state is ordinary globals, and scr_cyc_append. The runtime's own release paths (scr_cyc_released) append inline, and scr_cyc_on_release tries the append first. - Optimized emitted code buffers through one sc_cyc_append helper per module, called with preserve_mostcc, so release fast paths keep the size of the plain call they had (inlining the append into every release fast path grew the self-hosted compiler's text by 4.6%) and their callers keep their registers. Its runtime call goes through a second preserve_mostcc function, which keeps the register saves off the append path. - scr_cycle.c keeps the window: recomputed where buffers, capacity or the trigger change in bulk, lowered by one on older buffering and nursery deaths, 0 during passes. The running total is gone; the slow path sums the buffers. - An appended candidate whose type bypasses the nursery would have been buffered past it. scr_cyc_route_bypassed moves such candidates before any scheduling decision or pass reads the buffers, consuming the bypass counts as scr_cyc_on_release does (skipped while no type bypasses). test_cycle checks the window (the release that reaches the trigger still collects) and the routing of inline-buffered candidates of a bypassed type; its mature-bypass probe routes before reading generations. Co-Authored-By: Claude Opus 5.5 --- packages/compiler/src/backend/llvm/alloc.ts | 91 ++++++++++ .../compiler/src/backend/llvm/shapes.test.ts | 37 ++++ packages/compiler/src/backend/llvm/shapes.ts | 57 +++++- packages/runtime/src/scr_cycle.c | 167 ++++++++++++++---- packages/runtime/src/scr_rc.h | 68 ++++++- packages/runtime/test/test_cycle.c | 74 +++++++- 6 files changed, 450 insertions(+), 44 deletions(-) diff --git a/packages/compiler/src/backend/llvm/alloc.ts b/packages/compiler/src/backend/llvm/alloc.ts index 7dfc385aa..72631ba1c 100644 --- a/packages/compiler/src/backend/llvm/alloc.ts +++ b/packages/compiler/src/backend/llvm/alloc.ts @@ -141,6 +141,97 @@ function declareGlobal(host: AllocHost, decl: string): void { if (host.threadLocalAlloc !== true) host.declare(decl); } +/** `scr_cyc_buffers` (scr_rc.h ScrCycBuffers): the three candidate buffers + * `{ v, n, cap, used }`, the inline buffering window and the allocation + * trigger. */ +const CYC_BUFFERS_TYPE = "{ [3 x { ptr, i64, i64, i64 }], i64, i64 }"; +/** Byte offsets in it (asserted in scr_rc.h). */ +const CYC_NURSERY_COUNT = 8; +const CYC_WINDOW = 96; +const CYC_ALLOC_TRIGGER = 104; + +/** Whether emitted release paths can append nursery candidates to the + * collector's buffer themselves (through {@link cycleAppendHelper}): where + * the runtime keeps that state in ordinary globals (executables without + * workers and classic libraries, on the 64-bit targets with inline + * allocation). */ +export function inlineBufferingSupported(host: AllocHost): boolean { + return host.inlineAlloc === true && host.threadLocalAlloc !== true && host.sizeType === "i64"; +} + +/** The emitted symbol of {@link cycleAppendHelper}. */ +export const CYC_APPEND = "sc_cyc_append"; + +/** The runtime symbols {@link cycleAppendHelper} uses. The helper is + * defined after the module's extern block, so its caller declares them. */ +export function cycleAppendDecls(): string[] { + return [ + `@scr_cyc_buffers = external global ${CYC_BUFFERS_TYPE}`, + `@scr_cyc_live = external global i64`, + `declare void @scr_cyc_on_release(ptr)`, + ]; +} + +/** The candidate-buffering step of every emitted release that leaves a + * headered object alive and unbuffered: scr_rc.h's scr_cyc_append, then + * scr_cyc_on_release when it declines. A nursery object goes onto the + * nursery buffer while the buffer holds fewer entries than the runtime's + * window and the live count is below the allocation trigger. + * + * One out-of-line definition per module keeps the release fast paths, which + * LLVM inlines into their callers, as small as the plain call they had. It + * uses preserve_mostcc, so the call clobbers almost no caller registers: + * the callers' code around a release stays what it was, and the append + * itself saves only the few registers it uses. Its slow path enters the + * runtime through a second preserve_mostcc function, which saves the rest. */ +export function cycleAppendHelper(): string[] { + return [ + `define internal preserve_mostcc void @${CYC_APPEND}(ptr %o) noinline #0 { ; scr_cyc_append, else scr_cyc_on_release`, + `entry:`, + ` %genp = getelementptr i8, ptr %o, i64 -${CYC_HDR - CYC_GEN}`, + ` %gen = load i8, ptr %genp ; ScrCycHdr.gen`, + ` %young = icmp eq i8 %gen, 0`, + ` br i1 %young, label %window, label %call`, + `window:`, + ` %np = getelementptr inbounds i8, ptr @scr_cyc_buffers, i64 ${CYC_NURSERY_COUNT}`, + ` %n = load i64, ptr %np ; nursery buffer entries`, + ` %wp = getelementptr inbounds i8, ptr @scr_cyc_buffers, i64 ${CYC_WINDOW}`, + ` %w = load i64, ptr %wp ; window`, + ` %room = icmp ult i64 %n, %w`, + ` br i1 %room, label %live, label %call`, + `live:`, + ` %lv = load i64, ptr @scr_cyc_live`, + ` %tp = getelementptr inbounds i8, ptr @scr_cyc_buffers, i64 ${CYC_ALLOC_TRIGGER}`, + ` %t = load i64, ptr %tp ; allocation trigger`, + ` %due = icmp uge i64 %lv, %t`, + ` br i1 %due, label %call, label %append`, + `append:`, + ` %v = load ptr, ptr @scr_cyc_buffers`, + ` %slot = getelementptr inbounds ptr, ptr %v, i64 %n`, + ` store ptr %o, ptr %slot`, + ` %n1 = add i64 %n, 1`, + ` store i64 %n1, ptr %np`, + ` %i = trunc i64 %n to i32`, + ` %b = or i32 %i, -2147483648 ; SCR_CYC_BUFFERED_BIT | position`, + ` %bufp = getelementptr i8, ptr %o, i64 -${CYC_HDR - CYC_BUF}`, + ` store i32 %b, ptr %bufp`, + ` ret void`, + `call:`, + ` tail call preserve_mostcc void @${CYC_APPEND}_call(ptr %o)`, + ` ret void`, + `}`, + // The runtime call clobbers every caller-saved register, which this + // preserve_mostcc function must restore: saving them here keeps those + // saves out of the append path above. + `define internal preserve_mostcc void @${CYC_APPEND}_call(ptr %o) noinline #0 {`, + `entry:`, + ` call void @scr_cyc_on_release(ptr %o) ; buffer the candidate; may collect`, + ` ret void`, + `}`, + ``, + ]; +} + /** A collector header to install: the object type's constant ScrCycType * descriptor (its trace and teardown; see cycTypeDescriptor). */ export interface CycHeader { diff --git a/packages/compiler/src/backend/llvm/shapes.test.ts b/packages/compiler/src/backend/llvm/shapes.test.ts index 8c2c46541..f25dca3a3 100644 --- a/packages/compiler/src/backend/llvm/shapes.test.ts +++ b/packages/compiler/src/backend/llvm/shapes.test.ts @@ -205,6 +205,43 @@ describe("LLVM runtime RC symbols", () => { expect(fn("sc_rc_retain_promise")).not.toContain("store i32"); }); + test("optimized builds buffer candidates through the module's append helper", () => { + const emit = (inlineRc: true | "release", extra: Partial) => { + const { host } = declarationHost(64, inlineRc); + const configured: ShapeHost = { ...host, inlineAlloc: true, ...extra }; + releaseSym(configured, { kind: "dyn" }); + return { + ir: emitInlineRcHelpers(configured).join("\n"), + decls: inlineRcDecls(configured), + }; + }; + for (const mode of [true, "release"] as const) { + const { ir, decls } = emit(mode, {}); + expect(ir).toContain("call preserve_mostcc void @sc_cyc_append(ptr %o)"); + expect(ir).toContain("define internal preserve_mostcc void @sc_cyc_append(ptr %o) noinline"); + // The young test reads the generation byte at obj-5; the append fills + // scr_cyc_buffers' nursery buffer and the header's position at obj-4. + expect(ir).toContain("getelementptr i8, ptr %o, i64 -5"); + expect(ir).toContain("getelementptr inbounds i8, ptr @scr_cyc_buffers, i64 96"); + expect(ir).toContain("getelementptr inbounds i8, ptr @scr_cyc_buffers, i64 104"); + expect(ir).toContain("load i64, ptr @scr_cyc_live"); + expect(ir).toContain("or i32 %i, -2147483648"); + expect(ir).toContain("call void @scr_cyc_on_release(ptr %o)"); + expect(decls).toContain( + "@scr_cyc_buffers = external global { [3 x { ptr, i64, i64, i64 }], i64, i64 }", + ); + expect(decls).toContain("@scr_cyc_live = external global i64"); + } + // Worker executables and targets without inline allocation call + // scr_cyc_on_release, which tries the same append first. + for (const extra of [{ threadLocalAlloc: true }, { inlineAlloc: false }]) { + const { ir, decls } = emit(true, extra); + expect(ir).not.toContain("sc_cyc_append"); + expect(ir).toContain("call void @scr_cyc_on_release(ptr %o)"); + expect(decls.join("\n")).not.toContain("scr_cyc_buffers"); + } + }); + test("uses the wasm32 header offsets", () => { const { host } = declarationHost(32, true); releaseSym(host, { kind: "union", unionId: "u0" } as never); diff --git a/packages/compiler/src/backend/llvm/shapes.ts b/packages/compiler/src/backend/llvm/shapes.ts index 23565eff4..b62e7d034 100644 --- a/packages/compiler/src/backend/llvm/shapes.ts +++ b/packages/compiler/src/backend/llvm/shapes.ts @@ -28,7 +28,15 @@ import { mangleRecordStruct, mangleRecordTrace, } from "../mangle.js"; -import { cycTypeDescriptor, emitObjectAlloc, emitObjectFree } from "./alloc.js"; +import { + cycTypeDescriptor, + emitObjectAlloc, + emitObjectFree, + CYC_APPEND, + cycleAppendDecls, + cycleAppendHelper, + inlineBufferingSupported, +} from "./alloc.js"; import type { BlockBuilder } from "./blocks.js"; import { SCR_ARR_FIELD, llvmCommentText } from "./common.js"; import type { NullableUnions } from "./nullable-unions.js"; @@ -308,6 +316,10 @@ export const INLINE_RC_FAMILIES: Readonly> = { type RcOp = "retain" | "release"; +/** The helper key (with the "family:op" keys) of the module's candidate + * append helper (alloc.ts cycleAppendHelper). */ +const CYC_APPEND_KEY = "cyc:append"; + /** The mirrored family for an RC operation on a runtime stem (`scr_str`), * or null when it stays a call. */ export function inlineRcFamily(host: ShapeHost, stem: string, op: RcOp): InlineRcFamily | null { @@ -369,6 +381,10 @@ function requestRcHelper(host: ShapeHost, family: string, op: RcOp): string { } host.rcHelpers.add(`${family}:${op}`); for (const decl of rcHelperDecls(family, op)) host.declare(decl); + // Its body is written last (emitInlineRcHelpers), after the requests are + // collected: a release that can buffer a candidate requests the append + // helper now. + if (op === "release" && INLINE_RC_FAMILIES[family]!.cycle === "always") requestCycleAppend(host); return `@${inlineRcName(family, op)}`; } @@ -466,6 +482,10 @@ function rcHelperKeys(host: ShapeHost): string[] { export function inlineRcDecls(host: ShapeHost): string[] { const decls = new Set(); for (const key of rcHelperKeys(host)) { + if (key === CYC_APPEND_KEY) { + for (const decl of cycleAppendDecls()) decls.add(decl); + continue; + } const [family, op] = key.split(":") as [string, RcOp]; for (const decl of rcHelperDecls(family, op)) decls.add(decl); } @@ -611,6 +631,10 @@ function releaseHelper(host: ShapeHost, family: string, row: InlineRcFamily): st export function emitInlineRcHelpers(host: ShapeHost): string[] { const out: string[] = []; for (const key of rcHelperKeys(host).sort()) { + if (key === CYC_APPEND_KEY) { + out.push(...cycleAppendHelper()); + continue; + } const [family, op] = key.split(":") as [string, RcOp]; const row = INLINE_RC_FAMILIES[family]!; out.push( @@ -1254,12 +1278,6 @@ export function releaseFastPath( ]; } -/** The possible-cycle-root step of a release that left a headered object - * alive: scr_cyc_on_release with its early return inlined. Test the sign - * of `buf` (the i32 at obj-4; negative for a queued candidate or a tenured - * object) and call the runtime only to enqueue a new candidate. - * Every emitted release of a headered object shares this sequence - * (scr_runtime.h asserts the offset); it ends by branching to `done`. */ /** One traced field of an emitted trace function: load the slot and visit * it unless it is null. Every collector visitor ignores null children, and * most traced slots of a typical heap are empty (optional links, unset @@ -1283,16 +1301,39 @@ export function traceFieldVisitLines( ]; } +/** Whether candidate buffering goes through the module's append helper + * (alloc.ts cycleAppendHelper), requesting its definition if so: in + * optimized builds (those with inline RC helpers) that support it. */ +function requestCycleAppend(host: ShapeHost): boolean { + if (host.rcHelpers === null || !inlineBufferingSupported(host)) return false; + if (!host.rcHelpers.has(CYC_APPEND_KEY)) { + host.rcHelpers.add(CYC_APPEND_KEY); + for (const decl of cycleAppendDecls()) host.declare(decl); + } + return true; +} + +/** The possible-cycle-root step of a release that left a headered object + * alive: scr_cyc_on_release with its early return inlined. Test the sign + * of `buf` (the i32 at obj-4; negative for a queued candidate or a tenured + * object), and buffer a new candidate: through the module's + * cycleAppendHelper in optimized builds that support it, through + * scr_cyc_on_release otherwise. Every emitted release of a headered object + * shares this sequence (scr_runtime.h asserts the offsets); it ends by + * branching to `done`. */ export function cycleRootLines(host: ShapeHost, done: string): string[] { const S = host.sizeType; host.declare(`declare void @scr_cyc_on_release(ptr)`); + const append = requestCycleAppend(host); return [ ` %bufp = getelementptr i8, ptr %o, ${S} -${host.cycleBufOffset}`, ` %buf = load i32, ptr %bufp`, ` %queued = icmp slt i32 %buf, 0`, ` br i1 %queued, label %${done}, label %enqueue`, `enqueue:`, - ` call void @scr_cyc_on_release(ptr %o) ; buffer the candidate; may collect`, + append + ? ` call preserve_mostcc void @${CYC_APPEND}(ptr %o) ; buffer the candidate; may collect` + : ` call void @scr_cyc_on_release(ptr %o) ; buffer the candidate; may collect`, ` br label %${done}`, ]; } diff --git a/packages/runtime/src/scr_cycle.c b/packages/runtime/src/scr_cycle.c index c2be13a03..974c2e66d 100644 --- a/packages/runtime/src/scr_cycle.c +++ b/packages/runtime/src/scr_cycle.c @@ -156,14 +156,11 @@ static void scr_cyc_oom(void) { * struct on purpose: buffering a candidate is the hot path, and splitting * them across three arrays indexed by generation would touch three cache * lines per release where the original single buffer touched one. */ -typedef struct { - void **v; - size_t n, cap; - /* The most entries a pass held, for the work vectors a pass fills - * (scr_cyc_push_work); scr_cyc_pass_done frees a large vector that the - * pass used little of. */ - size_t used; -} ScrVec; +/* Its layout is scr_rc.h's ScrCycBuffer (the candidate buffers are part of + * the inline buffering ABI). `used`: the most entries a pass held, for the + * work vectors a pass fills (scr_cyc_push_work); scr_cyc_pass_done frees a + * large vector that the pass used little of. */ +typedef ScrCycBuffer ScrVec; typedef struct { void *object; @@ -201,7 +198,6 @@ typedef struct { unsigned char growth_shift[SCR_CYC_NGENS]; unsigned char growth_streak[SCR_CYC_NGENS]; unsigned backlog_shift[SCR_CYC_NGENS]; - size_t nbuffered; size_t trigger; size_t alloc_trigger; size_t gray_referenced; @@ -235,7 +231,6 @@ static _Thread_local ScrCycThread scr_cyc_thread = {.gen_limit = SCR_CYC_OLD}; #define scr_cyc_growth_shift (scr_cyc_thread.growth_shift) #define scr_cyc_growth_streak (scr_cyc_thread.growth_streak) #define scr_cyc_backlog_shift (scr_cyc_thread.backlog_shift) -#define scr_cyc_nbuffered (scr_cyc_thread.nbuffered) #define scr_cyc_trigger (scr_cyc_thread.trigger) #define scr_cyc_alloc_trigger (scr_cyc_thread.alloc_trigger) #define scr_gray_referenced (scr_cyc_thread.gray_referenced) @@ -361,7 +356,16 @@ static inline void scr_cyc_note_used(ScrVec *vec) { /* ── the candidate-root buffers (one per generation) ──────────────────── */ #ifndef SCR_WORKERS -static SCR_TL ScrVec scr_roots[SCR_CYC_NGENS]; +/* Emitted and inline releases append nursery candidates to these buffers + * directly where the build allows it (scr_rc.h). Explicitly initialized so + * library localization can demote it (see scr_cyc_types). */ +#if SCR_CYC_INLINE_BUFFERING +ScrCycBuffers scr_cyc_buffers = {0}; +#else +static SCR_TL ScrCycBuffers scr_cyc_buffers; +#endif +#define scr_roots (scr_cyc_buffers.roots) +#define scr_cyc_alloc_trigger (scr_cyc_buffers.alloc_trigger) static SCR_TL bool scr_collecting = false; #endif @@ -409,6 +413,7 @@ static SCR_TL unsigned scr_gen_limit = SCR_CYC_OLD; #endif static size_t scr_cyc_pass(unsigned gen_limit); +static void scr_cyc_route_bypassed(void); /* A limited pass (scr_cyc_collect_set) takes only the candidates in this * set from the buffers up to `scr_cyc_take_gen`; NULL takes them all. */ static SCR_TL const ScrCycSet *scr_cyc_take = NULL; @@ -505,18 +510,18 @@ static size_t scr_cyc_nursery_threshold(void) { return cached; } -/* Buffered candidates across every generation, and the count at which the - * release path next collects. Keeping a running total (rather than summing - * the buffers) is what holds the hot path to one load and one compare, as - * it was when there was a single buffer. The trigger is re-armed at the end - * of every pass to "whatever survived, plus a nursery's worth": without that +/* The number of buffered candidates (across every generation) at which the + * release path next collects, and the live count at which it collects + * however few candidates there are. The trigger is re-armed at the end of + * every pass to "whatever survived, plus a nursery's worth": without that * hysteresis a large older buffer — which a nursery pass cannot drain — * would re-arm on every single release. Both start at zero, so the first - * release runs one trivial pass that arms them properly. */ + * release runs one trivial pass that arms them properly. The inline + * appends (scr_rc.h) compare against the window scr_cyc_window_update + * derives from these, so the slow path sums the buffers when it needs the + * total. */ #ifndef SCR_WORKERS -static SCR_TL size_t scr_cyc_nbuffered = 0; static SCR_TL size_t scr_cyc_trigger = 0; -static SCR_TL size_t scr_cyc_alloc_trigger = 0; #endif /* Gray objects whose count is above zero after the trial decrements so far * (see scr_mg_visit). markGray records the gray objects in scr_white while @@ -549,6 +554,34 @@ static size_t scr_cyc_buffered(void) { + scr_roots[SCR_CYC_OLD].n; } +/* The inline buffering window (scr_rc.h): how many entries the nursery + * buffer may hold before an append would bring the buffered total to the + * trigger or would need the buffer to grow. While a pass runs, every + * release takes the call. Recomputed wherever the buffers, the nursery + * buffer's capacity or the trigger change in bulk; the release and death + * paths adjust it by one (scr_cyc_window_shrink). */ +static inline void scr_cyc_window_update(void) { +#if SCR_CYC_INLINE_BUFFERING + size_t window = 0; + if (!scr_collecting) { + size_t older = scr_roots[SCR_CYC_MATURE].n + scr_roots[SCR_CYC_OLD].n; + if (scr_cyc_trigger > older + 1) window = scr_cyc_trigger - older - 1; + if (window > scr_roots[SCR_CYC_NURSERY].cap) window = scr_roots[SCR_CYC_NURSERY].cap; + } + scr_cyc_buffers.window = window; +#endif +} + +/* One step of the window's formula going down by one: an older candidate + * was buffered, or a nursery death lowered the trigger. A window below the + * formula only sends more releases through the call, so the clamp to the + * capacity can stay where it is. */ +static inline void scr_cyc_window_shrink(void) { +#if SCR_CYC_INLINE_BUFFERING + if (scr_cyc_buffers.window) scr_cyc_buffers.window--; +#endif +} + static bool scr_cyc_older_backlog(void) { return scr_roots[SCR_CYC_MATURE].n != 0 || scr_roots[SCR_CYC_OLD].n != 0; } @@ -682,6 +715,7 @@ static SCR_CYC_SCHEDULING void scr_cyc_scheduled_pass(unsigned gen, bool by_grow * per pass, so it is kept out of line and built for size, which leaves the * hot buffering path in scr_cyc_on_release small. */ static SCR_CYC_SCHEDULING void scr_cyc_collect_due(void) { + scr_cyc_route_bypassed(); unsigned gen = SCR_CYC_OLD; bool by_growth = scr_cyc_grown(SCR_CYC_OLD); if (!by_growth && scr_roots[SCR_CYC_OLD].n < scr_cyc_backlog_threshold(SCR_CYC_OLD)) { @@ -699,6 +733,7 @@ static SCR_CYC_SCHEDULING void scr_cyc_collect_due(void) { * per turn would walk the whole live heap every turn — which is exactly the * cost the generations exist to avoid. */ void scr_cyc_collect_scheduled(void) { + scr_cyc_route_bypassed(); if (scr_cyc_buffered() == 0) { scr_cyc_scheduled_mature_age = 0; return; @@ -778,6 +813,10 @@ typedef struct ScrCycKind { static SCR_TL ScrCycKind *scr_cyc_kinds; static SCR_TL size_t scr_cyc_kinds_cap; #endif +#if SCR_CYC_INLINE_BUFFERING +/* Types that bypass the nursery right now (scr_cyc_route_bypassed). */ +static size_t scr_cyc_nursery_bypassing = 0; +#endif /* The statistics of `kind` (nonzero), growing the table to cover it. */ static SCR_CYC_COMPACT ScrCycKind *scr_cyc_kind_grow(uint32_t kind) { @@ -802,7 +841,12 @@ static inline bool scr_cyc_bypassed(const ScrCycType *type, unsigned gen) { if (kind >= scr_cyc_kinds_cap) return false; ScrCycLevelStat *k = &scr_cyc_kinds[kind].level[gen]; if (k->bypass == 0) return false; - if (--k->bypass == 0) k->walked = k->freed = 0; + if (--k->bypass == 0) { + k->walked = k->freed = 0; +#if SCR_CYC_INLINE_BUFFERING + if (gen == SCR_CYC_NURSERY) scr_cyc_nursery_bypassing--; +#endif + } return true; } @@ -820,9 +864,16 @@ static inline void scr_cyc_walked(const ScrCycType *type, unsigned gen) { if (!type->kind) return; ScrCycLevelStat *k = &scr_cyc_kind(type->kind)->level[gen]; if (++k->walked == SCR_CYC_BYPASS_WINDOW) { +#if SCR_CYC_INLINE_BUFFERING + bool before = k->bypass != 0; +#endif k->bypass = k->freed * SCR_CYC_BYPASS_RATIO < SCR_CYC_BYPASS_WINDOW ? SCR_CYC_BYPASS_WINDOW * SCR_CYC_BYPASS_PROBE : 0; +#if SCR_CYC_INLINE_BUFFERING + if (gen == SCR_CYC_NURSERY && before != (k->bypass != 0)) + scr_cyc_nursery_bypassing += before ? (size_t)-1 : 1; +#endif k->walked = k->freed = 0; } } @@ -831,6 +882,40 @@ static inline void scr_cyc_freed(const ScrCycType *type, unsigned gen) { if (type->kind) scr_cyc_kind(type->kind)->level[gen].freed++; } +/* An inline append (scr_rc.h) leaves every nursery candidate in the nursery + * buffer, including one whose type bypasses the nursery, which + * scr_cyc_on_release would have buffered past it. Where a candidate waits + * matters only once something reads the buffers, so every scheduling + * decision and every pass first moves such candidates to the generation + * scr_cyc_on_release would have chosen, consuming the bypass counts as it + * would have, in buffer order. Candidates that scr_cyc_on_release buffered + * already passed the same test with the counts unchanged since (they only + * change during passes), so they stay. */ +static SCR_CYC_COMPACT void scr_cyc_route_bypassed(void) { +#if SCR_CYC_INLINE_BUFFERING + if (scr_cyc_nursery_bypassing == 0) return; + ScrVec *nursery = &scr_roots[SCR_CYC_NURSERY]; + for (size_t i = 0; i < nursery->n;) { + void *obj = nursery->v[i]; + ScrCycHdr *h = scr_cyc_hdr(obj); + unsigned gen = scr_cyc_buffer_gen(scr_cyc_type(h), SCR_CYC_NURSERY); + if (gen == SCR_CYC_NURSERY) { + i++; + continue; + } + void *last = nursery->v[--nursery->n]; + nursery->v[i] = last; + if (last != obj) scr_cyc_set_buffered(scr_cyc_hdr(last), (uint32_t)i); + scr_cyc_set_gen(h, gen); + ScrVec *b = &scr_roots[gen]; + if (b->n == b->cap) scr_cyc_grow(b); + scr_cyc_set_buffered(h, (uint32_t)b->n); + b->v[b->n++] = obj; + } + scr_cyc_window_update(); +#endif +} + void scr_cyc_on_dead(void *obj) { ScrCycHdr *h = scr_cyc_hdr(obj); if (!scr_cyc_is_buffered(h)) return; @@ -843,11 +928,12 @@ void scr_cyc_on_dead(void *obj) { b->v[i] = last; if (last != obj) scr_cyc_set_buffered(scr_cyc_hdr(last), (uint32_t)i); scr_cyc_clear_buffered(h); - scr_cyc_nbuffered--; /* A death must not consume room reserved for a future candidate. */ scr_cyc_trigger--; - if (scr_cyc_gen(h) != SCR_CYC_NURSERY && !scr_cyc_older_backlog()) - scr_cyc_scheduled_mature_age = 0; + /* An older death lowers the older count with the trigger: the window + * stays. A nursery death lowers only the trigger. */ + if (scr_cyc_gen(h) == SCR_CYC_NURSERY) scr_cyc_window_shrink(); + else if (!scr_cyc_older_backlog()) scr_cyc_scheduled_mature_age = 0; } /* Publication (scr_publish.c) takes buffered objects out of their candidate @@ -896,31 +982,39 @@ void scr_cyc_pub_fill(void) { } holes->n = 0; size_t closed = before - b->n; - scr_cyc_nbuffered -= closed; /* A death must not consume room reserved for a future candidate. */ scr_cyc_trigger -= closed; if (closed && gen != SCR_CYC_NURSERY) older = true; } if (older && !scr_cyc_older_backlog()) scr_cyc_scheduled_mature_age = 0; + scr_cyc_window_update(); } /* Every release that leaves an unbuffered object alive lands here (the - * emitted and inline release paths test `buffered` first), so it stays what - * it has always been: buffer inline, then one compare. */ + * emitted and inline release paths test `buffered` first) unless the window + * let it append a nursery candidate on its own fast path (scr_rc.h), so it + * stays what it has always been: the same append, then buffer and compare. */ void scr_cyc_on_release(void *obj) { ScrCycHdr *h = scr_cyc_hdr(obj); /* Releasing an existing candidate cannot advance the trigger. A pass * always re-arms after its teardowns, including any candidates they add. */ if (scr_cyc_is_buffered(h)) return; +#if SCR_CYC_INLINE_BUFFERING + if (scr_cyc_append(h, obj)) return; +#endif if (scr_cyc_gen(h) < SCR_CYC_OLD) scr_cyc_set_gen(h, scr_cyc_buffer_gen(scr_cyc_type(h), scr_cyc_gen(h))); - ScrVec *b = &scr_roots[scr_cyc_gen(h)]; - if (b->n == b->cap) scr_cyc_grow(b); + unsigned gen = scr_cyc_gen(h); + ScrVec *b = &scr_roots[gen]; + if (b->n == b->cap) { + scr_cyc_grow(b); + scr_cyc_window_update(); + } scr_cyc_set_buffered(h, (uint32_t)b->n); b->v[b->n++] = obj; - scr_cyc_nbuffered++; + if (gen != SCR_CYC_NURSERY) scr_cyc_window_shrink(); /* Never re-entered: a teardown's releases of untraced children can buffer * new candidates mid-collection, but they only wait for the next pass. */ - if (!scr_collecting && (scr_cyc_nbuffered >= scr_cyc_trigger || scr_cyc_live >= scr_cyc_alloc_trigger)) + if (!scr_collecting && (scr_cyc_buffered() >= scr_cyc_trigger || scr_cyc_live >= scr_cyc_alloc_trigger)) scr_cyc_collect_due(); } @@ -1272,10 +1366,10 @@ static SCR_CYC_COMPACT size_t scr_cyc_settle(unsigned gen_limit, bool doomed) { } /* markRoots drained buffers in bulk and teardowns moved the count around; * resync from the buffers themselves and re-arm. */ - scr_cyc_nbuffered = scr_cyc_buffered(); - scr_cyc_trigger = scr_cyc_nbuffered + scr_cyc_nursery_threshold(); + scr_cyc_trigger = scr_cyc_buffered() + scr_cyc_nursery_threshold(); scr_cyc_alloc_trigger = scr_cyc_live + SCR_CYC_NURSERY_ALLOCATIONS; scr_collecting = false; + scr_cyc_window_update(); return freed; } @@ -1315,7 +1409,9 @@ static size_t scr_cyc_pass_done(size_t freed) { static size_t scr_cyc_pass(unsigned gen_limit) { if (scr_collecting) return 0; + scr_cyc_route_bypassed(); scr_collecting = true; + scr_cyc_window_update(); scr_gen_limit = gen_limit; /* markRoots: every buffered entry is a candidate (retains do not mark @@ -1488,6 +1584,9 @@ static size_t scr_cyc_pass(unsigned gen_limit) { #ifdef SCR_CYC_TESTING /* Tests only: one pass at exactly `gen`, outside the schedule. */ size_t scr_cyc_pass_at(unsigned gen) { return scr_cyc_pass(gen); } +/* Tests only: route appended candidates of bypassing types, as the next + * scheduling decision or pass would. */ +void scr_cyc_route_bypassed_at(void) { scr_cyc_route_bypassed(); } #endif size_t scr_cyc_collect_set(const ScrCycSet *set, unsigned gen) { @@ -1526,4 +1625,8 @@ void scr_cyc_context_cleanup(void) { free(scr_cyc_kinds); scr_cyc_kinds = NULL; scr_cyc_kinds_cap = 0; +#if SCR_CYC_INLINE_BUFFERING + scr_cyc_nursery_bypassing = 0; +#endif + scr_cyc_window_update(); } diff --git a/packages/runtime/src/scr_rc.h b/packages/runtime/src/scr_rc.h index d5d83a72a..b4c61a6ff 100644 --- a/packages/runtime/src/scr_rc.h +++ b/packages/runtime/src/scr_rc.h @@ -463,12 +463,76 @@ uint32_t scr_cyc_pub_take(void *obj); void scr_cyc_pub_restore(void *obj, uint32_t index); void scr_cyc_pub_fill(void); +/* The candidate buffers, one per generation (scr_cycle.c), and the window + * through which a release buffers a nursery candidate on its fast path. A + * release that leaves an unbuffered nursery object alive appends it to the + * nursery buffer while that buffer holds fewer than `window` entries and the + * live count is below `alloc_trigger` (scr_cyc_append); anything else takes + * the full scr_cyc_on_release. scr_cycle.c keeps `window` at most at the + * number of nursery entries the buffers can hold before a collection falls + * due or the buffer must grow, and at 0 while a pass runs, so the append is + * exactly what scr_cyc_on_release would do with a nursery candidate. A + * candidate whose type bypasses the nursery is moved past it before the + * buffers are next read (scr_cycle.c, "level bypass"). + * + * The runtime's own release paths append inline (scr_cyc_released), and + * scr_cyc_on_release tries the append first. The layout is an ABI: + * optimized emitted code appends through a per-module helper + * (llvm/alloc.ts cycleAppendHelper) that addresses roots[0].v, roots[0].n, + * window and alloc_trigger at fixed offsets. Builds whose collector state + * is thread-local (worker executables, thread-instanced libraries) never + * append on a fast path. */ +typedef struct ScrCycBuffer { + void **v; + size_t n, cap; + /* The most entries a pass held, for the work vectors a pass fills + * (scr_cycle.c ScrVec); unused by the candidate buffers. */ + size_t used; +} ScrCycBuffer; +typedef struct ScrCycBuffers { + ScrCycBuffer roots[SCR_CYC_NGENS]; + size_t window; + size_t alloc_trigger; +} ScrCycBuffers; +#if !defined(SCR_WORKERS) && !(defined(SCR_LIB) && defined(SCR_THREAD_INSTANCES)) +#define SCR_CYC_INLINE_BUFFERING 1 +extern ScrCycBuffers scr_cyc_buffers; +#if UINTPTR_MAX == UINT64_MAX +_Static_assert(offsetof(ScrCycBuffers, roots) == 0 && offsetof(ScrCycBuffer, v) == 0 && + offsetof(ScrCycBuffer, n) == 8 && offsetof(ScrCycBuffers, window) == 96 && + offsetof(ScrCycBuffers, alloc_trigger) == 104, + "llvm/alloc.ts appends nursery candidates against this layout"); +#endif +/* Append the unbuffered object `obj` (header `h`) to the nursery buffer if + * it is a nursery object and the window allows; false leaves it to the rest + * of scr_cyc_on_release. */ +static inline bool scr_cyc_append(ScrCycHdr *h, void *obj) { + ScrCycBuffer *nursery = &scr_cyc_buffers.roots[SCR_CYC_NURSERY]; + size_t n = nursery->n; + if (scr_cyc_gen(h) != SCR_CYC_NURSERY || n >= scr_cyc_buffers.window || + scr_cyc_live >= scr_cyc_buffers.alloc_trigger) + return false; + nursery->v[n] = obj; + nursery->n = n + 1; + scr_cyc_set_buffered(h, (uint32_t)n); + return true; +} +#else +#define SCR_CYC_INLINE_BUFFERING 0 +#endif + /* Inline halves of the hooks for runtime release paths, mirroring the * emitted ones (llvm/shapes.ts): a release of an already-buffered - * object, and the death of an object that was never buffered, need no call. + * object, and the death of an object that was never buffered, need no call, + * and a nursery candidate is appended inline while the window allows. * Most releases of long-lived containers take the first early return. */ static inline void scr_cyc_released(void *obj) { - if (!scr_cyc_is_buffered(scr_cyc_hdr(obj))) scr_cyc_on_release(obj); + ScrCycHdr *h = scr_cyc_hdr(obj); + if (scr_cyc_is_buffered(h)) return; +#if SCR_CYC_INLINE_BUFFERING + if (scr_cyc_append(h, obj)) return; +#endif + scr_cyc_on_release(obj); } static inline void scr_cyc_dead(void *obj) { if (scr_cyc_is_buffered(scr_cyc_hdr(obj))) scr_cyc_on_dead(obj); diff --git a/packages/runtime/test/test_cycle.c b/packages/runtime/test/test_cycle.c index ba0bc0819..c50f0d5c5 100644 --- a/packages/runtime/test/test_cycle.c +++ b/packages/runtime/test/test_cycle.c @@ -712,8 +712,11 @@ static void check_deferred_white_restoration(void) { * once the type's objects die in nursery cycles again, the next probe window * finds that and it is walked again. Window, ratio and probe interval mirror * SCR_CYC_BYPASS_*. */ -/* Tests only (SCR_CYC_TESTING): one pass at exactly `gen`. */ +/* Tests only (SCR_CYC_TESTING): one pass at exactly `gen`, and the routing + * of appended candidates whose type bypasses a level, which the next + * scheduling decision or pass would do. */ size_t scr_cyc_pass_at(unsigned gen); +void scr_cyc_route_bypassed_at(void); static size_t bypass_traced; /* trace calls on bypass-kind nodes */ static size_t bypass_freed; /* bypass-kind nodes freed */ static void bypass_trace(void *obj, ScrTraceVisit visit, void *ctx) { @@ -785,6 +788,22 @@ static void check_nursery_bypass(void) { scr_collect_cycles(); check(bypass_freed - before == 2 * RINGS, "an older pass missed dead rings of a bypassed type"); + /* The same through the inline release path, which leaves the candidates + * in the nursery buffer: the next pass routes them past the nursery. */ + before = bypass_freed; + for (size_t r = 0; r < RINGS; r++) { + Node *a = make_bypass_node(), *b = make_bypass_node(); + a->next = b; + b->next = a; + a->rc++; + a->rc--; + scr_cyc_released(a); + } + run_release_passes(); + check(bypass_freed - before < 2 * RINGS, "inline-buffered bypassed rings were walked by the nursery"); + scr_collect_cycles(); + check(bypass_freed - before == 2 * RINGS, "an older pass missed inline-buffered rings"); + /* Now the type's objects die in nursery cycles. Once INTERVAL windows' * worth of candidates went past the nursery, a probe window walks them * there again, and its yield turns the bypass off. A nursery pass after @@ -832,13 +851,16 @@ static Node *make_mature_kind_node(void) { } /* How many of PROBE fresh, externally owned candidates a release buffers - * straight into the old generation. */ + * straight into the old generation, by the time anything reads the buffers + * (a release may append the candidate to the nursery buffer, which the next + * decision routes past the levels its type bypasses). */ static size_t probe_old_on_release(Node **owned, size_t probe) { scr_collect_cycles(); /* reset the growth baselines: no pass runs below */ size_t old = 0; for (size_t i = 0; i < probe; i++) { owned[i] = make_mature_kind_node(); read_live(owned[i]); + scr_cyc_route_bypassed_at(); old += scr_cyc_gen(scr_cyc_hdr(owned[i])) == SCR_CYC_OLD; } return old; @@ -1187,7 +1209,55 @@ static void check_large_old_buffer_with_nursery_candidates(void) { } } +/* Releases through scr_cyc_released append nursery candidates inline while + * the window allows, and fall back to scr_cyc_on_release exactly when a + * collection is due: the candidate that brings the buffered total to the + * trigger runs the pass, as it would through the call. */ +static void check_inline_buffering(void) { + enum { MAX = 4096 }; + static Node *nodes[MAX]; + size_t threshold = configured_nursery_threshold(); + if (threshold > MAX) return; + scr_collect_cycles(); + size_t before = traced; + for (size_t i = 0; i < threshold; i++) { + nodes[i] = make_ring(1); + nodes[i]->rc--; /* drop the self edge: a live leaf held by its owner */ + nodes[i]->next = NULL; + nodes[i]->rc++; + nodes[i]->rc--; + scr_cyc_released(nodes[i]); + ScrCycHdr *h = scr_cyc_hdr(nodes[i]); + if (i + 1 < threshold) { + check(scr_cyc_is_buffered(h) && scr_cyc_gen(h) == SCR_CYC_NURSERY, + "a nursery release did not buffer its candidate"); +#if SCR_CYC_INLINE_BUFFERING + check(scr_cyc_buffers.roots[SCR_CYC_NURSERY].v[scr_cyc_buf_index(h)] == nodes[i], + "an inline append is not where the header says"); +#endif + check(traced == before, "a pass ran before the trigger"); + } + } + /* The last release reached the trigger: one nursery pass walked them all + * and promoted the survivors, which wait in the mature buffer. */ + check(traced > before, "the release that reached the trigger did not collect"); + for (size_t i = 0; i < threshold; i++) { + ScrCycHdr *h = scr_cyc_hdr(nodes[i]); + check(scr_cyc_gen(h) == SCR_CYC_MATURE, "a nursery survivor was not promoted"); + check(nodes[i]->rc == 1, "a survivor lost its count"); + } + size_t freed_before = freed; + for (size_t i = 0; i < threshold; i++) { + nodes[i]->rc = 0; + scr_cyc_on_dead(nodes[i]); + node_free(nodes[i]); + } + check(freed - freed_before == threshold, "the leaves were not freed"); + scr_collect_cycles(); +} + int main(void) { + check_inline_buffering(); check_old_garbage_reclaimed_while_growing(); check_older_rings_into_old(); check_cycle_spanning_mature_and_old();