Buffer nursery cycle candidates on the release fast paths

Every release that left a headered object alive and unbuffered called
scr_cyc_on_release, which looked up the type's bypass statistics, pushed
the object, kept a running total and compared it with two triggers. The
nursery case now has a fast path that appends the object to the nursery
buffer itself, while the runtime-maintained window says the append can
neither make a collection due nor grow the buffer, and the live count is
below the allocation trigger:

- scr_rc.h exports the candidate buffers (scr_cyc_buffers: the three
  buffers, the window and the allocation trigger) where the collector
  state is ordinary globals, and scr_cyc_append. The runtime's own
  release paths (scr_cyc_released) append inline, and scr_cyc_on_release
  tries the append first.
- Optimized emitted code buffers through one sc_cyc_append helper per
  module, called with preserve_mostcc, so release fast paths keep the
  size of the plain call they had (inlining the append into every
  release fast path grew the self-hosted compiler's text by 4.6%) and
  their callers keep their registers. Its runtime call goes through a
  second preserve_mostcc function, which keeps the register saves off the
  append path.
- scr_cycle.c keeps the window: recomputed where buffers, capacity or the
  trigger change in bulk, lowered by one on older buffering and nursery
  deaths, 0 during passes. The running total is gone; the slow path sums
  the buffers.
- An appended candidate whose type bypasses the nursery would have been
  buffered past it. scr_cyc_route_bypassed moves such candidates before
  any scheduling decision or pass reads the buffers, consuming the bypass
  counts as scr_cyc_on_release does (skipped while no type bypasses).

test_cycle checks the window (the release that reaches the trigger still
collects) and the routing of inline-buffered candidates of a bypassed
type; its mature-bypass probe routes before reading generations.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

packages/compiler/src/backend/llvm/alloc.ts

@@ -141,6 +141,97 @@ function declareGlobal(host: AllocHost, decl: string): void {
   if (host.threadLocalAlloc !== true) host.declare(decl);
 }
 
+/** `scr_cyc_buffers` (scr_rc.h ScrCycBuffers): the three candidate buffers
+ * `{ v, n, cap, used }`, the inline buffering window and the allocation
+ * trigger. */
+const CYC_BUFFERS_TYPE = "{ [3 x { ptr, i64, i64, i64 }], i64, i64 }";
+/** Byte offsets in it (asserted in scr_rc.h). */
+const CYC_NURSERY_COUNT = 8;
+const CYC_WINDOW = 96;
+const CYC_ALLOC_TRIGGER = 104;
+
+/** Whether emitted release paths can append nursery candidates to the
+ * collector's buffer themselves (through {@link cycleAppendHelper}): where
+ * the runtime keeps that state in ordinary globals (executables without
+ * workers and classic libraries, on the 64-bit targets with inline
+ * allocation). */
+export function inlineBufferingSupported(host: AllocHost): boolean {
+  return host.inlineAlloc === true && host.threadLocalAlloc !== true && host.sizeType === "i64";
+}
+
+/** The emitted symbol of {@link cycleAppendHelper}. */
+export const CYC_APPEND = "sc_cyc_append";
+
+/** The runtime symbols {@link cycleAppendHelper} uses. The helper is
+ * defined after the module's extern block, so its caller declares them. */
+export function cycleAppendDecls(): string[] {
+  return [
+    `@scr_cyc_buffers = external global ${CYC_BUFFERS_TYPE}`,
+    `@scr_cyc_live = external global i64`,
+    `declare void @scr_cyc_on_release(ptr)`,
+  ];
+}
+
+/** The candidate-buffering step of every emitted release that leaves a
+ * headered object alive and unbuffered: scr_rc.h's scr_cyc_append, then
+ * scr_cyc_on_release when it declines. A nursery object goes onto the
+ * nursery buffer while the buffer holds fewer entries than the runtime's
+ * window and the live count is below the allocation trigger.
+ *
+ * One out-of-line definition per module keeps the release fast paths, which
+ * LLVM inlines into their callers, as small as the plain call they had. It
+ * uses preserve_mostcc, so the call clobbers almost no caller registers:
+ * the callers' code around a release stays what it was, and the append
+ * itself saves only the few registers it uses. Its slow path enters the
+ * runtime through a second preserve_mostcc function, which saves the rest. */
+export function cycleAppendHelper(): string[] {
+  return [
+    `define internal preserve_mostcc void @${CYC_APPEND}(ptr %o) noinline #0 { ; scr_cyc_append, else scr_cyc_on_release`,
+    `entry:`,
+    `  %genp = getelementptr i8, ptr %o, i64 -${CYC_HDR - CYC_GEN}`,
+    `  %gen = load i8, ptr %genp ; ScrCycHdr.gen`,
+    `  %young = icmp eq i8 %gen, 0`,
+    `  br i1 %young, label %window, label %call`,
+    `window:`,
+    `  %np = getelementptr inbounds i8, ptr @scr_cyc_buffers, i64 ${CYC_NURSERY_COUNT}`,
+    `  %n = load i64, ptr %np ; nursery buffer entries`,
+    `  %wp = getelementptr inbounds i8, ptr @scr_cyc_buffers, i64 ${CYC_WINDOW}`,
+    `  %w = load i64, ptr %wp ; window`,
+    `  %room = icmp ult i64 %n, %w`,
+    `  br i1 %room, label %live, label %call`,
+    `live:`,
+    `  %lv = load i64, ptr @scr_cyc_live`,
+    `  %tp = getelementptr inbounds i8, ptr @scr_cyc_buffers, i64 ${CYC_ALLOC_TRIGGER}`,
+    `  %t = load i64, ptr %tp ; allocation trigger`,
+    `  %due = icmp uge i64 %lv, %t`,
+    `  br i1 %due, label %call, label %append`,
+    `append:`,
+    `  %v = load ptr, ptr @scr_cyc_buffers`,
+    `  %slot = getelementptr inbounds ptr, ptr %v, i64 %n`,
+    `  store ptr %o, ptr %slot`,
+    `  %n1 = add i64 %n, 1`,
+    `  store i64 %n1, ptr %np`,
+    `  %i = trunc i64 %n to i32`,
+    `  %b = or i32 %i, -2147483648 ; SCR_CYC_BUFFERED_BIT | position`,
+    `  %bufp = getelementptr i8, ptr %o, i64 -${CYC_HDR - CYC_BUF}`,
+    `  store i32 %b, ptr %bufp`,
+    `  ret void`,
+    `call:`,
+    `  tail call preserve_mostcc void @${CYC_APPEND}_call(ptr %o)`,
+    `  ret void`,
+    `}`,
+    // The runtime call clobbers every caller-saved register, which this
+    // preserve_mostcc function must restore: saving them here keeps those
+    // saves out of the append path above.
+    `define internal preserve_mostcc void @${CYC_APPEND}_call(ptr %o) noinline #0 {`,
+    `entry:`,
+    `  call void @scr_cyc_on_release(ptr %o) ; buffer the candidate; may collect`,
+    `  ret void`,
+    `}`,
+    ``,
+  ];
+}
+
 /** A collector header to install: the object type's constant ScrCycType
  * descriptor (its trace and teardown; see cycTypeDescriptor). */
 export interface CycHeader {

packages/compiler/src/backend/llvm/shapes.test.ts

@@ -205,6 +205,43 @@ describe("LLVM runtime RC symbols", () => {
     expect(fn("sc_rc_retain_promise")).not.toContain("store i32");
   });
 
+  test("optimized builds buffer candidates through the module's append helper", () => {
+    const emit = (inlineRc: true | "release", extra: Partial<ShapeHost>) => {
+      const { host } = declarationHost(64, inlineRc);
+      const configured: ShapeHost = { ...host, inlineAlloc: true, ...extra };
+      releaseSym(configured, { kind: "dyn" });
+      return {
+        ir: emitInlineRcHelpers(configured).join("\n"),
+        decls: inlineRcDecls(configured),
+      };
+    };
+    for (const mode of [true, "release"] as const) {
+      const { ir, decls } = emit(mode, {});
+      expect(ir).toContain("call preserve_mostcc void @sc_cyc_append(ptr %o)");
+      expect(ir).toContain("define internal preserve_mostcc void @sc_cyc_append(ptr %o) noinline");
+      // The young test reads the generation byte at obj-5; the append fills
+      // scr_cyc_buffers' nursery buffer and the header's position at obj-4.
+      expect(ir).toContain("getelementptr i8, ptr %o, i64 -5");
+      expect(ir).toContain("getelementptr inbounds i8, ptr @scr_cyc_buffers, i64 96");
+      expect(ir).toContain("getelementptr inbounds i8, ptr @scr_cyc_buffers, i64 104");
+      expect(ir).toContain("load i64, ptr @scr_cyc_live");
+      expect(ir).toContain("or i32 %i, -2147483648");
+      expect(ir).toContain("call void @scr_cyc_on_release(ptr %o)");
+      expect(decls).toContain(
+        "@scr_cyc_buffers = external global { [3 x { ptr, i64, i64, i64 }], i64, i64 }",
+      );
+      expect(decls).toContain("@scr_cyc_live = external global i64");
+    }
+    // Worker executables and targets without inline allocation call
+    // scr_cyc_on_release, which tries the same append first.
+    for (const extra of [{ threadLocalAlloc: true }, { inlineAlloc: false }]) {
+      const { ir, decls } = emit(true, extra);
+      expect(ir).not.toContain("sc_cyc_append");
+      expect(ir).toContain("call void @scr_cyc_on_release(ptr %o)");
+      expect(decls.join("\n")).not.toContain("scr_cyc_buffers");
+    }
+  });
+
   test("uses the wasm32 header offsets", () => {
     const { host } = declarationHost(32, true);
     releaseSym(host, { kind: "union", unionId: "u0" } as never);

packages/compiler/src/backend/llvm/shapes.ts

@@ -28,7 +28,15 @@ import {
   mangleRecordStruct,
   mangleRecordTrace,
 } from "../mangle.js";
-import { cycTypeDescriptor, emitObjectAlloc, emitObjectFree } from "./alloc.js";
+import {
+  cycTypeDescriptor,
+  emitObjectAlloc,
+  emitObjectFree,
+  CYC_APPEND,
+  cycleAppendDecls,
+  cycleAppendHelper,
+  inlineBufferingSupported,
+} from "./alloc.js";
 import type { BlockBuilder } from "./blocks.js";
 import { SCR_ARR_FIELD, llvmCommentText } from "./common.js";
 import type { NullableUnions } from "./nullable-unions.js";
@@ -308,6 +316,10 @@ export const INLINE_RC_FAMILIES: Readonly<Record<string, InlineRcFamily>> = {
 
 type RcOp = "retain" | "release";
 
+/** The helper key (with the "family:op" keys) of the module's candidate
+ * append helper (alloc.ts cycleAppendHelper). */
+const CYC_APPEND_KEY = "cyc:append";
+
 /** The mirrored family for an RC operation on a runtime stem (`scr_str`),
  * or null when it stays a call. */
 export function inlineRcFamily(host: ShapeHost, stem: string, op: RcOp): InlineRcFamily | null {
@@ -369,6 +381,10 @@ function requestRcHelper(host: ShapeHost, family: string, op: RcOp): string {
   }
   host.rcHelpers.add(`${family}:${op}`);
   for (const decl of rcHelperDecls(family, op)) host.declare(decl);
+  // Its body is written last (emitInlineRcHelpers), after the requests are
+  // collected: a release that can buffer a candidate requests the append
+  // helper now.
+  if (op === "release" && INLINE_RC_FAMILIES[family]!.cycle === "always") requestCycleAppend(host);
   return `@${inlineRcName(family, op)}`;
 }
 
@@ -466,6 +482,10 @@ function rcHelperKeys(host: ShapeHost): string[] {
 export function inlineRcDecls(host: ShapeHost): string[] {
   const decls = new Set<string>();
   for (const key of rcHelperKeys(host)) {
+    if (key === CYC_APPEND_KEY) {
+      for (const decl of cycleAppendDecls()) decls.add(decl);
+      continue;
+    }
     const [family, op] = key.split(":") as [string, RcOp];
     for (const decl of rcHelperDecls(family, op)) decls.add(decl);
   }
@@ -611,6 +631,10 @@ function releaseHelper(host: ShapeHost, family: string, row: InlineRcFamily): st
 export function emitInlineRcHelpers(host: ShapeHost): string[] {
   const out: string[] = [];
   for (const key of rcHelperKeys(host).sort()) {
+    if (key === CYC_APPEND_KEY) {
+      out.push(...cycleAppendHelper());
+      continue;
+    }
     const [family, op] = key.split(":") as [string, RcOp];
     const row = INLINE_RC_FAMILIES[family]!;
     out.push(
@@ -1254,12 +1278,6 @@ export function releaseFastPath(
   ];
 }
 
-/** The possible-cycle-root step of a release that left a headered object
- * alive: scr_cyc_on_release with its early return inlined. Test the sign
- * of `buf` (the i32 at obj-4; negative for a queued candidate or a tenured
- * object) and call the runtime only to enqueue a new candidate.
- * Every emitted release of a headered object shares this sequence
- * (scr_runtime.h asserts the offset); it ends by branching to `done`. */
 /** One traced field of an emitted trace function: load the slot and visit
  * it unless it is null. Every collector visitor ignores null children, and
  * most traced slots of a typical heap are empty (optional links, unset
@@ -1283,16 +1301,39 @@ export function traceFieldVisitLines(
   ];
 }
 
+/** Whether candidate buffering goes through the module's append helper
+ * (alloc.ts cycleAppendHelper), requesting its definition if so: in
+ * optimized builds (those with inline RC helpers) that support it. */
+function requestCycleAppend(host: ShapeHost): boolean {
+  if (host.rcHelpers === null || !inlineBufferingSupported(host)) return false;
+  if (!host.rcHelpers.has(CYC_APPEND_KEY)) {
+    host.rcHelpers.add(CYC_APPEND_KEY);
+    for (const decl of cycleAppendDecls()) host.declare(decl);
+  }
+  return true;
+}
+
+/** The possible-cycle-root step of a release that left a headered object
+ * alive: scr_cyc_on_release with its early return inlined. Test the sign
+ * of `buf` (the i32 at obj-4; negative for a queued candidate or a tenured
+ * object), and buffer a new candidate: through the module's
+ * cycleAppendHelper in optimized builds that support it, through
+ * scr_cyc_on_release otherwise. Every emitted release of a headered object
+ * shares this sequence (scr_runtime.h asserts the offsets); it ends by
+ * branching to `done`. */
 export function cycleRootLines(host: ShapeHost, done: string): string[] {
   const S = host.sizeType;
   host.declare(`declare void @scr_cyc_on_release(ptr)`);
+  const append = requestCycleAppend(host);
   return [
     `  %bufp = getelementptr i8, ptr %o, ${S} -${host.cycleBufOffset}`,
     `  %buf = load i32, ptr %bufp`,
     `  %queued = icmp slt i32 %buf, 0`,
     `  br i1 %queued, label %${done}, label %enqueue`,
     `enqueue:`,
-    `  call void @scr_cyc_on_release(ptr %o) ; buffer the candidate; may collect`,
+    append
+      ? `  call preserve_mostcc void @${CYC_APPEND}(ptr %o) ; buffer the candidate; may collect`
+      : `  call void @scr_cyc_on_release(ptr %o) ; buffer the candidate; may collect`,
     `  br label %${done}`,
   ];
 }

packages/runtime/src/scr_cycle.c

@@ -156,14 +156,11 @@ static void scr_cyc_oom(void) {
  * struct on purpose: buffering a candidate is the hot path, and splitting
  * them across three arrays indexed by generation would touch three cache
  * lines per release where the original single buffer touched one. */
-typedef struct {
-  void **v;
-  size_t n, cap;
-  /* The most entries a pass held, for the work vectors a pass fills
-   * (scr_cyc_push_work); scr_cyc_pass_done frees a large vector that the
-   * pass used little of. */
-  size_t used;
-} ScrVec;
+/* Its layout is scr_rc.h's ScrCycBuffer (the candidate buffers are part of
+ * the inline buffering ABI). `used`: the most entries a pass held, for the
+ * work vectors a pass fills (scr_cyc_push_work); scr_cyc_pass_done frees a
+ * large vector that the pass used little of. */
+typedef ScrCycBuffer ScrVec;
 
 typedef struct {
   void *object;
@@ -201,7 +198,6 @@ typedef struct {
   unsigned char growth_shift[SCR_CYC_NGENS];
   unsigned char growth_streak[SCR_CYC_NGENS];
   unsigned backlog_shift[SCR_CYC_NGENS];
-  size_t nbuffered;
   size_t trigger;
   size_t alloc_trigger;
   size_t gray_referenced;
@@ -235,7 +231,6 @@ static _Thread_local ScrCycThread scr_cyc_thread = {.gen_limit = SCR_CYC_OLD};
 #define scr_cyc_growth_shift (scr_cyc_thread.growth_shift)
 #define scr_cyc_growth_streak (scr_cyc_thread.growth_streak)
 #define scr_cyc_backlog_shift (scr_cyc_thread.backlog_shift)
-#define scr_cyc_nbuffered (scr_cyc_thread.nbuffered)
 #define scr_cyc_trigger (scr_cyc_thread.trigger)
 #define scr_cyc_alloc_trigger (scr_cyc_thread.alloc_trigger)
 #define scr_gray_referenced (scr_cyc_thread.gray_referenced)
@@ -361,7 +356,16 @@ static inline void scr_cyc_note_used(ScrVec *vec) {
 /* ── the candidate-root buffers (one per generation) ──────────────────── */
 
 #ifndef SCR_WORKERS
-static SCR_TL ScrVec scr_roots[SCR_CYC_NGENS];
+/* Emitted and inline releases append nursery candidates to these buffers
+ * directly where the build allows it (scr_rc.h). Explicitly initialized so
+ * library localization can demote it (see scr_cyc_types). */
+#if SCR_CYC_INLINE_BUFFERING
+ScrCycBuffers scr_cyc_buffers = {0};
+#else
+static SCR_TL ScrCycBuffers scr_cyc_buffers;
+#endif
+#define scr_roots (scr_cyc_buffers.roots)
+#define scr_cyc_alloc_trigger (scr_cyc_buffers.alloc_trigger)
 static SCR_TL bool scr_collecting = false;
 #endif
 
@@ -409,6 +413,7 @@ static SCR_TL unsigned scr_gen_limit = SCR_CYC_OLD;
 #endif
 
 static size_t scr_cyc_pass(unsigned gen_limit);
+static void scr_cyc_route_bypassed(void);
 /* A limited pass (scr_cyc_collect_set) takes only the candidates in this
  * set from the buffers up to `scr_cyc_take_gen`; NULL takes them all. */
 static SCR_TL const ScrCycSet *scr_cyc_take = NULL;
@@ -505,18 +510,18 @@ static size_t scr_cyc_nursery_threshold(void) {
   return cached;
 }
 
-/* Buffered candidates across every generation, and the count at which the
- * release path next collects. Keeping a running total (rather than summing
- * the buffers) is what holds the hot path to one load and one compare, as
- * it was when there was a single buffer. The trigger is re-armed at the end
- * of every pass to "whatever survived, plus a nursery's worth": without that
+/* The number of buffered candidates (across every generation) at which the
+ * release path next collects, and the live count at which it collects
+ * however few candidates there are. The trigger is re-armed at the end of
+ * every pass to "whatever survived, plus a nursery's worth": without that
  * hysteresis a large older buffer — which a nursery pass cannot drain —
  * would re-arm on every single release. Both start at zero, so the first
- * release runs one trivial pass that arms them properly. */
+ * release runs one trivial pass that arms them properly. The inline
+ * appends (scr_rc.h) compare against the window scr_cyc_window_update
+ * derives from these, so the slow path sums the buffers when it needs the
+ * total. */
 #ifndef SCR_WORKERS
-static SCR_TL size_t scr_cyc_nbuffered = 0;
 static SCR_TL size_t scr_cyc_trigger = 0;
-static SCR_TL size_t scr_cyc_alloc_trigger = 0;
 #endif
 /* Gray objects whose count is above zero after the trial decrements so far
  * (see scr_mg_visit). markGray records the gray objects in scr_white while
@@ -549,6 +554,34 @@ static size_t scr_cyc_buffered(void) {
          + scr_roots[SCR_CYC_OLD].n;
 }
 
+/* The inline buffering window (scr_rc.h): how many entries the nursery
+ * buffer may hold before an append would bring the buffered total to the
+ * trigger or would need the buffer to grow. While a pass runs, every
+ * release takes the call. Recomputed wherever the buffers, the nursery
+ * buffer's capacity or the trigger change in bulk; the release and death
+ * paths adjust it by one (scr_cyc_window_shrink). */
+static inline void scr_cyc_window_update(void) {
+#if SCR_CYC_INLINE_BUFFERING
+  size_t window = 0;
+  if (!scr_collecting) {
+    size_t older = scr_roots[SCR_CYC_MATURE].n + scr_roots[SCR_CYC_OLD].n;
+    if (scr_cyc_trigger > older + 1) window = scr_cyc_trigger - older - 1;
+    if (window > scr_roots[SCR_CYC_NURSERY].cap) window = scr_roots[SCR_CYC_NURSERY].cap;
+  }
+  scr_cyc_buffers.window = window;
+#endif
+}
+
+/* One step of the window's formula going down by one: an older candidate
+ * was buffered, or a nursery death lowered the trigger. A window below the
+ * formula only sends more releases through the call, so the clamp to the
+ * capacity can stay where it is. */
+static inline void scr_cyc_window_shrink(void) {
+#if SCR_CYC_INLINE_BUFFERING
+  if (scr_cyc_buffers.window) scr_cyc_buffers.window--;
+#endif
+}
+
 static bool scr_cyc_older_backlog(void) {
   return scr_roots[SCR_CYC_MATURE].n != 0 || scr_roots[SCR_CYC_OLD].n != 0;
 }
@@ -682,6 +715,7 @@ static SCR_CYC_SCHEDULING void scr_cyc_scheduled_pass(unsigned gen, bool by_grow
  * per pass, so it is kept out of line and built for size, which leaves the
  * hot buffering path in scr_cyc_on_release small. */
 static SCR_CYC_SCHEDULING void scr_cyc_collect_due(void) {
+  scr_cyc_route_bypassed();
   unsigned gen = SCR_CYC_OLD;
   bool by_growth = scr_cyc_grown(SCR_CYC_OLD);
   if (!by_growth && scr_roots[SCR_CYC_OLD].n < scr_cyc_backlog_threshold(SCR_CYC_OLD)) {
@@ -699,6 +733,7 @@ static SCR_CYC_SCHEDULING void scr_cyc_collect_due(void) {
  * per turn would walk the whole live heap every turn — which is exactly the
  * cost the generations exist to avoid. */
 void scr_cyc_collect_scheduled(void) {
+  scr_cyc_route_bypassed();
   if (scr_cyc_buffered() == 0) {
     scr_cyc_scheduled_mature_age = 0;
     return;
@@ -778,6 +813,10 @@ typedef struct ScrCycKind {
 static SCR_TL ScrCycKind *scr_cyc_kinds;
 static SCR_TL size_t scr_cyc_kinds_cap;
 #endif
+#if SCR_CYC_INLINE_BUFFERING
+/* Types that bypass the nursery right now (scr_cyc_route_bypassed). */
+static size_t scr_cyc_nursery_bypassing = 0;
+#endif
 
 /* The statistics of `kind` (nonzero), growing the table to cover it. */
 static SCR_CYC_COMPACT ScrCycKind *scr_cyc_kind_grow(uint32_t kind) {
@@ -802,7 +841,12 @@ static inline bool scr_cyc_bypassed(const ScrCycType *type, unsigned gen) {
   if (kind >= scr_cyc_kinds_cap) return false;
   ScrCycLevelStat *k = &scr_cyc_kinds[kind].level[gen];
   if (k->bypass == 0) return false;
-  if (--k->bypass == 0) k->walked = k->freed = 0;
+  if (--k->bypass == 0) {
+    k->walked = k->freed = 0;
+#if SCR_CYC_INLINE_BUFFERING
+    if (gen == SCR_CYC_NURSERY) scr_cyc_nursery_bypassing--;
+#endif
+  }
   return true;
 }
 
@@ -820,9 +864,16 @@ static inline void scr_cyc_walked(const ScrCycType *type, unsigned gen) {
   if (!type->kind) return;
   ScrCycLevelStat *k = &scr_cyc_kind(type->kind)->level[gen];
   if (++k->walked == SCR_CYC_BYPASS_WINDOW) {
+#if SCR_CYC_INLINE_BUFFERING
+    bool before = k->bypass != 0;
+#endif
     k->bypass = k->freed * SCR_CYC_BYPASS_RATIO < SCR_CYC_BYPASS_WINDOW
                     ? SCR_CYC_BYPASS_WINDOW * SCR_CYC_BYPASS_PROBE
                     : 0;
+#if SCR_CYC_INLINE_BUFFERING
+    if (gen == SCR_CYC_NURSERY && before != (k->bypass != 0))
+      scr_cyc_nursery_bypassing += before ? (size_t)-1 : 1;
+#endif
     k->walked = k->freed = 0;
   }
 }
@@ -831,6 +882,40 @@ static inline void scr_cyc_freed(const ScrCycType *type, unsigned gen) {
   if (type->kind) scr_cyc_kind(type->kind)->level[gen].freed++;
 }
 
+/* An inline append (scr_rc.h) leaves every nursery candidate in the nursery
+ * buffer, including one whose type bypasses the nursery, which
+ * scr_cyc_on_release would have buffered past it. Where a candidate waits
+ * matters only once something reads the buffers, so every scheduling
+ * decision and every pass first moves such candidates to the generation
+ * scr_cyc_on_release would have chosen, consuming the bypass counts as it
+ * would have, in buffer order. Candidates that scr_cyc_on_release buffered
+ * already passed the same test with the counts unchanged since (they only
+ * change during passes), so they stay. */
+static SCR_CYC_COMPACT void scr_cyc_route_bypassed(void) {
+#if SCR_CYC_INLINE_BUFFERING
+  if (scr_cyc_nursery_bypassing == 0) return;
+  ScrVec *nursery = &scr_roots[SCR_CYC_NURSERY];
+  for (size_t i = 0; i < nursery->n;) {
+    void *obj = nursery->v[i];
+    ScrCycHdr *h = scr_cyc_hdr(obj);
+    unsigned gen = scr_cyc_buffer_gen(scr_cyc_type(h), SCR_CYC_NURSERY);
+    if (gen == SCR_CYC_NURSERY) {
+      i++;
+      continue;
+    }
+    void *last = nursery->v[--nursery->n];
+    nursery->v[i] = last;
+    if (last != obj) scr_cyc_set_buffered(scr_cyc_hdr(last), (uint32_t)i);
+    scr_cyc_set_gen(h, gen);
+    ScrVec *b = &scr_roots[gen];
+    if (b->n == b->cap) scr_cyc_grow(b);
+    scr_cyc_set_buffered(h, (uint32_t)b->n);
+    b->v[b->n++] = obj;
+  }
+  scr_cyc_window_update();
+#endif
+}
+
 void scr_cyc_on_dead(void *obj) {
   ScrCycHdr *h = scr_cyc_hdr(obj);
   if (!scr_cyc_is_buffered(h)) return;
@@ -843,11 +928,12 @@ void scr_cyc_on_dead(void *obj) {
   b->v[i] = last;
   if (last != obj) scr_cyc_set_buffered(scr_cyc_hdr(last), (uint32_t)i);
   scr_cyc_clear_buffered(h);
-  scr_cyc_nbuffered--;
   /* A death must not consume room reserved for a future candidate. */
   scr_cyc_trigger--;
-  if (scr_cyc_gen(h) != SCR_CYC_NURSERY && !scr_cyc_older_backlog())
-    scr_cyc_scheduled_mature_age = 0;
+  /* An older death lowers the older count with the trigger: the window
+   * stays. A nursery death lowers only the trigger. */
+  if (scr_cyc_gen(h) == SCR_CYC_NURSERY) scr_cyc_window_shrink();
+  else if (!scr_cyc_older_backlog()) scr_cyc_scheduled_mature_age = 0;
 }
 
 /* Publication (scr_publish.c) takes buffered objects out of their candidate
@@ -896,31 +982,39 @@ void scr_cyc_pub_fill(void) {
     }
     holes->n = 0;
     size_t closed = before - b->n;
-    scr_cyc_nbuffered -= closed;
     /* A death must not consume room reserved for a future candidate. */
     scr_cyc_trigger -= closed;
     if (closed && gen != SCR_CYC_NURSERY) older = true;
   }
   if (older && !scr_cyc_older_backlog()) scr_cyc_scheduled_mature_age = 0;
+  scr_cyc_window_update();
 }
 
 /* Every release that leaves an unbuffered object alive lands here (the
- * emitted and inline release paths test `buffered` first), so it stays what
- * it has always been: buffer inline, then one compare. */
+ * emitted and inline release paths test `buffered` first) unless the window
+ * let it append a nursery candidate on its own fast path (scr_rc.h), so it
+ * stays what it has always been: the same append, then buffer and compare. */
 void scr_cyc_on_release(void *obj) {
   ScrCycHdr *h = scr_cyc_hdr(obj);
   /* Releasing an existing candidate cannot advance the trigger. A pass
    * always re-arms after its teardowns, including any candidates they add. */
   if (scr_cyc_is_buffered(h)) return;
+#if SCR_CYC_INLINE_BUFFERING
+  if (scr_cyc_append(h, obj)) return;
+#endif
   if (scr_cyc_gen(h) < SCR_CYC_OLD) scr_cyc_set_gen(h, scr_cyc_buffer_gen(scr_cyc_type(h), scr_cyc_gen(h)));
-  ScrVec *b = &scr_roots[scr_cyc_gen(h)];
-  if (b->n == b->cap) scr_cyc_grow(b);
+  unsigned gen = scr_cyc_gen(h);
+  ScrVec *b = &scr_roots[gen];
+  if (b->n == b->cap) {
+    scr_cyc_grow(b);
+    scr_cyc_window_update();
+  }
   scr_cyc_set_buffered(h, (uint32_t)b->n);
   b->v[b->n++] = obj;
-  scr_cyc_nbuffered++;
+  if (gen != SCR_CYC_NURSERY) scr_cyc_window_shrink();
   /* Never re-entered: a teardown's releases of untraced children can buffer
    * new candidates mid-collection, but they only wait for the next pass. */
-  if (!scr_collecting && (scr_cyc_nbuffered >= scr_cyc_trigger || scr_cyc_live >= scr_cyc_alloc_trigger))
+  if (!scr_collecting && (scr_cyc_buffered() >= scr_cyc_trigger || scr_cyc_live >= scr_cyc_alloc_trigger))
     scr_cyc_collect_due();
 }
 
@@ -1272,10 +1366,10 @@ static SCR_CYC_COMPACT size_t scr_cyc_settle(unsigned gen_limit, bool doomed) {
   }
   /* markRoots drained buffers in bulk and teardowns moved the count around;
    * resync from the buffers themselves and re-arm. */
-  scr_cyc_nbuffered = scr_cyc_buffered();
-  scr_cyc_trigger = scr_cyc_nbuffered + scr_cyc_nursery_threshold();
+  scr_cyc_trigger = scr_cyc_buffered() + scr_cyc_nursery_threshold();
   scr_cyc_alloc_trigger = scr_cyc_live + SCR_CYC_NURSERY_ALLOCATIONS;
   scr_collecting = false;
+  scr_cyc_window_update();
   return freed;
 }
 
@@ -1315,7 +1409,9 @@ static size_t scr_cyc_pass_done(size_t freed) {
 
 static size_t scr_cyc_pass(unsigned gen_limit) {
   if (scr_collecting) return 0;
+  scr_cyc_route_bypassed();
   scr_collecting = true;
+  scr_cyc_window_update();
   scr_gen_limit = gen_limit;
 
   /* markRoots: every buffered entry is a candidate (retains do not mark
@@ -1488,6 +1584,9 @@ static size_t scr_cyc_pass(unsigned gen_limit) {
 #ifdef SCR_CYC_TESTING
 /* Tests only: one pass at exactly `gen`, outside the schedule. */
 size_t scr_cyc_pass_at(unsigned gen) { return scr_cyc_pass(gen); }
+/* Tests only: route appended candidates of bypassing types, as the next
+ * scheduling decision or pass would. */
+void scr_cyc_route_bypassed_at(void) { scr_cyc_route_bypassed(); }
 #endif
 
 size_t scr_cyc_collect_set(const ScrCycSet *set, unsigned gen) {
@@ -1526,4 +1625,8 @@ void scr_cyc_context_cleanup(void) {
   free(scr_cyc_kinds);
   scr_cyc_kinds = NULL;
   scr_cyc_kinds_cap = 0;
+#if SCR_CYC_INLINE_BUFFERING
+  scr_cyc_nursery_bypassing = 0;
+#endif
+  scr_cyc_window_update();
 }

packages/runtime/src/scr_rc.h

@@ -463,12 +463,76 @@ uint32_t scr_cyc_pub_take(void *obj);
 void scr_cyc_pub_restore(void *obj, uint32_t index);
 void scr_cyc_pub_fill(void);
 
+/* The candidate buffers, one per generation (scr_cycle.c), and the window
+ * through which a release buffers a nursery candidate on its fast path. A
+ * release that leaves an unbuffered nursery object alive appends it to the
+ * nursery buffer while that buffer holds fewer than `window` entries and the
+ * live count is below `alloc_trigger` (scr_cyc_append); anything else takes
+ * the full scr_cyc_on_release. scr_cycle.c keeps `window` at most at the
+ * number of nursery entries the buffers can hold before a collection falls
+ * due or the buffer must grow, and at 0 while a pass runs, so the append is
+ * exactly what scr_cyc_on_release would do with a nursery candidate. A
+ * candidate whose type bypasses the nursery is moved past it before the
+ * buffers are next read (scr_cycle.c, "level bypass").
+ *
+ * The runtime's own release paths append inline (scr_cyc_released), and
+ * scr_cyc_on_release tries the append first. The layout is an ABI:
+ * optimized emitted code appends through a per-module helper
+ * (llvm/alloc.ts cycleAppendHelper) that addresses roots[0].v, roots[0].n,
+ * window and alloc_trigger at fixed offsets. Builds whose collector state
+ * is thread-local (worker executables, thread-instanced libraries) never
+ * append on a fast path. */
+typedef struct ScrCycBuffer {
+  void **v;
+  size_t n, cap;
+  /* The most entries a pass held, for the work vectors a pass fills
+   * (scr_cycle.c ScrVec); unused by the candidate buffers. */
+  size_t used;
+} ScrCycBuffer;
+typedef struct ScrCycBuffers {
+  ScrCycBuffer roots[SCR_CYC_NGENS];
+  size_t window;
+  size_t alloc_trigger;
+} ScrCycBuffers;
+#if !defined(SCR_WORKERS) && !(defined(SCR_LIB) && defined(SCR_THREAD_INSTANCES))
+#define SCR_CYC_INLINE_BUFFERING 1
+extern ScrCycBuffers scr_cyc_buffers;
+#if UINTPTR_MAX == UINT64_MAX
+_Static_assert(offsetof(ScrCycBuffers, roots) == 0 && offsetof(ScrCycBuffer, v) == 0 &&
+                   offsetof(ScrCycBuffer, n) == 8 && offsetof(ScrCycBuffers, window) == 96 &&
+                   offsetof(ScrCycBuffers, alloc_trigger) == 104,
+               "llvm/alloc.ts appends nursery candidates against this layout");
+#endif
+/* Append the unbuffered object `obj` (header `h`) to the nursery buffer if
+ * it is a nursery object and the window allows; false leaves it to the rest
+ * of scr_cyc_on_release. */
+static inline bool scr_cyc_append(ScrCycHdr *h, void *obj) {
+  ScrCycBuffer *nursery = &scr_cyc_buffers.roots[SCR_CYC_NURSERY];
+  size_t n = nursery->n;
+  if (scr_cyc_gen(h) != SCR_CYC_NURSERY || n >= scr_cyc_buffers.window ||
+      scr_cyc_live >= scr_cyc_buffers.alloc_trigger)
+    return false;
+  nursery->v[n] = obj;
+  nursery->n = n + 1;
+  scr_cyc_set_buffered(h, (uint32_t)n);
+  return true;
+}
+#else
+#define SCR_CYC_INLINE_BUFFERING 0
+#endif
+
 /* Inline halves of the hooks for runtime release paths, mirroring the
  * emitted ones (llvm/shapes.ts): a release of an already-buffered
- * object, and the death of an object that was never buffered, need no call.
+ * object, and the death of an object that was never buffered, need no call,
+ * and a nursery candidate is appended inline while the window allows.
  * Most releases of long-lived containers take the first early return. */
 static inline void scr_cyc_released(void *obj) {
-  if (!scr_cyc_is_buffered(scr_cyc_hdr(obj))) scr_cyc_on_release(obj);
+  ScrCycHdr *h = scr_cyc_hdr(obj);
+  if (scr_cyc_is_buffered(h)) return;
+#if SCR_CYC_INLINE_BUFFERING
+  if (scr_cyc_append(h, obj)) return;
+#endif
+  scr_cyc_on_release(obj);
 }
 static inline void scr_cyc_dead(void *obj) {
   if (scr_cyc_is_buffered(scr_cyc_hdr(obj))) scr_cyc_on_dead(obj);

packages/runtime/test/test_cycle.c

@@ -712,8 +712,11 @@ static void check_deferred_white_restoration(void) {
  * once the type's objects die in nursery cycles again, the next probe window
  * finds that and it is walked again. Window, ratio and probe interval mirror
  * SCR_CYC_BYPASS_*. */
-/* Tests only (SCR_CYC_TESTING): one pass at exactly `gen`. */
+/* Tests only (SCR_CYC_TESTING): one pass at exactly `gen`, and the routing
+ * of appended candidates whose type bypasses a level, which the next
+ * scheduling decision or pass would do. */
 size_t scr_cyc_pass_at(unsigned gen);
+void scr_cyc_route_bypassed_at(void);
 static size_t bypass_traced; /* trace calls on bypass-kind nodes */
 static size_t bypass_freed;  /* bypass-kind nodes freed */
 static void bypass_trace(void *obj, ScrTraceVisit visit, void *ctx) {
@@ -785,6 +788,22 @@ static void check_nursery_bypass(void) {
   scr_collect_cycles();
   check(bypass_freed - before == 2 * RINGS, "an older pass missed dead rings of a bypassed type");
 
+  /* The same through the inline release path, which leaves the candidates
+   * in the nursery buffer: the next pass routes them past the nursery. */
+  before = bypass_freed;
+  for (size_t r = 0; r < RINGS; r++) {
+    Node *a = make_bypass_node(), *b = make_bypass_node();
+    a->next = b;
+    b->next = a;
+    a->rc++;
+    a->rc--;
+    scr_cyc_released(a);
+  }
+  run_release_passes();
+  check(bypass_freed - before < 2 * RINGS, "inline-buffered bypassed rings were walked by the nursery");
+  scr_collect_cycles();
+  check(bypass_freed - before == 2 * RINGS, "an older pass missed inline-buffered rings");
+
   /* Now the type's objects die in nursery cycles. Once INTERVAL windows'
    * worth of candidates went past the nursery, a probe window walks them
    * there again, and its yield turns the bypass off. A nursery pass after
@@ -832,13 +851,16 @@ static Node *make_mature_kind_node(void) {
 }
 
 /* How many of PROBE fresh, externally owned candidates a release buffers
- * straight into the old generation. */
+ * straight into the old generation, by the time anything reads the buffers
+ * (a release may append the candidate to the nursery buffer, which the next
+ * decision routes past the levels its type bypasses). */
 static size_t probe_old_on_release(Node **owned, size_t probe) {
   scr_collect_cycles(); /* reset the growth baselines: no pass runs below */
   size_t old = 0;
   for (size_t i = 0; i < probe; i++) {
     owned[i] = make_mature_kind_node();
     read_live(owned[i]);
+    scr_cyc_route_bypassed_at();
     old += scr_cyc_gen(scr_cyc_hdr(owned[i])) == SCR_CYC_OLD;
   }
   return old;
@@ -1187,7 +1209,55 @@ static void check_large_old_buffer_with_nursery_candidates(void) {
   }
 }
 
+/* Releases through scr_cyc_released append nursery candidates inline while
+ * the window allows, and fall back to scr_cyc_on_release exactly when a
+ * collection is due: the candidate that brings the buffered total to the
+ * trigger runs the pass, as it would through the call. */
+static void check_inline_buffering(void) {
+  enum { MAX = 4096 };
+  static Node *nodes[MAX];
+  size_t threshold = configured_nursery_threshold();
+  if (threshold > MAX) return;
+  scr_collect_cycles();
+  size_t before = traced;
+  for (size_t i = 0; i < threshold; i++) {
+    nodes[i] = make_ring(1);
+    nodes[i]->rc--; /* drop the self edge: a live leaf held by its owner */
+    nodes[i]->next = NULL;
+    nodes[i]->rc++;
+    nodes[i]->rc--;
+    scr_cyc_released(nodes[i]);
+    ScrCycHdr *h = scr_cyc_hdr(nodes[i]);
+    if (i + 1 < threshold) {
+      check(scr_cyc_is_buffered(h) && scr_cyc_gen(h) == SCR_CYC_NURSERY,
+            "a nursery release did not buffer its candidate");
+#if SCR_CYC_INLINE_BUFFERING
+      check(scr_cyc_buffers.roots[SCR_CYC_NURSERY].v[scr_cyc_buf_index(h)] == nodes[i],
+            "an inline append is not where the header says");
+#endif
+      check(traced == before, "a pass ran before the trigger");
+    }
+  }
+  /* The last release reached the trigger: one nursery pass walked them all
+   * and promoted the survivors, which wait in the mature buffer. */
+  check(traced > before, "the release that reached the trigger did not collect");
+  for (size_t i = 0; i < threshold; i++) {
+    ScrCycHdr *h = scr_cyc_hdr(nodes[i]);
+    check(scr_cyc_gen(h) == SCR_CYC_MATURE, "a nursery survivor was not promoted");
+    check(nodes[i]->rc == 1, "a survivor lost its count");
+  }
+  size_t freed_before = freed;
+  for (size_t i = 0; i < threshold; i++) {
+    nodes[i]->rc = 0;
+    scr_cyc_on_dead(nodes[i]);
+    node_free(nodes[i]);
+  }
+  check(freed - freed_before == threshold, "the leaves were not freed");
+  scr_collect_cycles();
+}
+
 int main(void) {
+  check_inline_buffering();
   check_old_garbage_reclaimed_while_growing();
   check_older_rings_into_old();
   check_cycle_spanning_mature_and_old();