Keep in-loop dyn releases inline in bodies over the release budget

A function with more than INLINE_DYN_RELEASES_PER_FUNCTION inline dyn
releases called scr_dyn_release at every site. A script's top-level loop
runs in its module initializer, which often has more sites than that, so
its hottest releases became calls. The emitter now marks inline dyn
releases emitted inside loops, and a body over the budget keeps up to the
budget of those inline, calling the runtime at the rest.

packages/compiler/src/backend/llvm/blocks.ts

@@ -11,6 +11,11 @@
  * definition can leak into a later block. */
 import { UNWIND_RESUME, UNWIND_SLOT } from "./unwind-abi.js";
 
+/** An inline dyn release call, and the comment that marks one emitted
+ * inside a loop: the sites limitInlineReleases keeps inline first. */
+export const DYN_RELEASE_CALL = "call void @sc_rc_release_dyn(";
+export const LOOP_SITE = " ; loop";
+
 export class BlockBuilder {
   private readonly blocks: { label: string; lines: string[]; term: string | null }[] = [];
   private cur: { label: string; lines: string[]; term: string | null };
@@ -35,6 +40,9 @@ export class BlockBuilder {
   /** Entry slots whose loads and stores render volatile, so no LLVM pass
    * promotes them to SSA values (see volatileSlotAccess). */
   volatileSlots: ReadonlySet<string> | null = null;
+  /** Loops of the IR function enclosing the lines being emitted; inline dyn
+   * releases inside one carry LOOP_SITE (shapes.ts limitInlineReleases). */
+  loopDepth = 0;
 
   constructor() {
     this.cur = { label: "entry", lines: [], term: null };
@@ -56,7 +64,10 @@ export class BlockBuilder {
   }
 
   line(s: string): void {
-    if (this.cur.term === null) this.cur.lines.push(`  ${this.withDebugLocation(s)}`);
+    if (this.cur.term === null) {
+      const marked = this.loopDepth > 0 && s.startsWith(DYN_RELEASE_CALL) ? s + LOOP_SITE : s;
+      this.cur.lines.push(`  ${this.withDebugLocation(marked)}`);
+    }
   }
 
   private withDebugLocation(s: string): string {

packages/compiler/src/backend/llvm/emitter.ts

@@ -6621,7 +6621,11 @@ export class LlEmitter {
     this.B.debugLocation = this.debug?.location(s.loc, this.debugScope) ?? null;
     const loop =
       s.kind === "while" || s.kind === "doWhile" || s.kind === "for" || s.kind === "forOf";
-    if (loop) this.loopDepth++;
+    const builder = this.B;
+    if (loop) {
+      this.loopDepth++;
+      builder.loopDepth++;
+    }
     // An int32 kind loop's fast-copy write leaves for its resume point.
     const kindResume = this.int32KindResume;
     this.int32KindResume = this.int32Kinds?.resume.get(s) ?? null;
@@ -6630,7 +6634,10 @@ export class LlEmitter {
     } finally {
       this.B.debugLocation = previous;
       this.int32KindResume = kindResume;
-      if (loop) this.loopDepth--;
+      if (loop) {
+        this.loopDepth--;
+        builder.loopDepth--;
+      }
     }
   }
   private emitStmtBody(s: IrStmt): void {

packages/compiler/src/backend/llvm/shapes.test.ts

@@ -15,6 +15,7 @@ import {
   vAdapters,
 } from "./shapes.js";
 import { mangleRecordRelease } from "../mangle.js";
+import { LOOP_SITE } from "./blocks.js";
 import { NullableUnions } from "./nullable-unions.js";
 import { InlineUnionFields } from "./inline-unions.js";
 
@@ -294,6 +295,19 @@ describe("inline release budgets", () => {
     );
   });
 
+  test("a function over the dyn budget keeps its in-loop sites inline", () => {
+    // Two sites emitted inside a loop, after more sites than the budget.
+    const large = body("large", INLINE_DYN_RELEASES_PER_FUNCTION + 1).replace(
+      "  ret void",
+      `  call void @sc_rc_release_dyn(ptr %v)${LOOP_SITE}\n  call void @sc_rc_release_dyn(ptr %v)${LOOP_SITE}\n  ret void`,
+    );
+    const limited = limitInlineReleases([large], "speed").join("\n");
+    expect(limited.split(`call void @sc_rc_release_dyn(ptr %v)${LOOP_SITE}`).length - 1).toBe(2);
+    expect(limited.split("call void @scr_dyn_release(ptr %v)\n").length - 1).toBe(
+      INLINE_DYN_RELEASES_PER_FUNCTION + 1,
+    );
+  });
+
   test("each family's budget applies to its own sites in a function that has both", () => {
     const strings = body("mixed", INLINE_STR_RELEASES_PER_FUNCTION + 1, "str");
     // One function: string sites over their budget, dyn sites within theirs.

packages/compiler/src/backend/llvm/shapes.ts

@@ -42,7 +42,7 @@ import {
   cycleAppendHelper,
   inlineBufferingSupported,
 } from "./alloc.js";
-import type { BlockBuilder } from "./blocks.js";
+import { LOOP_SITE, type BlockBuilder } from "./blocks.js";
 import { SCR_ARR_FIELD, llvmCommentText } from "./common.js";
 import {
   NULLABLE_NULL,
@@ -805,7 +805,9 @@ function inlineRcName(family: string, op: RcOp): string {
  * grows faster than the number of inlined releases in one function, and
  * generated dispatch tables and module initializers of large packages have
  * thousands of release sites; a body with more calls scr_dyn_release, which
- * the inline helper mirrors, at every site. */
+ * the inline helper mirrors, at every site except up to this many sites
+ * inside its loops (a script's top-level loop runs in its module
+ * initializer). */
 export const INLINE_DYN_RELEASES_PER_FUNCTION = 256;
 
 /** Inlined string releases one function body keeps in `release` emission,
@@ -878,6 +880,14 @@ export function limitInlineReleases(
     for (let f = from; f <= to; f++) if (anyOver.has(f)) return true;
     return false;
   };
+  // A body over its dyn limit keeps inline the releases emitted inside
+  // loops (LOOP_SITE), up to the limit, and calls the runtime at the rest.
+  const dynFamily = families.indexOf("dyn");
+  const keptInLoops = new Map<number, number>();
+  const inLoop = (part: string, at: number): boolean => {
+    const end = part.indexOf("\n", at);
+    return (end < 0 ? part.slice(at) : part.slice(at, end)).endsWith(LOOP_SITE);
+  };
   return parts.map((part, index) => {
     const from = spans[2 * index]!;
     if (!touches(from, spans[2 * index + 1]!)) return part;
@@ -885,7 +895,15 @@ export function limitInlineReleases(
     sites(
       part,
       (at, family, f) => {
-        if (over[family]!.has(f)) cuts.push({ at, family });
+        if (!over[family]!.has(f)) return;
+        if (family === dynFamily && inLoop(part, at)) {
+          const kept = keptInLoops.get(f) ?? 0;
+          if (kept < limits[family]!) {
+            keptInLoops.set(f, kept + 1);
+            return;
+          }
+        }
+        cuts.push({ at, family });
       },
       from,
     );