diff --git a/packages/compiler/src/backend/llvm/blocks.ts b/packages/compiler/src/backend/llvm/blocks.ts index 2aef040a1..99ab46f22 100644 --- a/packages/compiler/src/backend/llvm/blocks.ts +++ b/packages/compiler/src/backend/llvm/blocks.ts @@ -11,6 +11,11 @@ * definition can leak into a later block. */ import { UNWIND_RESUME, UNWIND_SLOT } from "./unwind-abi.js"; +/** An inline dyn release call, and the comment that marks one emitted + * inside a loop: the sites limitInlineReleases keeps inline first. */ +export const DYN_RELEASE_CALL = "call void @sc_rc_release_dyn("; +export const LOOP_SITE = " ; loop"; + export class BlockBuilder { private readonly blocks: { label: string; lines: string[]; term: string | null }[] = []; private cur: { label: string; lines: string[]; term: string | null }; @@ -35,6 +40,9 @@ export class BlockBuilder { /** Entry slots whose loads and stores render volatile, so no LLVM pass * promotes them to SSA values (see volatileSlotAccess). */ volatileSlots: ReadonlySet | null = null; + /** Loops of the IR function enclosing the lines being emitted; inline dyn + * releases inside one carry LOOP_SITE (shapes.ts limitInlineReleases). */ + loopDepth = 0; constructor() { this.cur = { label: "entry", lines: [], term: null }; @@ -56,7 +64,10 @@ export class BlockBuilder { } line(s: string): void { - if (this.cur.term === null) this.cur.lines.push(` ${this.withDebugLocation(s)}`); + if (this.cur.term === null) { + const marked = this.loopDepth > 0 && s.startsWith(DYN_RELEASE_CALL) ? s + LOOP_SITE : s; + this.cur.lines.push(` ${this.withDebugLocation(marked)}`); + } } private withDebugLocation(s: string): string { diff --git a/packages/compiler/src/backend/llvm/emitter.ts b/packages/compiler/src/backend/llvm/emitter.ts index 2cb5d2f87..9d85bb8da 100644 --- a/packages/compiler/src/backend/llvm/emitter.ts +++ b/packages/compiler/src/backend/llvm/emitter.ts @@ -6621,7 +6621,11 @@ export class LlEmitter { this.B.debugLocation = this.debug?.location(s.loc, this.debugScope) ?? null; const loop = s.kind === "while" || s.kind === "doWhile" || s.kind === "for" || s.kind === "forOf"; - if (loop) this.loopDepth++; + const builder = this.B; + if (loop) { + this.loopDepth++; + builder.loopDepth++; + } // An int32 kind loop's fast-copy write leaves for its resume point. const kindResume = this.int32KindResume; this.int32KindResume = this.int32Kinds?.resume.get(s) ?? null; @@ -6630,7 +6634,10 @@ export class LlEmitter { } finally { this.B.debugLocation = previous; this.int32KindResume = kindResume; - if (loop) this.loopDepth--; + if (loop) { + this.loopDepth--; + builder.loopDepth--; + } } } private emitStmtBody(s: IrStmt): void { diff --git a/packages/compiler/src/backend/llvm/shapes.test.ts b/packages/compiler/src/backend/llvm/shapes.test.ts index d9e5fa641..4503b5437 100644 --- a/packages/compiler/src/backend/llvm/shapes.test.ts +++ b/packages/compiler/src/backend/llvm/shapes.test.ts @@ -15,6 +15,7 @@ import { vAdapters, } from "./shapes.js"; import { mangleRecordRelease } from "../mangle.js"; +import { LOOP_SITE } from "./blocks.js"; import { NullableUnions } from "./nullable-unions.js"; import { InlineUnionFields } from "./inline-unions.js"; @@ -294,6 +295,19 @@ describe("inline release budgets", () => { ); }); + test("a function over the dyn budget keeps its in-loop sites inline", () => { + // Two sites emitted inside a loop, after more sites than the budget. + const large = body("large", INLINE_DYN_RELEASES_PER_FUNCTION + 1).replace( + " ret void", + ` call void @sc_rc_release_dyn(ptr %v)${LOOP_SITE}\n call void @sc_rc_release_dyn(ptr %v)${LOOP_SITE}\n ret void`, + ); + const limited = limitInlineReleases([large], "speed").join("\n"); + expect(limited.split(`call void @sc_rc_release_dyn(ptr %v)${LOOP_SITE}`).length - 1).toBe(2); + expect(limited.split("call void @scr_dyn_release(ptr %v)\n").length - 1).toBe( + INLINE_DYN_RELEASES_PER_FUNCTION + 1, + ); + }); + test("each family's budget applies to its own sites in a function that has both", () => { const strings = body("mixed", INLINE_STR_RELEASES_PER_FUNCTION + 1, "str"); // One function: string sites over their budget, dyn sites within theirs. diff --git a/packages/compiler/src/backend/llvm/shapes.ts b/packages/compiler/src/backend/llvm/shapes.ts index 993a1ffa5..87983c8b9 100644 --- a/packages/compiler/src/backend/llvm/shapes.ts +++ b/packages/compiler/src/backend/llvm/shapes.ts @@ -42,7 +42,7 @@ import { cycleAppendHelper, inlineBufferingSupported, } from "./alloc.js"; -import type { BlockBuilder } from "./blocks.js"; +import { LOOP_SITE, type BlockBuilder } from "./blocks.js"; import { SCR_ARR_FIELD, llvmCommentText } from "./common.js"; import { NULLABLE_NULL, @@ -805,7 +805,9 @@ function inlineRcName(family: string, op: RcOp): string { * grows faster than the number of inlined releases in one function, and * generated dispatch tables and module initializers of large packages have * thousands of release sites; a body with more calls scr_dyn_release, which - * the inline helper mirrors, at every site. */ + * the inline helper mirrors, at every site except up to this many sites + * inside its loops (a script's top-level loop runs in its module + * initializer). */ export const INLINE_DYN_RELEASES_PER_FUNCTION = 256; /** Inlined string releases one function body keeps in `release` emission, @@ -878,6 +880,14 @@ export function limitInlineReleases( for (let f = from; f <= to; f++) if (anyOver.has(f)) return true; return false; }; + // A body over its dyn limit keeps inline the releases emitted inside + // loops (LOOP_SITE), up to the limit, and calls the runtime at the rest. + const dynFamily = families.indexOf("dyn"); + const keptInLoops = new Map(); + const inLoop = (part: string, at: number): boolean => { + const end = part.indexOf("\n", at); + return (end < 0 ? part.slice(at) : part.slice(at, end)).endsWith(LOOP_SITE); + }; return parts.map((part, index) => { const from = spans[2 * index]!; if (!touches(from, spans[2 * index + 1]!)) return part; @@ -885,7 +895,15 @@ export function limitInlineReleases( sites( part, (at, family, f) => { - if (over[family]!.has(f)) cuts.push({ at, family }); + if (!over[family]!.has(f)) return; + if (family === dynFamily && inLoop(part, at)) { + const kept = keptInLoops.get(f) ?? 0; + if (kept < limits[family]!) { + keptInLoops.set(f, kept + 1); + return; + } + } + cuts.push({ at, family }); }, from, );