diff --git a/packages/compiler/src/backend/llvm/emit-function.ts b/packages/compiler/src/backend/llvm/emit-function.ts index 8f306035a..d7efafd36 100644 --- a/packages/compiler/src/backend/llvm/emit-function.ts +++ b/packages/compiler/src/backend/llvm/emit-function.ts @@ -1307,8 +1307,10 @@ export function emitArgPackAndTrampolinePrologue( * code. */ export function parkingFunctions(emitter: LlEmitter): Set | null { if (emitter.wasi || emitter.mod.workers === true) return null; - for (const decl of emitter.decls) if (/@scr_test_sub\b/.test(decl)) return null; - for (const text of emitter.fnDefText.values()) if (/@scr_test_sub\b/.test(text)) return null; + const testSub = (text: string): boolean => + text.includes("@scr_test_sub") && /@scr_test_sub\b/.test(text); + for (const decl of emitter.decls) if (testSub(decl)) return null; + for (const text of emitter.fnDefText.values()) if (testSub(text)) return null; // Reverse mention edges, then propagate "may park" to every mentioner. const mentionedBy = new Map>(); const parks = new Set(); diff --git a/packages/compiler/src/backend/llvm/emitter.ts b/packages/compiler/src/backend/llvm/emitter.ts index e5b59c849..59b4b64a4 100644 --- a/packages/compiler/src/backend/llvm/emitter.ts +++ b/packages/compiler/src/backend/llvm/emitter.ts @@ -621,6 +621,16 @@ export function llStrBytes(text: string): string { const NO_BORROWED: ReadonlySet = new Set(); +/** [A-Za-z0-9_] */ +function isIdentifierByte(code: number): boolean { + return ( + (code >= 48 && code <= 57) || + (code >= 65 && code <= 90) || + (code >= 97 && code <= 122) || + code === 95 + ); +} + /** Runtime entry points (and await-call entries) that may park the running * execution context, as they appear in emitted function text. */ export const SUSPENDS = @@ -2263,9 +2273,23 @@ export class LlEmitter { this.mangledFunctions.set(mangleBorrowedFunction(fn.name), fn.name); } } - for (const match of text.matchAll(/@(sc_b?f_[A-Za-z0-9_]+)/g)) { - const callee = this.mangledFunctions.get(match[1]!); + // The names `@(sc_b?f_[A-Za-z0-9_]+)` matches, found with indexOf. + for (let at = text.indexOf("@sc_"); at >= 0;) { + let start = at + 4; + if (text.charCodeAt(start) === 98 /* b */) start++; + if (text.charCodeAt(start) !== 102 /* f */ || text.charCodeAt(start + 1) !== 95 /* _ */) { + at = text.indexOf("@sc_", at + 1); + continue; + } + let end = start + 2; + while (end < text.length && isIdentifierByte(text.charCodeAt(end))) end++; + if (end === start + 2) { + at = text.indexOf("@sc_", at + 1); + continue; + } + const callee = this.mangledFunctions.get(text.slice(at + 1, end)); if (callee !== undefined) yield callee; + at = text.indexOf("@sc_", end); } } diff --git a/packages/compiler/src/backend/llvm/shapes.test.ts b/packages/compiler/src/backend/llvm/shapes.test.ts index 7af16ab83..f91ee2f58 100644 --- a/packages/compiler/src/backend/llvm/shapes.test.ts +++ b/packages/compiler/src/backend/llvm/shapes.test.ts @@ -280,6 +280,22 @@ describe("inline release budgets", () => { ); }); + test("each family's budget applies to its own sites in a function that has both", () => { + const strings = body("mixed", INLINE_STR_RELEASES_PER_FUNCTION + 1, "str"); + // One function: string sites over their budget, dyn sites within theirs. + const mixed = strings.replace( + " ret void", + ` call void @sc_rc_release_dyn(ptr %v)\n call void @sc_rc_release_other(ptr %v)\n ret void`, + ); + const other = body("other", 2, "str"); + const limited = limitInlineReleases([mixed, other], "release").join("\n"); + const out = limited.slice(0, limited.indexOf("@other(")); + expect(out).not.toContain("@sc_rc_release_str("); + expect(out).toContain("call void @sc_rc_release_dyn(ptr %v)"); + expect(out).toContain("call void @sc_rc_release_other(ptr %v)"); + expect(limited).toContain(other); + }); + test("a function over the string budget calls the runtime string release at every site", () => { const small = body("small", INLINE_STR_RELEASES_PER_FUNCTION, "str"); const large = body("large", INLINE_STR_RELEASES_PER_FUNCTION + 1, "str"); diff --git a/packages/compiler/src/backend/llvm/shapes.ts b/packages/compiler/src/backend/llvm/shapes.ts index b3f5084d3..2c851b462 100644 --- a/packages/compiler/src/backend/llvm/shapes.ts +++ b/packages/compiler/src/backend/llvm/shapes.ts @@ -692,34 +692,32 @@ export function limitInlineReleases( parts: readonly string[], mode: InlineRcMode | undefined, ): string[] { - const dyn = limitInlineReleaseFamily(parts, "dyn", INLINE_DYN_RELEASES_PER_FUNCTION); - return mode === "release" - ? limitInlineReleaseFamily(dyn, "str", INLINE_STR_RELEASES_PER_FUNCTION) - : dyn; -} - -function limitInlineReleaseFamily( - parts: readonly string[], - family: "dyn" | "str", - max: number, -): string[] { - const call = `call void @${inlineRcName(family, "release")}(`; - const runtime = `call void @${INLINE_RC_FAMILIES[family]!.release}(`; + const families: readonly ("dyn" | "str")[] = mode === "release" ? ["dyn", "str"] : ["dyn"]; + const limits = families.map((family) => + family === "dyn" ? INLINE_DYN_RELEASES_PER_FUNCTION : INLINE_STR_RELEASES_PER_FUNCTION, + ); + const calls = families.map((family) => `${inlineRcName(family, "release")}(`); + const runtimes = families.map((family) => `call void @${INLINE_RC_FAMILIES[family]!.release}(`); + const head = "call void @"; + const prefix = `${head}${inlineRcName("", "release")}`; // The function each release site belongs to: the last `define` line - // before it, counted over the joined text. + // before it, counted over the joined text. One walk over each part finds + // the sites of every family. const sites = ( part: string, - each: (offset: number, fn: number) => void, + each: (offset: number, family: number, fn: number) => void, first: number, ): number => { let fn = first; let define = part.startsWith("define ") ? 0 : part.indexOf("\ndefine "); - for (let at = part.indexOf(call); at >= 0; at = part.indexOf(call, at + call.length)) { + for (let at = part.indexOf(prefix); at >= 0; at = part.indexOf(prefix, at + prefix.length)) { + const family = calls.findIndex((call) => part.startsWith(call, at + head.length)); + if (family < 0) continue; while (define >= 0 && define < at) { fn++; define = part.indexOf("\ndefine ", define + 1); } - each(at, fn); + each(at, family, fn); } while (define >= 0) { fn++; @@ -727,29 +725,47 @@ function limitInlineReleaseFamily( } return fn; }; - const counts = new Map(); + const counts = families.map(() => new Map()); + // The functions each part's sites can belong to. + const spans: number[] = []; let fn = 0; - for (const part of parts) fn = sites(part, (_, f) => counts.set(f, (counts.get(f) ?? 0) + 1), fn); - const over = new Set([...counts].filter(([, n]) => n > max).map(([f]) => f)); - if (over.size === 0) return [...parts]; - fn = 0; - return parts.map((part) => { - const cuts: number[] = []; + for (const part of parts) { + spans.push(fn); fn = sites( part, - (at, f) => { - if (over.has(f)) cuts.push(at); - }, + (_, family, f) => counts[family]!.set(f, (counts[family]!.get(f) ?? 0) + 1), fn, ); + spans.push(fn); + } + const over = counts.map( + (count, family) => new Set([...count].filter(([, n]) => n > limits[family]!).map(([f]) => f)), + ); + if (over.every((set) => set.size === 0)) return [...parts]; + const anyOver = new Set(over.flatMap((set) => [...set])); + const touches = (from: number, to: number): boolean => { + for (let f = from; f <= to; f++) if (anyOver.has(f)) return true; + return false; + }; + return parts.map((part, index) => { + const from = spans[2 * index]!; + if (!touches(from, spans[2 * index + 1]!)) return part; + const cuts: { at: number; family: number }[] = []; + sites( + part, + (at, family, f) => { + if (over[family]!.has(f)) cuts.push({ at, family }); + }, + from, + ); if (cuts.length === 0) return part; let out = ""; - let from = 0; - for (const at of cuts) { - out += part.slice(from, at) + runtime; - from = at + call.length; + let next = 0; + for (const cut of cuts) { + out += part.slice(next, cut.at) + runtimes[cut.family]!; + next = cut.at + head.length + calls[cut.family]!.length; } - return out + part.slice(from); + return out + part.slice(next); }); } diff --git a/packages/compiler/src/backend/may-throw.ts b/packages/compiler/src/backend/may-throw.ts index 0cfab9244..693d20b3b 100644 --- a/packages/compiler/src/backend/may-throw.ts +++ b/packages/compiler/src/backend/may-throw.ts @@ -93,7 +93,7 @@ export function computeMayThrow( const classes = new Map((mod.classes ?? []).map((cls) => [cls.name, cls])); const constructorClasses = new Map(); const classValueCallers = new Map>(); - const tdzGlobals = (mod.globals ?? []).filter((g) => g.tdz).map((g) => g.id); + const tdzGlobals = new Set((mod.globals ?? []).filter((g) => g.tdz).map((g) => g.id)); for (const cls of mod.classes ?? []) { constructorClasses.set(`%${cls.name}.constructor`, cls.name); for (const m of cls.methods ?? []) { @@ -113,11 +113,10 @@ export function computeMayThrow( if (explain) f.calleeLocs = new Map(); // TDZ reads and non-initializing writes can throw ReferenceError. // Capture locals carry the same flag as the declaring binding. - const tdzIds = new Set([...tdzGlobals, ...fn.locals.filter((l) => l.tdz).map((l) => l.id)]); - const mutableTdzIds = new Set([ - ...tdzGlobals, - ...fn.locals.filter((l) => l.tdz && l.mutable).map((l) => l.id), - ]); + // Every TDZ global counts for both reads and writes; the module's set is + // shared by every function. + const tdzLocals = new Set(fn.locals.filter((l) => l.tdz).map((l) => l.id)); + const mutableTdzLocals = new Set(fn.locals.filter((l) => l.tdz && l.mutable).map((l) => l.id)); // Traverse typed executable nodes without copying the IR into unknown. const visit = (rec: IrExpr | IrStmt): boolean => { const threw = f.throws; @@ -133,13 +132,17 @@ export function computeMayThrow( case "varRef": case "incDec": case "assignExpr": - if (tdzIds.has(rec.localId)) f.throws = true; + if (tdzGlobals.has(rec.localId) || tdzLocals.has(rec.localId)) f.throws = true; break; case "assign": // A declaration is allowed to fill an empty box. Legacy const // TDZ stores also initialize; only mutable subsequent stores // introduce the new write-side exception edge. - if (rec.initializes !== true && mutableTdzIds.has(rec.localId)) f.throws = true; + if ( + rec.initializes !== true && + (tdzGlobals.has(rec.localId) || mutableTdzLocals.has(rec.localId)) + ) + f.throws = true; break; case "dynCheck": case "caughtCheck": diff --git a/packages/compiler/src/ir/callback-specialization.ts b/packages/compiler/src/ir/callback-specialization.ts index 6038ccaa5..4764754af 100644 --- a/packages/compiler/src/ir/callback-specialization.ts +++ b/packages/compiler/src/ir/callback-specialization.ts @@ -130,10 +130,7 @@ export function specializeCallbackCalls( const { written, closureWritten } = rebindings(mod); const copies = new Map(); const candidates = new Map(); - let totalNodes = 0; for (const fn of mod.functions) { - const nodes = countNodes(fn); - totalNodes += nodes; if ( !(fn.ownsPrototype || fn.name.startsWith("%arr.")) || fn.name === mod.entry || @@ -142,10 +139,11 @@ export function specializeCallbackCalls( fn.captures !== undefined || fn.classCaptures?.length || fn.fastRead !== undefined || - fn.speculativeDispatch || - nodes > MAX_FUNCTION_NODES + fn.speculativeDispatch ) continue; + const nodes = countNodes(fn); + if (nodes > MAX_FUNCTION_NODES) continue; const rebound = written.get(fn.name)!; const slots: number[] = []; fn.params.forEach((param, index) => { @@ -165,19 +163,30 @@ export function specializeCallbackCalls( const definitions = new Map>(); const globalIds = new Set(mod.globals?.map((global) => global.id)); const globalWrites = new Map(); + // The sum of countNodes over every function, counted by the walk below. + let totalNodes = 0; for (const fn of mod.functions) { + let nodes = 0; const values = new Map(); const counts = new Map(); const record = (id: string, value: IrExpr | null, weight: number): void => { counts.set(id, (counts.get(id) ?? 0) + weight); - if (value) values.set(id, [...(values.get(id) ?? []), value]); + if (value) { + const list = values.get(id); + if (list === undefined) values.set(id, [value]); + else list.push(value); + } }; everyStmtList(fn.body, { stmt: (stmt) => { + nodes++; if (stmt.kind === "varDecl") record(stmt.localId, stmt.init, stmt.init ? 1 : 0); if (stmt.kind === "assign") { - if (globalIds.has(stmt.localId)) - globalWrites.set(stmt.localId, [...(globalWrites.get(stmt.localId) ?? []), stmt.value]); + if (globalIds.has(stmt.localId)) { + const list = globalWrites.get(stmt.localId); + if (list === undefined) globalWrites.set(stmt.localId, [stmt.value]); + else list.push(stmt.value); + } record(stmt.localId, stmt.value, 1); } if (stmt.kind === "forOf") record(stmt.localId, null, 2); @@ -186,6 +195,7 @@ export function specializeCallbackCalls( return true; }, expr: (expr) => { + nodes++; if (expr.kind === "assignExpr" || expr.kind === "incDec") { record(expr.localId, null, 2); if (globalIds.has(expr.localId)) globalWrites.set(expr.localId, []); @@ -208,6 +218,7 @@ export function specializeCallbackCalls( ) single.set(id, list[0]!); definitions.set(fn.name, single); + totalNodes += Math.min(nodes, MAX_FUNCTION_NODES + 1); } const globalDefinitions = new Map(); for (const global of mod.globals ?? []) diff --git a/packages/compiler/src/ir/capture-values.ts b/packages/compiler/src/ir/capture-values.ts index 779ca1b76..bcc587f1b 100644 --- a/packages/compiler/src/ir/capture-values.ts +++ b/packages/compiler/src/ir/capture-values.ts @@ -538,7 +538,7 @@ export class CaptureFlow { facts.callees.add(e.callee.localId); break; } - return everyExprChild(e, expr, (s) => stmt(s, false)); + return everyExprChild(e, expr, nested); }; // `split` mirrors rewrite(): only statement lists reached through // ordinary nesting from the body are expanded. Switch cases share one @@ -599,8 +599,10 @@ export class CaptureFlow { list(s.body, split) ); } - return everyStmtChild(s, expr, (child) => stmt(child, false)); + return everyStmtChild(s, expr, nested); }; + // Statements nested in expressions and in other statements' heads. + const nested = (s: IrStmt): boolean => stmt(s, false); list(fn.body, true); return facts; } diff --git a/packages/compiler/src/ir/self-captures.ts b/packages/compiler/src/ir/self-captures.ts index e009e9a11..74f634d4a 100644 --- a/packages/compiler/src/ir/self-captures.ts +++ b/packages/compiler/src/ir/self-captures.ts @@ -118,7 +118,12 @@ export function resolveSelfCaptures(mod: IrModule): IrModule { } if (resolved.size === 0) return mod; + // Only the resolved bodies and the functions that create their closures + // change. + const touched = new Set(resolved.keys()); + for (const name of resolved.keys()) for (const site of sites.get(name)!) touched.add(site.owner); const rewritten = mod.functions.map((fn) => { + if (!touched.has(fn.name)) return fn; const own = resolved.get(fn.name); const self = own === undefined ? null : fn.captures![own]!; let changed = false;