diff --git a/packages/compiler/src/backend/llvm/emit-function.ts b/packages/compiler/src/backend/llvm/emit-function.ts index 3be3d979f..427e439ad 100644 --- a/packages/compiler/src/backend/llvm/emit-function.ts +++ b/packages/compiler/src/backend/llvm/emit-function.ts @@ -31,6 +31,7 @@ import { findLocalStackUnions } from "./stack-unions.js"; import { findLocalStackObjects, findReassignedStackObjects } from "./stack-objects.js"; import { findLocalUnionStorage } from "./local-union-storage.js"; import { INT32_RANGE, analyzeIntegerRangeFacts } from "../../ir/integer-ranges.js"; +import { beginLoopEntryVersions } from "./loop-entry-versions.js"; import { findBytesBounds } from "./bytes-bounds.js"; import { findInt32Locals } from "./int32-locals.js"; import { findIntegerViews } from "./integer-views.js"; @@ -246,6 +247,7 @@ export function emitFunction(emitter: LlEmitter, fn: IrFunction): string { const rangeFacts = analyzeIntegerRangeFacts(numericFn, slotFacts); emitter.integerRanges = rangeFacts.ranges; emitter.integerLoopRanges = rangeFacts.loopRanges; + beginLoopEntryVersions(emitter, numericFn, slotFacts, rangeFacts); emitter.int32Elements = slotFacts.elements ?? null; emitter.integerOrNaNLocals = findIntegerOrNaNLocals(emitter, numericFn); emitter.bytesBounds = findBytesBounds(numericFn, emitter.integerRanges); diff --git a/packages/compiler/src/backend/llvm/emit-statements.ts b/packages/compiler/src/backend/llvm/emit-statements.ts index 1eeceff4a..74c84ae86 100644 --- a/packages/compiler/src/backend/llvm/emit-statements.ts +++ b/packages/compiler/src/backend/llvm/emit-statements.ts @@ -46,6 +46,7 @@ import { emitBorrowedInput } from "./borrowed-inputs.js"; import { f64Lit, llvmCommentText, loadArrCount, SCR_ARR_FIELD, sizeToDouble } from "./common.js"; import { planInt32KindLoop, type Int32KindLoopPlan } from "./int32-kind-loops.js"; import { withinInt32 } from "../../ir/integer-ranges.js"; +import { emitLoopEntryVersion, loopEntryVersion } from "./loop-entry-versions.js"; import { resultSlotStore } from "../result-slots.js"; import { matchIntegerArrayForLoop, @@ -2349,6 +2350,13 @@ export function emitLoopCopies( exit: string, emitCopy: (exit: string) => void, ): void { + const entry = allowed ? loopEntryVersion(emitter, loop) : null; + if (entry) { + emitLoopEntryVersion(emitter, loop, entry, () => + emitLoopCopies(emitter, loop, allowed, exit, emitCopy), + ); + return; + } const plan = allowed && emitter.speculativeLoopsEnabled && diff --git a/packages/compiler/src/backend/llvm/emitter.ts b/packages/compiler/src/backend/llvm/emitter.ts index 425c4f4d4..2d2ebfb8d 100644 --- a/packages/compiler/src/backend/llvm/emitter.ts +++ b/packages/compiler/src/backend/llvm/emitter.ts @@ -1,4 +1,5 @@ import type { TargetCpu } from "../target-cpu.js"; +import { loopEntryInlines } from "./loop-entry-versions.js"; import type { FunctionLayout } from "./function-layout.js"; import { relieveHeapPressure } from "../../heap-pressure.js"; import { wasmBoundsChecks } from "./wasm-modules.js"; @@ -4009,8 +4010,10 @@ export class LlEmitter { emitDirectCall(line: string, callee: string): boolean { const B = this.B; // A small leaf called only from loops inlines into each of them, past - // LLVM's cost threshold (loop-callees.ts). Debug builds keep the call. - if (this.debug === null && this.loopCallees.has(callee)) { + // LLVM's cost threshold (loop-callees.ts), and so does a function called + // only from a loop's entry-range version (loop-entry-versions.ts). + // Debug builds keep the call. + if (this.debug === null && (this.loopCallees.has(callee) || loopEntryInlines(this, callee))) { const comment = line.indexOf(" ; "); line = comment < 0 diff --git a/packages/compiler/src/backend/llvm/loop-entry-versions.ts b/packages/compiler/src/backend/llvm/loop-entry-versions.ts new file mode 100644 index 000000000..2c448cae4 --- /dev/null +++ b/packages/compiler/src/backend/llvm/loop-entry-versions.ts @@ -0,0 +1,231 @@ +/* Loop-entry range versions (ir/loop-entry-ranges.ts): a while loop whose + * integer work depends on locals entering it without range facts runs as + * two copies. A test on entry checks that each such local holds an integer + * in its assumed range; the fast copy is emitted under the facts that hold + * then, and the other copy is the loop as it would be emitted otherwise. */ +import type { IrFunction, IrModule, IrStmt } from "../../ir/ir.js"; +import { everyStmt, everyStmtList } from "../../ir/traverse.js"; +import { exprFunctionReference, moduleFunctionReferences } from "../../ir/function-references.js"; +import type { IntegerRangeFacts, IntegerSlotFacts } from "../../ir/integer-ranges.js"; +import { findLoopEntryVersions, type LoopEntryVersion } from "../../ir/loop-entry-ranges.js"; +import { f64Lit } from "./common.js"; +import type { LlEmitter } from "./emitter.js"; +import { defineModuleState } from "./emitter-state.js"; + +interface LoopEntryState { + fn: IrFunction | null; + facts: IntegerRangeFacts | null; + slots: IntegerSlotFacts; + /** Found on the first while loop the function emits. */ + versions: Map | null; + /** Loops whose copies are being emitted: a loop is versioned once. */ + readonly versioning: Set; + /** Direct calls per module function, or -1 for a function some value or + * the module's metadata references. */ + calls: Map | null; + /** The callees whose calls the fast copy being emitted inlines. */ + inlined: ReadonlySet | null; +} + +const ENTRY_VERSIONS = defineModuleState(() => ({ + fn: null, + facts: null, + slots: {}, + versions: null, + versioning: new Set(), + calls: null, + inlined: null, +})); + +/** The most IR nodes a callee may have for its call in a fast copy to be + * inlined (singleCallees). */ +const MAX_INLINED_NODES = 400; + +/** Sets the state for the function about to be emitted: `numericFn` and the + * facts emitFunction computed for it. Debug builds keep every loop as is. */ +export function beginLoopEntryVersions( + emitter: LlEmitter, + numericFn: IrFunction, + slots: IntegerSlotFacts, + facts: IntegerRangeFacts, +): void { + const state = ENTRY_VERSIONS.of(emitter); + state.fn = emitter.debug === null ? numericFn : null; + state.slots = slots; + state.facts = facts; + state.versions = null; + state.versioning.clear(); +} + +/** The entry version of `loop`, when it has one and the emitter can test + * its locals: plain double slots of the current function, outside a + * speculative copy and a coroutine. */ +export function loopEntryVersion(emitter: LlEmitter, loop: IrStmt): LoopEntryVersion | null { + if (loop.kind !== "while") return null; + const state = ENTRY_VERSIONS.of(emitter); + if (state.fn === null || state.facts === null || state.versioning.has(loop)) return null; + if ( + !emitter.speculativeLoopsEnabled || + emitter.speculation !== null || + emitter.speculationResume !== null || + emitter.currentCoro !== null + ) + return null; + state.versions ??= findLoopEntryVersions(state.fn, state.facts, state.slots); + const version = state.versions.get(loop); + if (!version) return null; + for (const id of version.entry.keys()) + if (!emitter.currentLocals.has(id) || !emitter.speculativeLocal(id)) return null; + return version; +} + +/** Emit the entry test and both copies of `loop`. `emitCopies` emits the + * loop as it would be emitted without the version; it runs once under the + * version's facts and once as is. */ +export function emitLoopEntryVersion( + emitter: LlEmitter, + loop: IrStmt, + version: LoopEntryVersion, + emitCopies: () => void, +): void { + const B = emitter.B; + let ok: string | null = null; + for (const [id, range] of version.entry) { + // An integer in [min, max] other than -0: the ordered comparisons + // reject NaN, and the bits of the value and its round trip agree only + // for integers that are not -0. A local the loop writes before it reads + // may be unset on entry, and a conversion out of range is poison: + // `freeze` makes both an arbitrary fixed value. + const raw = B.tmp(), + value = B.tmp(), + low = B.tmp(), + high = B.tmp(), + converted = B.tmp(), + integer = B.tmp(), + back = B.tmp(), + bits = B.tmp(), + backBits = B.tmp(), + exact = B.tmp(), + inRange = B.tmp(), + passed = B.tmp(); + B.line(`${raw} = load double, ptr ${emitter.binding(id).slot}`); + B.line(`${value} = freeze double ${raw}`); + B.line(`${low} = fcmp oge double ${value}, ${f64Lit(range.min)}`); + B.line(`${high} = fcmp ole double ${value}, ${f64Lit(range.max)}`); + B.line(`${converted} = fptosi double ${value} to i64`); + B.line(`${integer} = freeze i64 ${converted}`); + B.line(`${back} = sitofp i64 ${integer} to double`); + B.line(`${bits} = bitcast double ${value} to i64`); + B.line(`${backBits} = bitcast double ${back} to i64`); + B.line(`${exact} = icmp eq i64 ${bits}, ${backBits}`); + B.line(`${inRange} = and i1 ${low}, ${high}`); + B.line(`${passed} = and i1 ${inRange}, ${exact}`); + if (ok === null) ok = passed; + else { + const both = B.tmp(); + B.line(`${both} = and i1 ${ok}, ${passed}`); + ok = both; + } + } + const fast = B.newLabel("entry.fast"), + slow = B.newLabel("entry.slow"); + // The test is expected to pass, which keeps the ordinary copy's calls + // cold for the inliner. + let likely = "true"; + if (ok !== null) { + emitter.declare("declare i1 @llvm.expect.i1(i1, i1)"); + likely = B.tmp(); + B.line(`${likely} = call i1 @llvm.expect.i1(i1 ${ok}, i1 true)`); + } + B.condBr(likely, fast, slow); + const state = ENTRY_VERSIONS.of(emitter); + const originalRanges = emitter.integerRanges; + const originalLoopRanges = emitter.integerLoopRanges; + state.versioning.add(loop); + try { + B.startBlock(fast); + // An enclosing version may have narrowed a range already. + const ranges = new Map(originalRanges); + for (const [node, range] of version.ranges) { + const known = originalRanges.get(node); + ranges.set( + node, + known + ? { min: Math.max(known.min, range.min), max: Math.min(known.max, range.max) } + : range, + ); + } + emitter.integerRanges = ranges; + state.inlined = singleCallees(emitter, state, loop); + if (version.loopRanges) { + const loopRanges = new Map(originalLoopRanges); + loopRanges.set(loop, version.loopRanges); + emitter.integerLoopRanges = loopRanges; + } + emitCopies(); + state.inlined = null; + emitter.integerRanges = originalRanges; + emitter.integerLoopRanges = originalLoopRanges; + B.startBlock(slow); + emitCopies(); + } finally { + emitter.integerRanges = originalRanges; + emitter.integerLoopRanges = originalLoopRanges; + state.versioning.delete(loop); + state.inlined = null; + } +} + +/** Whether the fast copy being emitted inlines its calls of `callee`. */ +export function loopEntryInlines(emitter: LlEmitter, callee: string): boolean { + return ENTRY_VERSIONS.of(emitter).inlined?.has(callee) === true; +} + +/** The functions the loop calls that have no other call or reference in + * the module and at most MAX_INLINED_NODES nodes. LLVM inlines a function's + * only call at almost any size; the two copies give each such function a + * second call, so the fast copy's call is marked to keep that inlining. */ +function singleCallees(emitter: LlEmitter, state: LoopEntryState, loop: IrStmt): Set { + const callees = new Set(); + everyStmt(loop, { + expr: (e) => { + if (e.kind === "call") callees.add(e.callee); + return true; + }, + stmt: () => true, + }); + const result = new Set(); + if (callees.size === 0) return result; + state.calls ??= countCalls(emitter.mod); + for (const name of callees) { + if (state.calls.get(name) !== 1 || name === state.fn?.name) continue; + const fn = emitter.mod.functions.find((f) => f.name === name); + if (fn && !fn.async && !fn.generator && nodeCount(fn.body) <= MAX_INLINED_NODES) + result.add(name); + } + return result; +} + +function countCalls(mod: IrModule): Map { + const calls = new Map(); + const referenced = new Set(moduleFunctionReferences(mod)); + for (const fn of mod.functions) + everyStmtList(fn.body, { + expr: (e) => { + if (e.kind === "call") calls.set(e.callee, (calls.get(e.callee) ?? 0) + 1); + const name = exprFunctionReference(e); + if (name !== undefined) referenced.add(name); + return true; + }, + stmt: () => true, + }); + for (const name of referenced) calls.set(name, -1); + return calls; +} + +function nodeCount(body: IrStmt[]): number { + let count = 0; + const visit = (): boolean => ++count <= MAX_INLINED_NODES; + everyStmtList(body, { expr: visit, stmt: visit }); + return count; +} diff --git a/packages/compiler/src/ir/integer-ranges.ts b/packages/compiler/src/ir/integer-ranges.ts index 51c0a929b..23e2df90a 100644 --- a/packages/compiler/src/ir/integer-ranges.ts +++ b/packages/compiler/src/ir/integer-ranges.ts @@ -23,6 +23,11 @@ export interface IntegerRangeFacts { ranges: IntegerRanges; loopRanges: ReadonlyMap>; } +/** Ranges assumed for locals on entry to while loops (loop-entry-ranges.ts): + * the facts inside such a loop hold in a copy of it that runs only after a + * test proves every assumed local is an integer in its range. After the + * loop, its assumed locals and its writes carry no facts. */ +export type LoopEntryAssumptions = ReadonlyMap>; const SIGNED: IntegerRange = { min: -2147483648, max: 2147483647 }; const UNSIGNED: IntegerRange = { min: 0, max: 4294967295 }; /** Int32 and uint32 at once: the bound under which `%` (modeled only for @@ -309,6 +314,7 @@ export function analyzeIntegerRanges(fn: IrFunction, slots: IntegerSlotFacts = { export function analyzeIntegerRangeFacts( fn: IrFunction, slots: IntegerSlotFacts = {}, + entries?: LoopEntryAssumptions, ): IntegerRangeFacts { const ranges = new Map(); const loopRanges = new Map>(); @@ -1074,6 +1080,8 @@ export function analyzeIntegerRangeFacts( } case "while": case "doWhile": { + const assumed = entries?.get(s); + if (assumed) for (const [id, range] of assumed) if (!facts.has(id)) set(facts, id, range); // Counters the condition bounds keep their range at every read // inside and after the loop. const guarded = @@ -1102,6 +1110,10 @@ export function analyzeIntegerRangeFacts( } else body(s.body, new Map(loop)); havocWith(facts, [s], facts, guarded); for (const [id, range] of guarded) set(facts, id, range); + if (assumed) { + invalidate(facts, [s]); + for (const id of assumed.keys()) facts.delete(id); + } break; } case "forOf": { diff --git a/packages/compiler/src/ir/loop-entry-ranges.test.ts b/packages/compiler/src/ir/loop-entry-ranges.test.ts new file mode 100644 index 000000000..ad1b78ef6 --- /dev/null +++ b/packages/compiler/src/ir/loop-entry-ranges.test.ts @@ -0,0 +1,143 @@ +import { expect, test } from "vitest"; +import { analyzeIntegerRangeFacts } from "./integer-ranges.js"; +import { findLoopEntryVersions, LOOP_ENTRY_RANGE } from "./loop-entry-ranges.js"; +import { BOOL, F64, type IrExpr, type IrFunction, type IrStmt } from "./ir.js"; + +const loc = { file: "test.ts", start: 0, end: 0 }; +const num = (value: number): IrExpr => ({ loc, kind: "numLit", value, type: F64 }); +const ref = (localId: string): IrExpr => ({ loc, kind: "varRef", localId, type: F64 }); +const bin = (op: "+" | "-" | "*" | "%", left: IrExpr, right: IrExpr): IrExpr => ({ + loc, + kind: "bin", + op, + left, + right, + type: F64, +}); +const condition = (op: "<" | "===", left: IrExpr, right: IrExpr): IrExpr => ({ + loc, + kind: "bin", + op, + left, + right, + type: BOOL, +}); +const set = (localId: string, value: IrExpr): IrStmt => ({ loc, kind: "assign", localId, value }); +const bump = (localId: string): IrStmt => ({ + loc, + kind: "exprStmt", + expr: { loc, kind: "incDec", op: "+", prefix: false, localId, type: F64 }, +}); +const observe = (value: IrExpr): IrStmt => ({ loc, kind: "exprStmt", expr: value }); +const fn = (body: IrStmt[], locals: string[]): IrFunction => ({ + loc, + name: "f", + params: [], + returnType: F64, + locals: locals.map((id) => ({ id, name: id, type: F64, mutable: true })), + body, +}); + +/** base = unknown; breaks = unknown; count = 0; i = 0; + * while (i < 300) { ...extra; i++; count++; last = (count + base - breaks) % 8; + * if (last === 0) breaks++; } + * observe(base) */ +function lineBreaks(extra: IrStmt[] = [], locals: string[] = []) { + const remainder = bin("%", bin("-", bin("+", ref("count"), ref("base")), ref("breaks")), num(8)); + const after = ref("base"); + const loop: IrStmt = { + loc, + kind: "while", + cond: condition("<", ref("i"), num(300)), + body: [ + ...extra, + bump("i"), + bump("count"), + set("last", remainder), + { + loc, + kind: "if", + cond: condition("===", ref("last"), num(0)), + then: [bump("breaks")], + else_: null, + }, + ], + }; + const f = fn( + [ + set("base", ref("unknown")), + set("breaks", ref("unknown")), + set("count", num(0)), + set("i", num(0)), + set("last", num(1)), + loop, + observe(after), + ], + ["base", "breaks", "count", "i", "last", ...locals], + ); + return { f, loop, remainder, after }; +} + +test("versions a while loop whose remainder reads locals without entry facts", () => { + const { f, loop, remainder } = lineBreaks(); + const facts = analyzeIntegerRangeFacts(f); + expect(facts.ranges.get(remainder)).toBeFalsy(); + const version = findLoopEntryVersions(f, facts).get(loop); + expect(version).toBeDefined(); + expect([...version!.entry]).toEqual([ + ["base", LOOP_ENTRY_RANGE], + ["breaks", LOOP_ENTRY_RANGE], + ]); + // The dividend is an exact integer; `%` itself is modeled only for + // non-negative dividends. + const dividend = (remainder as Extract).left; + expect(version!.ranges.get(dividend)).toBeDefined(); + // The loop's counters, `breaks` among them, are bounded in the copy. + expect(version!.loopRanges?.get("breaks")).toEqual({ + min: LOOP_ENTRY_RANGE.min, + max: LOOP_ENTRY_RANGE.max + 300, + }); +}); + +test("the entry test covers only the locals the remainder depends on", () => { + // acc = acc + scale reads two more locals without facts. + const { f, loop } = lineBreaks( + [set("acc", bin("+", ref("acc"), ref("scale")))], + ["acc", "scale"], + ); + f.body.unshift(set("acc", ref("unknown")), set("scale", ref("unknown"))); + const version = findLoopEntryVersions(f, analyzeIntegerRangeFacts(f)).get(loop); + expect([...version!.entry.keys()]).toEqual(["base", "breaks"]); +}); + +test("assumed facts do not leave the loop", () => { + const { f, loop, after } = lineBreaks(); + const assumed = new Map([[loop, new Map([["base", LOOP_ENTRY_RANGE]])]]); + expect(analyzeIntegerRangeFacts(f, {}, assumed).ranges.get(after)).toBeFalsy(); +}); + +test("loops without a remainder the assumption proves keep one copy", () => { + // A local the loop makes fractional, a product without a remainder, and a + // remainder the function already proves. + const fractional = lineBreaks([set("base", bin("+", ref("base"), num(0.5)))]); + expect(findLoopEntryVersions(fractional.f, analyzeIntegerRangeFacts(fractional.f)).size).toBe(0); + const product: IrStmt = { + loc, + kind: "while", + cond: condition("<", ref("i"), num(300)), + body: [set("acc", bin("*", ref("acc"), ref("scale"))), bump("i")], + }; + const products = fn( + [set("acc", ref("unknown")), set("scale", ref("unknown")), set("i", num(0)), product], + ["acc", "scale", "i"], + ); + expect(findLoopEntryVersions(products, analyzeIntegerRangeFacts(products)).size).toBe(0); + const proven: IrStmt = { + loc, + kind: "while", + cond: condition("<", ref("i"), num(300)), + body: [set("acc", bin("%", ref("i"), num(7))), bump("i")], + }; + const known = fn([set("acc", ref("unknown")), set("i", num(0)), proven], ["acc", "i"]); + expect(findLoopEntryVersions(known, analyzeIntegerRangeFacts(known)).size).toBe(0); +}); diff --git a/packages/compiler/src/ir/loop-entry-ranges.ts b/packages/compiler/src/ir/loop-entry-ranges.ts new file mode 100644 index 000000000..a5b7628be --- /dev/null +++ b/packages/compiler/src/ir/loop-entry-ranges.ts @@ -0,0 +1,218 @@ +import type { IrExpr, IrFunction, IrStmt } from "./ir.js"; +import { everyExpr, everyStmt, everyStmtList } from "./traverse.js"; +import { + analyzeIntegerRangeFacts, + type IntegerRange, + type IntegerRangeFacts, + type IntegerSlotFacts, + type LoopEntryAssumptions, +} from "./integer-ranges.js"; + +type WhileLoop = Extract; + +/** The range a versioned loop requires of each assumed local on entry. A + * loop's counters stay below a typed array or string length (at most 2^48), + * so sums of a few such values and assumed locals stay exact integers. */ +export const LOOP_ENTRY_RANGE: IntegerRange = { min: -(2 ** 48), max: 2 ** 48 }; + +/** A copy of a while loop that runs when every local in `entry` holds an + * integer in its range on entry (never -0). `ranges` are the facts that + * hold for the loop's expressions in that copy and that the function's own + * analysis does not prove; `loopRanges` are the loop's bounded bindings + * there. */ +export interface LoopEntryVersion { + entry: ReadonlyMap; + ranges: ReadonlyMap; + loopRanges: ReadonlyMap | undefined; +} + +/** Versions of innermost while loops whose integer work depends on locals + * that enter the loop without range facts: copies of module globals, + * parameters, fields or call results (`const base = offset;` before a loop + * that computes `(count + base) % 60`). + * + * The function is analyzed again with each number local the loop reads + * without a fact assumed in LOOP_ENTRY_RANGE at the loop's entry. A loop is + * versioned when that analysis proves both operands of a remainder that the + * function's own analysis leaves unproven. Only remainders qualify: the + * operands of a product or a sum of such locals are as often fractional, + * and a loop whose test fails runs the ordinary copy in a larger function. + * The entry test covers only the assumed locals that such an operation reads, directly, + * through the loop's writes of other locals, or through the loop condition; + * when that drops any, the function is analyzed once more with the smaller + * sets, so the facts never rest on a local the test does not check. No + * fact leaves an assumed loop, so the loops' assumptions are independent. */ +export function findLoopEntryVersions( + fn: IrFunction, + facts: IntegerRangeFacts, + slots: IntegerSlotFacts = {}, +): Map { + const result = new Map(); + if (fn.async || fn.generator) return result; + const captures = new Set( + [...(fn.captures ?? []), ...(fn.classCaptures ?? [])].map((c) => c.localId), + ); + const eligible = new Set( + fn.locals + .filter((l) => l.type.kind === "f64" && !l.boxed && !l.tdz && !captures.has(l.id)) + .map((l) => l.id), + ); + const known = (e: IrExpr): boolean => !!facts.ranges.get(e); + let candidates = new Map>(); + everyStmtList(fn.body, { + expr: () => true, + stmt: (s) => { + if (s.kind !== "while" || containsLoop(s.body)) return true; + const assumed = entryCandidates(s, eligible, known); + if (assumed.size > 0) candidates.set(s, assumed); + return true; + }, + }); + if (candidates.size === 0) return result; + let versioned = analyzeIntegerRangeFacts(fn, slots, assumptionsOf(candidates)); + const pruned = new Map>(); + let dropped = false; + for (const [loop, entry] of candidates) { + const roots = gainedOperands(loop, known, versioned); + if (roots.length === 0) continue; + const relevant = relevantLocals(loop, roots); + const kept = new Map(); + for (const [id, range] of entry) if (relevant.has(id)) kept.set(id, range); + if (kept.size < entry.size) dropped = true; + if (kept.size > 0) pruned.set(loop, kept); + } + if (dropped) { + candidates = pruned; + if (candidates.size === 0) return result; + versioned = analyzeIntegerRangeFacts(fn, slots, assumptionsOf(candidates)); + } + for (const [loop, entry] of candidates) { + if (gainedOperands(loop, known, versioned).length === 0) continue; + const ranges = new Map(); + const visit = (e: IrExpr): boolean => { + const range = versioned.ranges.get(e); + const prior = facts.ranges.get(e); + if (range && !prior) ranges.set(e, range); + else if (range && prior && (range.min > prior.min || range.max < prior.max)) + ranges.set(e, { min: Math.max(range.min, prior.min), max: Math.min(range.max, prior.max) }); + return true; + }; + everyStmt(loop, { expr: visit, stmt: () => true }); + result.set(loop, { entry, ranges, loopRanges: versioned.loopRanges.get(loop) }); + } + return result; +} + +function assumptionsOf( + candidates: ReadonlyMap>, +): LoopEntryAssumptions { + const result = new Map>(); + for (const [loop, entry] of candidates) result.set(loop, entry); + return result; +} + +/** Number locals the loop reads without a range fact and does not declare, + * when the loop has a remainder without one. */ +function entryCandidates( + loop: WhileLoop, + eligible: ReadonlySet, + known: (e: IrExpr) => boolean, +): Map { + const read = new Set(); + const declared = new Set(); + let integerWork = false; + everyStmt(loop, { + expr: (e) => { + if (e.kind === "varRef" && eligible.has(e.localId) && !known(e)) read.add(e.localId); + else if (e.kind === "bin" && e.op === "%" && !known(e)) integerWork = true; + return true; + }, + stmt: (s) => { + if (s.kind === "varDecl") declared.add(s.localId); + return true; + }, + }); + const result = new Map(); + if (!integerWork) return result; + for (const id of read) if (!declared.has(id)) result.set(id, LOOP_ENTRY_RANGE); + return result; +} + +/** Operands of the loop's remainders that the versioned analysis proves + * where the function's own analysis does not prove both. */ +function gainedOperands( + loop: WhileLoop, + known: (e: IrExpr) => boolean, + versioned: IntegerRangeFacts, +): IrExpr[] { + const operands: IrExpr[] = []; + everyStmt(loop, { + expr: (e) => { + if ( + e.kind === "bin" && + e.op === "%" && + !(known(e.left) && known(e.right)) && + !!versioned.ranges.get(e.left) && + !!versioned.ranges.get(e.right) + ) + operands.push(e.left, e.right); + return true; + }, + stmt: () => true, + }); + return operands; +} + +/** The locals `roots` read, closed over the loop's writes (a local written + * from others reads them) and over its condition (a local the condition + * reads makes every local of the condition relevant). */ +function relevantLocals(loop: WhileLoop, roots: readonly IrExpr[]): Set { + const relevant = new Set(); + const readsOf = (e: IrExpr): string[] => { + const ids: string[] = []; + everyExpr(e, { + expr: (x) => { + if (x.kind === "varRef") ids.push(x.localId); + return true; + }, + stmt: () => true, + }); + return ids; + }; + const written = new Map(); + everyStmt(loop, { + expr: (e) => { + if (e.kind === "assignExpr") + written.set(e.localId, [...(written.get(e.localId) ?? []), e.value]); + return true; + }, + stmt: (s) => { + const value = s.kind === "assign" ? s.value : s.kind === "varDecl" ? s.init : undefined; + if (value && (s.kind === "assign" || s.kind === "varDecl")) + written.set(s.localId, [...(written.get(s.localId) ?? []), value]); + return true; + }, + }); + const condition = readsOf(loop.cond); + const pending: string[] = []; + const add = (id: string): void => { + if (relevant.has(id)) return; + relevant.add(id); + pending.push(id); + }; + for (const root of roots) for (const id of readsOf(root)) add(id); + while (pending.length > 0) { + const id = pending.pop()!; + for (const value of written.get(id) ?? []) for (const other of readsOf(value)) add(other); + if (condition.includes(id)) for (const other of condition) add(other); + } + return relevant; +} + +function containsLoop(body: IrStmt[]): boolean { + return !everyStmtList(body, { + expr: () => true, + stmt: (s) => + s.kind !== "while" && s.kind !== "doWhile" && s.kind !== "for" && s.kind !== "forOf", + }); +} diff --git a/packages/compiler/test/loop-entry-versions.test.ts b/packages/compiler/test/loop-entry-versions.test.ts new file mode 100644 index 000000000..22e41e3e7 --- /dev/null +++ b/packages/compiler/test/loop-entry-versions.test.ts @@ -0,0 +1,45 @@ +import { mkdtemp, readFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { expect, test } from "vitest"; +import { compile, deserializeModule } from "../src/index.js"; +import { emitLlvmModule } from "../src/backend/llvm/emitter.js"; + +test("while loops test their entry values and run integer remainders in the fast copy", async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-entry-versions-")); + try { + const entry = join(import.meta.dirname, "../../../tests/corpus/while-entry-range-versions.ts"); + const outPath = join(dir, "versions.ir.json"); + const result = await compile(entry, { outPath, outDir: dir, outputKind: "ir" }); + if (!result.ok) throw new Error(result.diagnostics.map((d) => d.message).join("\n")); + const mod = deserializeModule(await readFile(outPath, "utf8")); + const ll = emitLlvmModule(mod, { pointerBits: 64 }); + const body = (name: string): string => + ll.match( + new RegExp(`define internal [^\\n]+ @sc_bf_${name}\\([^\\n]*\\{([\\s\\S]*?)\\n\\}`), + )![1]!; + for (const name of ["wrap", "ring", "_x25_Cursor_stripes"]) { + const fn = body(name); + const fast = fn.slice(fn.indexOf("\nentry.fast"), fn.indexOf("\nentry.slow")); + const slow = fn.slice(fn.indexOf("\nentry.slow")); + // The fast copy adds the dividend's integers; the ordinary copy adds + // doubles and tests the sum before its integer remainder. + const adds = (text: string): number => text.split("fadd double").length - 1; + expect(fast, name).toContain("srem i64"); + expect(adds(fast), name).toBeLessThan(adds(slow)); + } + // `complement` has no other caller: the fast copy's call keeps it inlined. + const wrap = body("wrap"); + const fast = wrap.slice(wrap.indexOf("\nentry.fast"), wrap.indexOf("\nentry.slow")); + expect(fast).toMatch(/call i32 @sc_f_complement\.i32\([^)]*\) alwaysinline/); + expect(wrap.slice(wrap.indexOf("\nentry.slow"))).not.toContain("alwaysinline"); + // Debug builds keep one copy. + const debug = emitLlvmModule(mod, { + pointerBits: 64, + debugSources: new Map([[entry, await readFile(entry, "utf8")]]), + }); + expect(debug).not.toContain("entry.fast"); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/while-entry-range-versions.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/while-entry-range-versions.ts.json new file mode 100644 index 000000000..ec9e8eeae --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/while-entry-range-versions.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/while-entry-range-versions.ts" + ], + "diags": [] +} diff --git a/tests/corpus/while-entry-range-versions.ts b/tests/corpus/while-entry-range-versions.ts new file mode 100644 index 000000000..7d96c243a --- /dev/null +++ b/tests/corpus/while-entry-range-versions.ts @@ -0,0 +1,160 @@ +// While loops whose remainder reads locals copied from module globals, +// parameters and fields. Entry values cover integers inside and outside +// [-2^48, 2^48], fractions, -0, NaN and infinities. + +let column = 0; +let lines = 0; + +// A switch with more cases than a small leaf function has statements. +function complement(code: number): number { + switch (code) { + case 65: + return 84; + case 67: + return 71; + case 71: + return 67; + case 84: + return 65; + case 97: + return 116; + case 99: + return 103; + case 103: + return 99; + case 116: + return 97; + case 85: + return 65; + case 117: + return 97; + case 82: + return 89; + case 89: + return 82; + case 75: + return 77; + case 77: + return 75; + } + return 78; +} + +// Copies bytes in reverse, complemented, with a newline after every 8 +// columns counted from the module-level column and line counters. +function wrap(input: Uint8Array, out: Uint8Array): number { + const base = column; + let breaks = lines; + let count = 0; + let o = 0; + let i = 0; + const len = input.length; + let last = 1; + let negativeZeros = 0; + while (i < len) { + out[o] = complement(input[len - 1 - i]!); + i++; + o++; + count++; + last = (count + base - breaks) % 8; + if (Object.is(last, -0)) negativeZeros++; + if (last === 0) { + out[o] = 10; + o++; + count++; + breaks++; + } + } + column = base + count; + lines = breaks; + console.log(` last ${Object.is(last, -0) ? "-0" : last}, ${negativeZeros} of them -0`); + return o; +} + +function text(out: Uint8Array, n: number): string { + let s = ""; + for (let i = 0; i < n; i++) s += out[i] === 10 ? "|" : String.fromCharCode(out[i]!); + return s; +} + +const input = new Uint8Array(29); +const bases = "ACGTTGCAAGCTTCGA"; +for (let i = 0; i < input.length; i++) input[i] = bases.charCodeAt((i * 7) % bases.length); +const out = new Uint8Array(64); + +const starts: [number, number][] = [ + [0, 0], + [5, 2], + [-13, 0], + [-16, 0], + [2 ** 48, 3], + [-(2 ** 48), 0], + [2 ** 48 + 1, 0], + [2 ** 50, 1], + [2 ** 53 - 5, 0], + [2 ** 53, 0], + [2 ** 53 + 2, 1], + [0.5, 0], + [3, 0.25], + [3, 2 ** 49], + [-0, 0], + [0, -0], + [NaN, 0], + [Infinity, 0], + [-Infinity, 0], + [1e300, 0], +]; +for (const [c, l] of starts) { + column = c; + lines = l; + console.log(`start column ${c} lines ${l}`); + const n = wrap(input, out); + console.log(` ${text(out, n)}`); + console.log(` column ${column} lines ${lines}`); +} + +// Two calls in a row: the second starts from the first one's counters. +column = 3; +lines = 0; +wrap(input, out); +const n = wrap(input, out); +console.log(`again ${text(out, n)} column ${column} lines ${lines}`); + +// A parameter and a field as the loop's entry values. +function ring(values: Int32Array, offset: number, size: number): number { + let sum = 0; + let i = 0; + while (i < values.length) { + sum += values[(i + offset) % size]!; + i++; + } + return sum; +} +const values = new Int32Array([3, 1, 4, 1, 5, 9, 2, 6]); +for (const offset of [0, 3, 7, 2 ** 40, 2 ** 49, 1.5, -0, -2]) { + const sum = ring(values, offset, values.length); + console.log(`ring ${offset} ${Number.isNaN(sum) ? "NaN" : sum}`); +} + +class Cursor { + position: number; + constructor(position: number) { + this.position = position; + } + stripes(bytes: Uint8Array): string { + const start = this.position; + let i = 0; + let marks = ""; + while (i < bytes.length) { + const r = (start + i) % 3; + marks += Object.is(r, -0) ? "z" : String(r); + i++; + } + this.position = start + i; + return marks; + } +} +for (const position of [0, 4, -5, 2 ** 48 + 7, 0.75, 2 ** 60]) { + const cursor = new Cursor(position); + console.log(`stripes ${position} ${cursor.stripes(input.subarray(0, 7))} ${cursor.position}`); +} diff --git a/tests/harness/corpus-costs/while-entry-range-versions.ts.json b/tests/harness/corpus-costs/while-entry-range-versions.ts.json new file mode 100644 index 000000000..2a685047e --- /dev/null +++ b/tests/harness/corpus-costs/while-entry-range-versions.ts.json @@ -0,0 +1 @@ +{ "plain": 3600, "san": 3000 }