From 4046b17c650b7fefec930e7769054af64711b8dc Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Mon, 26 Oct 2026 23:02:32 -0500 Subject: [PATCH] Version while loops on the integer range of locals they enter with A while loop whose remainder reads number locals that enter it without range facts, such as copies of module globals, parameters or fields (`const base = column;` before `(count + base - breaks) % 60`), now runs as two copies. On entry, a test checks that each such local holds an integer in [-2^48, 2^48] other than -0. The fast copy uses the facts the integer range analysis proves when the function is analyzed again with those entry ranges assumed: the loop's counters become bounded, and the sums and remainders run as i64 arithmetic. When the test fails, the loop runs as it did before. - ir/loop-entry-ranges.ts picks innermost while loops with a remainder that the assumed ranges prove and the function's own analysis does not. The test covers only the locals the remainder depends on, directly, through the loop's writes or through its condition. - ir/integer-ranges.ts takes the assumed entry ranges per loop, and keeps no fact about the assumed locals or the loop's writes after it. - backend/llvm/loop-entry-versions.ts emits the test and both copies. The test is marked likely. A function called only from the loop gets `alwaysinline` on the fast copy's call, because the second copy would otherwise cost it the inlining of a single call site. reverse-complement (25M bases): 2.08 s -> 1.19 s; Node 3.97 s. --- .../src/backend/llvm/emit-function.ts | 2 + .../src/backend/llvm/emit-statements.ts | 8 + packages/compiler/src/backend/llvm/emitter.ts | 7 +- .../src/backend/llvm/loop-entry-versions.ts | 231 ++++++++++++++++++ packages/compiler/src/ir/integer-ranges.ts | 12 + .../compiler/src/ir/loop-entry-ranges.test.ts | 143 +++++++++++ packages/compiler/src/ir/loop-entry-ranges.ts | 218 +++++++++++++++++ .../compiler/test/loop-entry-versions.test.ts | 45 ++++ .../corpus/while-entry-range-versions.ts.json | 6 + tests/corpus/while-entry-range-versions.ts | 160 ++++++++++++ .../while-entry-range-versions.ts.json | 1 + 11 files changed, 831 insertions(+), 2 deletions(-) create mode 100644 packages/compiler/src/backend/llvm/loop-entry-versions.ts create mode 100644 packages/compiler/src/ir/loop-entry-ranges.test.ts create mode 100644 packages/compiler/src/ir/loop-entry-ranges.ts create mode 100644 packages/compiler/test/loop-entry-versions.test.ts create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/while-entry-range-versions.ts.json create mode 100644 tests/corpus/while-entry-range-versions.ts create mode 100644 tests/harness/corpus-costs/while-entry-range-versions.ts.json diff --git a/packages/compiler/src/backend/llvm/emit-function.ts b/packages/compiler/src/backend/llvm/emit-function.ts index 3be3d979f..427e439ad 100644 --- a/packages/compiler/src/backend/llvm/emit-function.ts +++ b/packages/compiler/src/backend/llvm/emit-function.ts @@ -31,6 +31,7 @@ import { findLocalStackUnions } from "./stack-unions.js"; import { findLocalStackObjects, findReassignedStackObjects } from "./stack-objects.js"; import { findLocalUnionStorage } from "./local-union-storage.js"; import { INT32_RANGE, analyzeIntegerRangeFacts } from "../../ir/integer-ranges.js"; +import { beginLoopEntryVersions } from "./loop-entry-versions.js"; import { findBytesBounds } from "./bytes-bounds.js"; import { findInt32Locals } from "./int32-locals.js"; import { findIntegerViews } from "./integer-views.js"; @@ -246,6 +247,7 @@ export function emitFunction(emitter: LlEmitter, fn: IrFunction): string { const rangeFacts = analyzeIntegerRangeFacts(numericFn, slotFacts); emitter.integerRanges = rangeFacts.ranges; emitter.integerLoopRanges = rangeFacts.loopRanges; + beginLoopEntryVersions(emitter, numericFn, slotFacts, rangeFacts); emitter.int32Elements = slotFacts.elements ?? null; emitter.integerOrNaNLocals = findIntegerOrNaNLocals(emitter, numericFn); emitter.bytesBounds = findBytesBounds(numericFn, emitter.integerRanges); diff --git a/packages/compiler/src/backend/llvm/emit-statements.ts b/packages/compiler/src/backend/llvm/emit-statements.ts index 1eeceff4a..74c84ae86 100644 --- a/packages/compiler/src/backend/llvm/emit-statements.ts +++ b/packages/compiler/src/backend/llvm/emit-statements.ts @@ -46,6 +46,7 @@ import { emitBorrowedInput } from "./borrowed-inputs.js"; import { f64Lit, llvmCommentText, loadArrCount, SCR_ARR_FIELD, sizeToDouble } from "./common.js"; import { planInt32KindLoop, type Int32KindLoopPlan } from "./int32-kind-loops.js"; import { withinInt32 } from "../../ir/integer-ranges.js"; +import { emitLoopEntryVersion, loopEntryVersion } from "./loop-entry-versions.js"; import { resultSlotStore } from "../result-slots.js"; import { matchIntegerArrayForLoop, @@ -2349,6 +2350,13 @@ export function emitLoopCopies( exit: string, emitCopy: (exit: string) => void, ): void { + const entry = allowed ? loopEntryVersion(emitter, loop) : null; + if (entry) { + emitLoopEntryVersion(emitter, loop, entry, () => + emitLoopCopies(emitter, loop, allowed, exit, emitCopy), + ); + return; + } const plan = allowed && emitter.speculativeLoopsEnabled && diff --git a/packages/compiler/src/backend/llvm/emitter.ts b/packages/compiler/src/backend/llvm/emitter.ts index 425c4f4d4..2d2ebfb8d 100644 --- a/packages/compiler/src/backend/llvm/emitter.ts +++ b/packages/compiler/src/backend/llvm/emitter.ts @@ -1,4 +1,5 @@ import type { TargetCpu } from "../target-cpu.js"; +import { loopEntryInlines } from "./loop-entry-versions.js"; import type { FunctionLayout } from "./function-layout.js"; import { relieveHeapPressure } from "../../heap-pressure.js"; import { wasmBoundsChecks } from "./wasm-modules.js"; @@ -4009,8 +4010,10 @@ export class LlEmitter { emitDirectCall(line: string, callee: string): boolean { const B = this.B; // A small leaf called only from loops inlines into each of them, past - // LLVM's cost threshold (loop-callees.ts). Debug builds keep the call. - if (this.debug === null && this.loopCallees.has(callee)) { + // LLVM's cost threshold (loop-callees.ts), and so does a function called + // only from a loop's entry-range version (loop-entry-versions.ts). + // Debug builds keep the call. + if (this.debug === null && (this.loopCallees.has(callee) || loopEntryInlines(this, callee))) { const comment = line.indexOf(" ; "); line = comment < 0 diff --git a/packages/compiler/src/backend/llvm/loop-entry-versions.ts b/packages/compiler/src/backend/llvm/loop-entry-versions.ts new file mode 100644 index 000000000..2c448cae4 --- /dev/null +++ b/packages/compiler/src/backend/llvm/loop-entry-versions.ts @@ -0,0 +1,231 @@ +/* Loop-entry range versions (ir/loop-entry-ranges.ts): a while loop whose + * integer work depends on locals entering it without range facts runs as + * two copies. A test on entry checks that each such local holds an integer + * in its assumed range; the fast copy is emitted under the facts that hold + * then, and the other copy is the loop as it would be emitted otherwise. */ +import type { IrFunction, IrModule, IrStmt } from "../../ir/ir.js"; +import { everyStmt, everyStmtList } from "../../ir/traverse.js"; +import { exprFunctionReference, moduleFunctionReferences } from "../../ir/function-references.js"; +import type { IntegerRangeFacts, IntegerSlotFacts } from "../../ir/integer-ranges.js"; +import { findLoopEntryVersions, type LoopEntryVersion } from "../../ir/loop-entry-ranges.js"; +import { f64Lit } from "./common.js"; +import type { LlEmitter } from "./emitter.js"; +import { defineModuleState } from "./emitter-state.js"; + +interface LoopEntryState { + fn: IrFunction | null; + facts: IntegerRangeFacts | null; + slots: IntegerSlotFacts; + /** Found on the first while loop the function emits. */ + versions: Map | null; + /** Loops whose copies are being emitted: a loop is versioned once. */ + readonly versioning: Set; + /** Direct calls per module function, or -1 for a function some value or + * the module's metadata references. */ + calls: Map | null; + /** The callees whose calls the fast copy being emitted inlines. */ + inlined: ReadonlySet | null; +} + +const ENTRY_VERSIONS = defineModuleState(() => ({ + fn: null, + facts: null, + slots: {}, + versions: null, + versioning: new Set(), + calls: null, + inlined: null, +})); + +/** The most IR nodes a callee may have for its call in a fast copy to be + * inlined (singleCallees). */ +const MAX_INLINED_NODES = 400; + +/** Sets the state for the function about to be emitted: `numericFn` and the + * facts emitFunction computed for it. Debug builds keep every loop as is. */ +export function beginLoopEntryVersions( + emitter: LlEmitter, + numericFn: IrFunction, + slots: IntegerSlotFacts, + facts: IntegerRangeFacts, +): void { + const state = ENTRY_VERSIONS.of(emitter); + state.fn = emitter.debug === null ? numericFn : null; + state.slots = slots; + state.facts = facts; + state.versions = null; + state.versioning.clear(); +} + +/** The entry version of `loop`, when it has one and the emitter can test + * its locals: plain double slots of the current function, outside a + * speculative copy and a coroutine. */ +export function loopEntryVersion(emitter: LlEmitter, loop: IrStmt): LoopEntryVersion | null { + if (loop.kind !== "while") return null; + const state = ENTRY_VERSIONS.of(emitter); + if (state.fn === null || state.facts === null || state.versioning.has(loop)) return null; + if ( + !emitter.speculativeLoopsEnabled || + emitter.speculation !== null || + emitter.speculationResume !== null || + emitter.currentCoro !== null + ) + return null; + state.versions ??= findLoopEntryVersions(state.fn, state.facts, state.slots); + const version = state.versions.get(loop); + if (!version) return null; + for (const id of version.entry.keys()) + if (!emitter.currentLocals.has(id) || !emitter.speculativeLocal(id)) return null; + return version; +} + +/** Emit the entry test and both copies of `loop`. `emitCopies` emits the + * loop as it would be emitted without the version; it runs once under the + * version's facts and once as is. */ +export function emitLoopEntryVersion( + emitter: LlEmitter, + loop: IrStmt, + version: LoopEntryVersion, + emitCopies: () => void, +): void { + const B = emitter.B; + let ok: string | null = null; + for (const [id, range] of version.entry) { + // An integer in [min, max] other than -0: the ordered comparisons + // reject NaN, and the bits of the value and its round trip agree only + // for integers that are not -0. A local the loop writes before it reads + // may be unset on entry, and a conversion out of range is poison: + // `freeze` makes both an arbitrary fixed value. + const raw = B.tmp(), + value = B.tmp(), + low = B.tmp(), + high = B.tmp(), + converted = B.tmp(), + integer = B.tmp(), + back = B.tmp(), + bits = B.tmp(), + backBits = B.tmp(), + exact = B.tmp(), + inRange = B.tmp(), + passed = B.tmp(); + B.line(`${raw} = load double, ptr ${emitter.binding(id).slot}`); + B.line(`${value} = freeze double ${raw}`); + B.line(`${low} = fcmp oge double ${value}, ${f64Lit(range.min)}`); + B.line(`${high} = fcmp ole double ${value}, ${f64Lit(range.max)}`); + B.line(`${converted} = fptosi double ${value} to i64`); + B.line(`${integer} = freeze i64 ${converted}`); + B.line(`${back} = sitofp i64 ${integer} to double`); + B.line(`${bits} = bitcast double ${value} to i64`); + B.line(`${backBits} = bitcast double ${back} to i64`); + B.line(`${exact} = icmp eq i64 ${bits}, ${backBits}`); + B.line(`${inRange} = and i1 ${low}, ${high}`); + B.line(`${passed} = and i1 ${inRange}, ${exact}`); + if (ok === null) ok = passed; + else { + const both = B.tmp(); + B.line(`${both} = and i1 ${ok}, ${passed}`); + ok = both; + } + } + const fast = B.newLabel("entry.fast"), + slow = B.newLabel("entry.slow"); + // The test is expected to pass, which keeps the ordinary copy's calls + // cold for the inliner. + let likely = "true"; + if (ok !== null) { + emitter.declare("declare i1 @llvm.expect.i1(i1, i1)"); + likely = B.tmp(); + B.line(`${likely} = call i1 @llvm.expect.i1(i1 ${ok}, i1 true)`); + } + B.condBr(likely, fast, slow); + const state = ENTRY_VERSIONS.of(emitter); + const originalRanges = emitter.integerRanges; + const originalLoopRanges = emitter.integerLoopRanges; + state.versioning.add(loop); + try { + B.startBlock(fast); + // An enclosing version may have narrowed a range already. + const ranges = new Map(originalRanges); + for (const [node, range] of version.ranges) { + const known = originalRanges.get(node); + ranges.set( + node, + known + ? { min: Math.max(known.min, range.min), max: Math.min(known.max, range.max) } + : range, + ); + } + emitter.integerRanges = ranges; + state.inlined = singleCallees(emitter, state, loop); + if (version.loopRanges) { + const loopRanges = new Map(originalLoopRanges); + loopRanges.set(loop, version.loopRanges); + emitter.integerLoopRanges = loopRanges; + } + emitCopies(); + state.inlined = null; + emitter.integerRanges = originalRanges; + emitter.integerLoopRanges = originalLoopRanges; + B.startBlock(slow); + emitCopies(); + } finally { + emitter.integerRanges = originalRanges; + emitter.integerLoopRanges = originalLoopRanges; + state.versioning.delete(loop); + state.inlined = null; + } +} + +/** Whether the fast copy being emitted inlines its calls of `callee`. */ +export function loopEntryInlines(emitter: LlEmitter, callee: string): boolean { + return ENTRY_VERSIONS.of(emitter).inlined?.has(callee) === true; +} + +/** The functions the loop calls that have no other call or reference in + * the module and at most MAX_INLINED_NODES nodes. LLVM inlines a function's + * only call at almost any size; the two copies give each such function a + * second call, so the fast copy's call is marked to keep that inlining. */ +function singleCallees(emitter: LlEmitter, state: LoopEntryState, loop: IrStmt): Set { + const callees = new Set(); + everyStmt(loop, { + expr: (e) => { + if (e.kind === "call") callees.add(e.callee); + return true; + }, + stmt: () => true, + }); + const result = new Set(); + if (callees.size === 0) return result; + state.calls ??= countCalls(emitter.mod); + for (const name of callees) { + if (state.calls.get(name) !== 1 || name === state.fn?.name) continue; + const fn = emitter.mod.functions.find((f) => f.name === name); + if (fn && !fn.async && !fn.generator && nodeCount(fn.body) <= MAX_INLINED_NODES) + result.add(name); + } + return result; +} + +function countCalls(mod: IrModule): Map { + const calls = new Map(); + const referenced = new Set(moduleFunctionReferences(mod)); + for (const fn of mod.functions) + everyStmtList(fn.body, { + expr: (e) => { + if (e.kind === "call") calls.set(e.callee, (calls.get(e.callee) ?? 0) + 1); + const name = exprFunctionReference(e); + if (name !== undefined) referenced.add(name); + return true; + }, + stmt: () => true, + }); + for (const name of referenced) calls.set(name, -1); + return calls; +} + +function nodeCount(body: IrStmt[]): number { + let count = 0; + const visit = (): boolean => ++count <= MAX_INLINED_NODES; + everyStmtList(body, { expr: visit, stmt: visit }); + return count; +} diff --git a/packages/compiler/src/ir/integer-ranges.ts b/packages/compiler/src/ir/integer-ranges.ts index 51c0a929b..23e2df90a 100644 --- a/packages/compiler/src/ir/integer-ranges.ts +++ b/packages/compiler/src/ir/integer-ranges.ts @@ -23,6 +23,11 @@ export interface IntegerRangeFacts { ranges: IntegerRanges; loopRanges: ReadonlyMap>; } +/** Ranges assumed for locals on entry to while loops (loop-entry-ranges.ts): + * the facts inside such a loop hold in a copy of it that runs only after a + * test proves every assumed local is an integer in its range. After the + * loop, its assumed locals and its writes carry no facts. */ +export type LoopEntryAssumptions = ReadonlyMap>; const SIGNED: IntegerRange = { min: -2147483648, max: 2147483647 }; const UNSIGNED: IntegerRange = { min: 0, max: 4294967295 }; /** Int32 and uint32 at once: the bound under which `%` (modeled only for @@ -309,6 +314,7 @@ export function analyzeIntegerRanges(fn: IrFunction, slots: IntegerSlotFacts = { export function analyzeIntegerRangeFacts( fn: IrFunction, slots: IntegerSlotFacts = {}, + entries?: LoopEntryAssumptions, ): IntegerRangeFacts { const ranges = new Map(); const loopRanges = new Map>(); @@ -1074,6 +1080,8 @@ export function analyzeIntegerRangeFacts( } case "while": case "doWhile": { + const assumed = entries?.get(s); + if (assumed) for (const [id, range] of assumed) if (!facts.has(id)) set(facts, id, range); // Counters the condition bounds keep their range at every read // inside and after the loop. const guarded = @@ -1102,6 +1110,10 @@ export function analyzeIntegerRangeFacts( } else body(s.body, new Map(loop)); havocWith(facts, [s], facts, guarded); for (const [id, range] of guarded) set(facts, id, range); + if (assumed) { + invalidate(facts, [s]); + for (const id of assumed.keys()) facts.delete(id); + } break; } case "forOf": { diff --git a/packages/compiler/src/ir/loop-entry-ranges.test.ts b/packages/compiler/src/ir/loop-entry-ranges.test.ts new file mode 100644 index 000000000..ad1b78ef6 --- /dev/null +++ b/packages/compiler/src/ir/loop-entry-ranges.test.ts @@ -0,0 +1,143 @@ +import { expect, test } from "vitest"; +import { analyzeIntegerRangeFacts } from "./integer-ranges.js"; +import { findLoopEntryVersions, LOOP_ENTRY_RANGE } from "./loop-entry-ranges.js"; +import { BOOL, F64, type IrExpr, type IrFunction, type IrStmt } from "./ir.js"; + +const loc = { file: "test.ts", start: 0, end: 0 }; +const num = (value: number): IrExpr => ({ loc, kind: "numLit", value, type: F64 }); +const ref = (localId: string): IrExpr => ({ loc, kind: "varRef", localId, type: F64 }); +const bin = (op: "+" | "-" | "*" | "%", left: IrExpr, right: IrExpr): IrExpr => ({ + loc, + kind: "bin", + op, + left, + right, + type: F64, +}); +const condition = (op: "<" | "===", left: IrExpr, right: IrExpr): IrExpr => ({ + loc, + kind: "bin", + op, + left, + right, + type: BOOL, +}); +const set = (localId: string, value: IrExpr): IrStmt => ({ loc, kind: "assign", localId, value }); +const bump = (localId: string): IrStmt => ({ + loc, + kind: "exprStmt", + expr: { loc, kind: "incDec", op: "+", prefix: false, localId, type: F64 }, +}); +const observe = (value: IrExpr): IrStmt => ({ loc, kind: "exprStmt", expr: value }); +const fn = (body: IrStmt[], locals: string[]): IrFunction => ({ + loc, + name: "f", + params: [], + returnType: F64, + locals: locals.map((id) => ({ id, name: id, type: F64, mutable: true })), + body, +}); + +/** base = unknown; breaks = unknown; count = 0; i = 0; + * while (i < 300) { ...extra; i++; count++; last = (count + base - breaks) % 8; + * if (last === 0) breaks++; } + * observe(base) */ +function lineBreaks(extra: IrStmt[] = [], locals: string[] = []) { + const remainder = bin("%", bin("-", bin("+", ref("count"), ref("base")), ref("breaks")), num(8)); + const after = ref("base"); + const loop: IrStmt = { + loc, + kind: "while", + cond: condition("<", ref("i"), num(300)), + body: [ + ...extra, + bump("i"), + bump("count"), + set("last", remainder), + { + loc, + kind: "if", + cond: condition("===", ref("last"), num(0)), + then: [bump("breaks")], + else_: null, + }, + ], + }; + const f = fn( + [ + set("base", ref("unknown")), + set("breaks", ref("unknown")), + set("count", num(0)), + set("i", num(0)), + set("last", num(1)), + loop, + observe(after), + ], + ["base", "breaks", "count", "i", "last", ...locals], + ); + return { f, loop, remainder, after }; +} + +test("versions a while loop whose remainder reads locals without entry facts", () => { + const { f, loop, remainder } = lineBreaks(); + const facts = analyzeIntegerRangeFacts(f); + expect(facts.ranges.get(remainder)).toBeFalsy(); + const version = findLoopEntryVersions(f, facts).get(loop); + expect(version).toBeDefined(); + expect([...version!.entry]).toEqual([ + ["base", LOOP_ENTRY_RANGE], + ["breaks", LOOP_ENTRY_RANGE], + ]); + // The dividend is an exact integer; `%` itself is modeled only for + // non-negative dividends. + const dividend = (remainder as Extract).left; + expect(version!.ranges.get(dividend)).toBeDefined(); + // The loop's counters, `breaks` among them, are bounded in the copy. + expect(version!.loopRanges?.get("breaks")).toEqual({ + min: LOOP_ENTRY_RANGE.min, + max: LOOP_ENTRY_RANGE.max + 300, + }); +}); + +test("the entry test covers only the locals the remainder depends on", () => { + // acc = acc + scale reads two more locals without facts. + const { f, loop } = lineBreaks( + [set("acc", bin("+", ref("acc"), ref("scale")))], + ["acc", "scale"], + ); + f.body.unshift(set("acc", ref("unknown")), set("scale", ref("unknown"))); + const version = findLoopEntryVersions(f, analyzeIntegerRangeFacts(f)).get(loop); + expect([...version!.entry.keys()]).toEqual(["base", "breaks"]); +}); + +test("assumed facts do not leave the loop", () => { + const { f, loop, after } = lineBreaks(); + const assumed = new Map([[loop, new Map([["base", LOOP_ENTRY_RANGE]])]]); + expect(analyzeIntegerRangeFacts(f, {}, assumed).ranges.get(after)).toBeFalsy(); +}); + +test("loops without a remainder the assumption proves keep one copy", () => { + // A local the loop makes fractional, a product without a remainder, and a + // remainder the function already proves. + const fractional = lineBreaks([set("base", bin("+", ref("base"), num(0.5)))]); + expect(findLoopEntryVersions(fractional.f, analyzeIntegerRangeFacts(fractional.f)).size).toBe(0); + const product: IrStmt = { + loc, + kind: "while", + cond: condition("<", ref("i"), num(300)), + body: [set("acc", bin("*", ref("acc"), ref("scale"))), bump("i")], + }; + const products = fn( + [set("acc", ref("unknown")), set("scale", ref("unknown")), set("i", num(0)), product], + ["acc", "scale", "i"], + ); + expect(findLoopEntryVersions(products, analyzeIntegerRangeFacts(products)).size).toBe(0); + const proven: IrStmt = { + loc, + kind: "while", + cond: condition("<", ref("i"), num(300)), + body: [set("acc", bin("%", ref("i"), num(7))), bump("i")], + }; + const known = fn([set("acc", ref("unknown")), set("i", num(0)), proven], ["acc", "i"]); + expect(findLoopEntryVersions(known, analyzeIntegerRangeFacts(known)).size).toBe(0); +}); diff --git a/packages/compiler/src/ir/loop-entry-ranges.ts b/packages/compiler/src/ir/loop-entry-ranges.ts new file mode 100644 index 000000000..a5b7628be --- /dev/null +++ b/packages/compiler/src/ir/loop-entry-ranges.ts @@ -0,0 +1,218 @@ +import type { IrExpr, IrFunction, IrStmt } from "./ir.js"; +import { everyExpr, everyStmt, everyStmtList } from "./traverse.js"; +import { + analyzeIntegerRangeFacts, + type IntegerRange, + type IntegerRangeFacts, + type IntegerSlotFacts, + type LoopEntryAssumptions, +} from "./integer-ranges.js"; + +type WhileLoop = Extract; + +/** The range a versioned loop requires of each assumed local on entry. A + * loop's counters stay below a typed array or string length (at most 2^48), + * so sums of a few such values and assumed locals stay exact integers. */ +export const LOOP_ENTRY_RANGE: IntegerRange = { min: -(2 ** 48), max: 2 ** 48 }; + +/** A copy of a while loop that runs when every local in `entry` holds an + * integer in its range on entry (never -0). `ranges` are the facts that + * hold for the loop's expressions in that copy and that the function's own + * analysis does not prove; `loopRanges` are the loop's bounded bindings + * there. */ +export interface LoopEntryVersion { + entry: ReadonlyMap; + ranges: ReadonlyMap; + loopRanges: ReadonlyMap | undefined; +} + +/** Versions of innermost while loops whose integer work depends on locals + * that enter the loop without range facts: copies of module globals, + * parameters, fields or call results (`const base = offset;` before a loop + * that computes `(count + base) % 60`). + * + * The function is analyzed again with each number local the loop reads + * without a fact assumed in LOOP_ENTRY_RANGE at the loop's entry. A loop is + * versioned when that analysis proves both operands of a remainder that the + * function's own analysis leaves unproven. Only remainders qualify: the + * operands of a product or a sum of such locals are as often fractional, + * and a loop whose test fails runs the ordinary copy in a larger function. + * The entry test covers only the assumed locals that such an operation reads, directly, + * through the loop's writes of other locals, or through the loop condition; + * when that drops any, the function is analyzed once more with the smaller + * sets, so the facts never rest on a local the test does not check. No + * fact leaves an assumed loop, so the loops' assumptions are independent. */ +export function findLoopEntryVersions( + fn: IrFunction, + facts: IntegerRangeFacts, + slots: IntegerSlotFacts = {}, +): Map { + const result = new Map(); + if (fn.async || fn.generator) return result; + const captures = new Set( + [...(fn.captures ?? []), ...(fn.classCaptures ?? [])].map((c) => c.localId), + ); + const eligible = new Set( + fn.locals + .filter((l) => l.type.kind === "f64" && !l.boxed && !l.tdz && !captures.has(l.id)) + .map((l) => l.id), + ); + const known = (e: IrExpr): boolean => !!facts.ranges.get(e); + let candidates = new Map>(); + everyStmtList(fn.body, { + expr: () => true, + stmt: (s) => { + if (s.kind !== "while" || containsLoop(s.body)) return true; + const assumed = entryCandidates(s, eligible, known); + if (assumed.size > 0) candidates.set(s, assumed); + return true; + }, + }); + if (candidates.size === 0) return result; + let versioned = analyzeIntegerRangeFacts(fn, slots, assumptionsOf(candidates)); + const pruned = new Map>(); + let dropped = false; + for (const [loop, entry] of candidates) { + const roots = gainedOperands(loop, known, versioned); + if (roots.length === 0) continue; + const relevant = relevantLocals(loop, roots); + const kept = new Map(); + for (const [id, range] of entry) if (relevant.has(id)) kept.set(id, range); + if (kept.size < entry.size) dropped = true; + if (kept.size > 0) pruned.set(loop, kept); + } + if (dropped) { + candidates = pruned; + if (candidates.size === 0) return result; + versioned = analyzeIntegerRangeFacts(fn, slots, assumptionsOf(candidates)); + } + for (const [loop, entry] of candidates) { + if (gainedOperands(loop, known, versioned).length === 0) continue; + const ranges = new Map(); + const visit = (e: IrExpr): boolean => { + const range = versioned.ranges.get(e); + const prior = facts.ranges.get(e); + if (range && !prior) ranges.set(e, range); + else if (range && prior && (range.min > prior.min || range.max < prior.max)) + ranges.set(e, { min: Math.max(range.min, prior.min), max: Math.min(range.max, prior.max) }); + return true; + }; + everyStmt(loop, { expr: visit, stmt: () => true }); + result.set(loop, { entry, ranges, loopRanges: versioned.loopRanges.get(loop) }); + } + return result; +} + +function assumptionsOf( + candidates: ReadonlyMap>, +): LoopEntryAssumptions { + const result = new Map>(); + for (const [loop, entry] of candidates) result.set(loop, entry); + return result; +} + +/** Number locals the loop reads without a range fact and does not declare, + * when the loop has a remainder without one. */ +function entryCandidates( + loop: WhileLoop, + eligible: ReadonlySet, + known: (e: IrExpr) => boolean, +): Map { + const read = new Set(); + const declared = new Set(); + let integerWork = false; + everyStmt(loop, { + expr: (e) => { + if (e.kind === "varRef" && eligible.has(e.localId) && !known(e)) read.add(e.localId); + else if (e.kind === "bin" && e.op === "%" && !known(e)) integerWork = true; + return true; + }, + stmt: (s) => { + if (s.kind === "varDecl") declared.add(s.localId); + return true; + }, + }); + const result = new Map(); + if (!integerWork) return result; + for (const id of read) if (!declared.has(id)) result.set(id, LOOP_ENTRY_RANGE); + return result; +} + +/** Operands of the loop's remainders that the versioned analysis proves + * where the function's own analysis does not prove both. */ +function gainedOperands( + loop: WhileLoop, + known: (e: IrExpr) => boolean, + versioned: IntegerRangeFacts, +): IrExpr[] { + const operands: IrExpr[] = []; + everyStmt(loop, { + expr: (e) => { + if ( + e.kind === "bin" && + e.op === "%" && + !(known(e.left) && known(e.right)) && + !!versioned.ranges.get(e.left) && + !!versioned.ranges.get(e.right) + ) + operands.push(e.left, e.right); + return true; + }, + stmt: () => true, + }); + return operands; +} + +/** The locals `roots` read, closed over the loop's writes (a local written + * from others reads them) and over its condition (a local the condition + * reads makes every local of the condition relevant). */ +function relevantLocals(loop: WhileLoop, roots: readonly IrExpr[]): Set { + const relevant = new Set(); + const readsOf = (e: IrExpr): string[] => { + const ids: string[] = []; + everyExpr(e, { + expr: (x) => { + if (x.kind === "varRef") ids.push(x.localId); + return true; + }, + stmt: () => true, + }); + return ids; + }; + const written = new Map(); + everyStmt(loop, { + expr: (e) => { + if (e.kind === "assignExpr") + written.set(e.localId, [...(written.get(e.localId) ?? []), e.value]); + return true; + }, + stmt: (s) => { + const value = s.kind === "assign" ? s.value : s.kind === "varDecl" ? s.init : undefined; + if (value && (s.kind === "assign" || s.kind === "varDecl")) + written.set(s.localId, [...(written.get(s.localId) ?? []), value]); + return true; + }, + }); + const condition = readsOf(loop.cond); + const pending: string[] = []; + const add = (id: string): void => { + if (relevant.has(id)) return; + relevant.add(id); + pending.push(id); + }; + for (const root of roots) for (const id of readsOf(root)) add(id); + while (pending.length > 0) { + const id = pending.pop()!; + for (const value of written.get(id) ?? []) for (const other of readsOf(value)) add(other); + if (condition.includes(id)) for (const other of condition) add(other); + } + return relevant; +} + +function containsLoop(body: IrStmt[]): boolean { + return !everyStmtList(body, { + expr: () => true, + stmt: (s) => + s.kind !== "while" && s.kind !== "doWhile" && s.kind !== "for" && s.kind !== "forOf", + }); +} diff --git a/packages/compiler/test/loop-entry-versions.test.ts b/packages/compiler/test/loop-entry-versions.test.ts new file mode 100644 index 000000000..22e41e3e7 --- /dev/null +++ b/packages/compiler/test/loop-entry-versions.test.ts @@ -0,0 +1,45 @@ +import { mkdtemp, readFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { expect, test } from "vitest"; +import { compile, deserializeModule } from "../src/index.js"; +import { emitLlvmModule } from "../src/backend/llvm/emitter.js"; + +test("while loops test their entry values and run integer remainders in the fast copy", async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-entry-versions-")); + try { + const entry = join(import.meta.dirname, "../../../tests/corpus/while-entry-range-versions.ts"); + const outPath = join(dir, "versions.ir.json"); + const result = await compile(entry, { outPath, outDir: dir, outputKind: "ir" }); + if (!result.ok) throw new Error(result.diagnostics.map((d) => d.message).join("\n")); + const mod = deserializeModule(await readFile(outPath, "utf8")); + const ll = emitLlvmModule(mod, { pointerBits: 64 }); + const body = (name: string): string => + ll.match( + new RegExp(`define internal [^\\n]+ @sc_bf_${name}\\([^\\n]*\\{([\\s\\S]*?)\\n\\}`), + )![1]!; + for (const name of ["wrap", "ring", "_x25_Cursor_stripes"]) { + const fn = body(name); + const fast = fn.slice(fn.indexOf("\nentry.fast"), fn.indexOf("\nentry.slow")); + const slow = fn.slice(fn.indexOf("\nentry.slow")); + // The fast copy adds the dividend's integers; the ordinary copy adds + // doubles and tests the sum before its integer remainder. + const adds = (text: string): number => text.split("fadd double").length - 1; + expect(fast, name).toContain("srem i64"); + expect(adds(fast), name).toBeLessThan(adds(slow)); + } + // `complement` has no other caller: the fast copy's call keeps it inlined. + const wrap = body("wrap"); + const fast = wrap.slice(wrap.indexOf("\nentry.fast"), wrap.indexOf("\nentry.slow")); + expect(fast).toMatch(/call i32 @sc_f_complement\.i32\([^)]*\) alwaysinline/); + expect(wrap.slice(wrap.indexOf("\nentry.slow"))).not.toContain("alwaysinline"); + // Debug builds keep one copy. + const debug = emitLlvmModule(mod, { + pointerBits: 64, + debugSources: new Map([[entry, await readFile(entry, "utf8")]]), + }); + expect(debug).not.toContain("entry.fast"); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/while-entry-range-versions.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/while-entry-range-versions.ts.json new file mode 100644 index 000000000..ec9e8eeae --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/while-entry-range-versions.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/while-entry-range-versions.ts" + ], + "diags": [] +} diff --git a/tests/corpus/while-entry-range-versions.ts b/tests/corpus/while-entry-range-versions.ts new file mode 100644 index 000000000..7d96c243a --- /dev/null +++ b/tests/corpus/while-entry-range-versions.ts @@ -0,0 +1,160 @@ +// While loops whose remainder reads locals copied from module globals, +// parameters and fields. Entry values cover integers inside and outside +// [-2^48, 2^48], fractions, -0, NaN and infinities. + +let column = 0; +let lines = 0; + +// A switch with more cases than a small leaf function has statements. +function complement(code: number): number { + switch (code) { + case 65: + return 84; + case 67: + return 71; + case 71: + return 67; + case 84: + return 65; + case 97: + return 116; + case 99: + return 103; + case 103: + return 99; + case 116: + return 97; + case 85: + return 65; + case 117: + return 97; + case 82: + return 89; + case 89: + return 82; + case 75: + return 77; + case 77: + return 75; + } + return 78; +} + +// Copies bytes in reverse, complemented, with a newline after every 8 +// columns counted from the module-level column and line counters. +function wrap(input: Uint8Array, out: Uint8Array): number { + const base = column; + let breaks = lines; + let count = 0; + let o = 0; + let i = 0; + const len = input.length; + let last = 1; + let negativeZeros = 0; + while (i < len) { + out[o] = complement(input[len - 1 - i]!); + i++; + o++; + count++; + last = (count + base - breaks) % 8; + if (Object.is(last, -0)) negativeZeros++; + if (last === 0) { + out[o] = 10; + o++; + count++; + breaks++; + } + } + column = base + count; + lines = breaks; + console.log(` last ${Object.is(last, -0) ? "-0" : last}, ${negativeZeros} of them -0`); + return o; +} + +function text(out: Uint8Array, n: number): string { + let s = ""; + for (let i = 0; i < n; i++) s += out[i] === 10 ? "|" : String.fromCharCode(out[i]!); + return s; +} + +const input = new Uint8Array(29); +const bases = "ACGTTGCAAGCTTCGA"; +for (let i = 0; i < input.length; i++) input[i] = bases.charCodeAt((i * 7) % bases.length); +const out = new Uint8Array(64); + +const starts: [number, number][] = [ + [0, 0], + [5, 2], + [-13, 0], + [-16, 0], + [2 ** 48, 3], + [-(2 ** 48), 0], + [2 ** 48 + 1, 0], + [2 ** 50, 1], + [2 ** 53 - 5, 0], + [2 ** 53, 0], + [2 ** 53 + 2, 1], + [0.5, 0], + [3, 0.25], + [3, 2 ** 49], + [-0, 0], + [0, -0], + [NaN, 0], + [Infinity, 0], + [-Infinity, 0], + [1e300, 0], +]; +for (const [c, l] of starts) { + column = c; + lines = l; + console.log(`start column ${c} lines ${l}`); + const n = wrap(input, out); + console.log(` ${text(out, n)}`); + console.log(` column ${column} lines ${lines}`); +} + +// Two calls in a row: the second starts from the first one's counters. +column = 3; +lines = 0; +wrap(input, out); +const n = wrap(input, out); +console.log(`again ${text(out, n)} column ${column} lines ${lines}`); + +// A parameter and a field as the loop's entry values. +function ring(values: Int32Array, offset: number, size: number): number { + let sum = 0; + let i = 0; + while (i < values.length) { + sum += values[(i + offset) % size]!; + i++; + } + return sum; +} +const values = new Int32Array([3, 1, 4, 1, 5, 9, 2, 6]); +for (const offset of [0, 3, 7, 2 ** 40, 2 ** 49, 1.5, -0, -2]) { + const sum = ring(values, offset, values.length); + console.log(`ring ${offset} ${Number.isNaN(sum) ? "NaN" : sum}`); +} + +class Cursor { + position: number; + constructor(position: number) { + this.position = position; + } + stripes(bytes: Uint8Array): string { + const start = this.position; + let i = 0; + let marks = ""; + while (i < bytes.length) { + const r = (start + i) % 3; + marks += Object.is(r, -0) ? "z" : String(r); + i++; + } + this.position = start + i; + return marks; + } +} +for (const position of [0, 4, -5, 2 ** 48 + 7, 0.75, 2 ** 60]) { + const cursor = new Cursor(position); + console.log(`stripes ${position} ${cursor.stripes(input.subarray(0, 7))} ${cursor.position}`); +} diff --git a/tests/harness/corpus-costs/while-entry-range-versions.ts.json b/tests/harness/corpus-costs/while-entry-range-versions.ts.json new file mode 100644 index 000000000..2a685047e --- /dev/null +++ b/tests/harness/corpus-costs/while-entry-range-versions.ts.json @@ -0,0 +1 @@ +{ "plain": 3600, "san": 3000 }