diff --git a/packages/compiler/src/backend/llvm/emitter.ts b/packages/compiler/src/backend/llvm/emitter.ts index f5fccab80..1aa1e4197 100644 --- a/packages/compiler/src/backend/llvm/emitter.ts +++ b/packages/compiler/src/backend/llvm/emitter.ts @@ -56,6 +56,14 @@ import { matchIntegerCountedForLoop, } from "../../ir/integer-loops.js"; import { emitCountedLoopLimit } from "./counted-loops.js"; +import { + emitSafeIntegerCheck, + emitSpeculativeInteger, + planSpeculativeLoop, + speculativeReads, + speculativeWriteTarget, + SPECULATIVE_RANGE, +} from "./speculative-integers.js"; import { emitLiteralSwitch } from "./switch-dispatch.js"; import { findBytesBounds } from "./bytes-bounds.js"; import { @@ -956,6 +964,22 @@ export class LlEmitter { { slot: string; type: "i32" | "i64"; signed?: boolean; range?: { min: number; max: number } } >(); private countedLoopsEnabled = false; + /** Speculative integer loops (speculative-integers.ts) run in this function. */ + private speculativeLoopsEnabled = false; + /** The fast copy of a speculative integer loop being emitted: its + * integer-computed writes and where each resumes in the ordinary copy. */ + private speculation: { + writes: Set; + resume: Map; + /** Globals whose double is written back only when the copy is left. */ + deferred: Set; + } | null = null; + /** The ordinary copy being emitted: resume labels still to place. */ + private speculationResume: Map | null = null; + /** The function being emitted, for speculative globals. */ + private speculativeFunction = ""; + /** Every function that writes each number global (computed on demand). */ + private globalWriters: Map> | null = null; /** Whole-program final reference fields (ir/final-fields.ts). */ private finalFields!: FinalFields; /** The current function may borrow final fields of borrowable objects: @@ -4920,6 +4944,266 @@ export class LlEmitter { return slot; } + /** Emit a loop's condition, body and update through `emitCopy`, twice + * when speculative-integers.ts plans a fast integer copy. Both copies + * share the loop's setup and exit; the ordinary copy comes first so the + * fast copy is only emitted when every resume point exists. */ + private emitLoopCopies( + loop: Extract, + allowed: boolean, + exit: string, + emitCopy: (exit: string) => void, + ): void { + const plan = + allowed && + this.speculativeLoopsEnabled && + this.speculation === null && + this.speculationResume === null + ? planSpeculativeLoop( + loop, + (id) => this.speculativeLocal(id), + (id) => this.integerLoopBindings.has(id) || this.int32Locals.has(id), + (e) => { + const range = this.integerRanges.get(e); + return ( + range !== undefined && + range !== null && + range.min >= -Number.MAX_SAFE_INTEGER && + range.max <= Number.MAX_SAFE_INTEGER + ); + }, + (id) => this.speculativeLocal(id) || this.speculativeInvariantGlobal(id), + ) + : null; + if (!plan) { + emitCopy(exit); + return; + } + const B = this.B; + const entries = new Map(); + let ok = "true"; + for (const [id, live] of plan.locals) { + if (!live) continue; + const value = B.tmp(); + B.line(`${value} = load double, ptr ${this.binding(id).slot}`); + const [integer, exact] = emitSafeIntegerCheck(this, value); + entries.set(id, integer); + if (ok === "true") ok = exact; + else { + const both = B.tmp(); + B.line(`${both} = and i1 ${ok}, ${exact}`); + ok = both; + } + } + const fast = B.newLabel("spec.fast"), + slow = B.newLabel("spec.slow"); + B.condBr(ok, fast, slow); + const resume = new Map(); + for (const s of plan.writes) resume.set(s, { label: B.newLabel("spec.resume"), placed: false }); + B.startBlock(slow); + this.speculationResume = resume; + try { + emitCopy(exit); + } finally { + this.speculationResume = null; + } + B.startBlock(fast); + if (![...resume.values()].every((r) => r.placed)) { + B.br(slow); + return; + } + for (const id of plan.locals.keys()) { + const shadow = B.slot(); + B.entryAllocas.push( + `${shadow} = alloca i64 ; speculative integer ${this.currentLocals.get(id)?.name ?? id}`, + ); + const entry = entries.get(id); + if (entry !== undefined) { + // The same double, now visibly an integer conversion: the double + // phis of the fast copy fold into integer phis. + const exact = B.tmp(); + B.line(`store i64 ${entry}, ptr ${shadow}`); + B.line(`${exact} = sitofp i64 ${entry} to double`); + B.line(`store double ${exact}, ptr ${this.binding(id).slot}`); + } else B.line(`store i64 0, ptr ${shadow}`); + this.integerLoopBindings.set(id, { + slot: shadow, + type: "i64", + signed: true, + range: SPECULATIVE_RANGE, + }); + } + // A global's store stays in the loop (calls may read it). When nothing + // in the loop can observe it, its double is written once on leaving the + // copy instead, so the loop keeps it in a register as the double copy + // would. + const deferred = new Set(); + // Worker loops also leave through their cancellation poll. + if (plan.observerFree && !this.mod.workers) + for (const s of plan.writes) { + const id = speculativeWriteTarget(s)!.localId; + if (!this.currentLocals.has(id)) deferred.add(id); + } + this.speculation = { + writes: plan.writes, + resume: new Map([...resume].map(([s, r]) => [s, r.label])), + deferred, + }; + try { + if (deferred.size === 0) emitCopy(exit); + else { + const leave = B.newLabel("spec.leave"); + emitCopy(leave); + B.startBlock(leave); + this.writeBackSpeculativeGlobals(deferred); + B.br(exit); + } + } finally { + this.speculation = null; + for (const id of plan.locals.keys()) this.integerLoopBindings.delete(id); + } + } + + private writeBackSpeculativeGlobals(deferred: Set): void { + const B = this.B; + for (const id of deferred) { + const integer = B.tmp(), + number = B.tmp(); + B.line(`${integer} = load i64, ptr ${this.integerLoopBindings.get(id)!.slot}`); + B.line(`${number} = sitofp i64 ${integer} to double`); + B.line(`store double ${number}, ptr ${this.binding(id).slot}`); + } + } + + /** A number local whose only storage is its own double slot, so an i64 + * shadow beside it can stand for it within one loop. */ + private speculativeLocal(id: string): boolean { + const local = this.currentLocals.get(id); + if (!local) return this.speculativeGlobal(id); + if (local.type.kind !== "f64" || local.boxed || local.tdz) return false; + return !( + this.captureIds.has(id) || + this.int32Locals.has(id) || + this.integerLoopBindings.has(id) || + this.integerViews.has(id) || + this.localUnionStorage.has(id) || + this.reboundOwnerSlots.has(id) || + this.lazyCaptureBoxes.has(id) || + this.privateSplitLocals.has(id) + ); + } + + /** A module-level number written only by the file's top-level code that + * is emitting this loop. Calls may still read it: the fast copy keeps its + * double current. An init function runs once and is never re-entered, so + * no call from the loop can write the global behind the shadow. */ + private speculativeGlobal(id: string): boolean { + if (!this.speculativeFunction.startsWith("%init.")) return false; + const writers = this.numberGlobalWriters(id); + return ( + writers !== null && + (writers.size === 0 || (writers.size === 1 && writers.has(this.speculativeFunction))) + ); + } + + /** A module-level number a loop of any function only reads, written by + * nothing but top-level code: an init function never runs during a loop + * of another function, so the entry value stays its value. */ + private speculativeInvariantGlobal(id: string): boolean { + const writers = this.numberGlobalWriters(id); + return writers !== null && [...writers].every((name) => name.startsWith("%init.")); + } + + /** The functions writing a number global a shadow may stand for, or null + * when the global is not such a number. */ + private numberGlobalWriters(id: string): ReadonlySet | null { + if (this.globalTypes.get(id)?.kind !== "f64") return null; + if (this.tdzGlobals.has(id) || this.integerViews.has(id)) return null; + if (!this.globalWriters) { + const writers = new Map>(); + for (const fn of this.mod.functions) { + const note = (localId: string): void => { + if (!localId.startsWith("%g.")) return; + const set = writers.get(localId) ?? new Set(); + set.add(fn.name); + writers.set(localId, set); + }; + everyStmtList(fn.body, { + stmt: (s) => { + if (s.kind === "assign" || s.kind === "varDecl") note(s.localId); + return true; + }, + expr: (e) => { + if (e.kind === "assignExpr" || e.kind === "incDec") note(e.localId); + return true; + }, + }); + } + this.globalWriters = writers; + } + return this.globalWriters.get(id) ?? new Set(); + } + + /** A speculated write in the fast copy: compute the value as a safe + * integer, store it to the shadow and its exact double to the local. + * Any failed check re-executes the statement in the ordinary copy. */ + private emitSpeculativeWrite(s: IrStmt, resume: string): void { + const B = this.B; + const target = speculativeWriteTarget(s)!; + const binding = this.integerLoopBindings.get(target.localId)!; + const failed = B.newLabel("spec.bail"); + let bails = false; + this.declare("declare i1 @llvm.expect.i1(i1, i1)"); + const bail = (condition: string): void => { + if (condition === "false") return; + bails = true; + const expected = B.tmp(); + B.line(`${expected} = call i1 @llvm.expect.i1(i1 ${condition}, i1 false)`); + const next = B.newLabel("spec.ok"); + B.condBr(expected, failed, next); + B.startBlock(next); + }; + // Heap reads come first, before any check, and release their temporaries + // at once: a bailing path then owns nothing, and reads have no effect + // whose reordering before the arithmetic could be observed. + const reads = new Map(); + if (target.value !== null) { + this.frames.push([]); + for (const read of speculativeReads(target.value)) reads.set(read, this.emitExpr(read)); + this.releaseFrame(this.frames.pop()!); + } + this.frames.push([]); + let result: string; + if (target.value === null) { + const old = B.tmp(); + result = B.tmp(); + B.line(`${old} = load i64, ptr ${binding.slot}`); + B.line(`${result} = ${target.op === "+" ? "add" : "sub"} nsw i64 ${old}, 1`); + const biased = B.tmp(), + outside = B.tmp(); + B.line(`${biased} = add i64 ${result}, ${Number.MAX_SAFE_INTEGER}`); + B.line(`${outside} = icmp ugt i64 ${biased}, ${2 * Number.MAX_SAFE_INTEGER}`); + bail(outside); + } else result = emitSpeculativeInteger(this, target.value, bail, reads); + // Speculable values read only numbers past this point. + if (this.frames.pop()!.length > 0) + throw new InternalCompilerError("llvm emitter bug: speculative write owns a temporary"); + B.line(`store i64 ${result}, ptr ${binding.slot}`); + const deferred = this.speculation!.deferred; + if (!deferred.has(target.localId)) { + const number = B.tmp(); + B.line(`${number} = sitofp i64 ${result} to double`); + B.line(`store double ${number}, ptr ${this.binding(target.localId).slot}`); + } + if (!bails) return; + const done = B.newLabel("spec.done"); + B.br(done); + B.startBlock(failed); + this.writeBackSpeculativeGlobals(deferred); + B.br(resume); + B.startBlock(done); + } + /** The amortized cancellation poll, emitted on the loop's back edge (the * continue target), after the body. A poll in the header would be a loop * exit ahead of the body; LLVM then cannot rotate the loop, so loads that @@ -6033,6 +6317,10 @@ export class LlEmitter { fn.captures !== undefined ? this.lazyCaptures.stableCaptures(fn.name) : new Set(); this.integerLoopBindings.clear(); this.countedLoopsEnabled = this.debug === null && !fn.async && !fn.generator; + this.speculativeLoopsEnabled = this.countedLoopsEnabled; + this.speculativeFunction = fn.name; + this.speculation = null; + this.speculationResume = null; this.finalFieldBorrows = !fn.async && !fn.generator && constructorClass(fn.name) === null && this.finalFields.size > 0; this.byteWindowsEnabled = this.countedLoopsEnabled; @@ -7056,6 +7344,17 @@ export class LlEmitter { } emitStmt(s: IrStmt): void { + const resume = this.speculationResume?.get(s); + if (resume && !resume.placed && !this.B.isTerminated()) { + // The fast copy's failed checks re-execute this statement here. + resume.placed = true; + this.B.br(resume.label); + this.B.startBlock(resume.label); + } + if (this.speculation?.writes.has(s)) { + this.emitSpeculativeWrite(s, this.speculation.resume.get(s)!); + return; + } const previous = this.B.debugLocation; this.B.debugLocation = this.debug?.location(s.loc, this.debugScope) ?? null; const loop = @@ -7716,57 +8015,61 @@ export class LlEmitter { } case "while": { const workerBudget = this.workerLoopBudget(s); - const lc = B.newLabel("loop.c"); - const lb = B.newLabel("loop.b"); const le = B.newLabel("loop.e"); - const lp = workerBudget !== null ? B.newLabel("loop.p") : lc; - B.br(lc); - B.startBlock(lc); - B.condBr(this.emitCondition(s.cond), lb, le); - B.startBlock(lb); - this.jumpTargets.push({ - kind: "loop", - brkLabel: le, - contLabel: lp, - labels: s.labels, - frameDepth: this.frames.length, - scopeDepth: this.scopes.length, - finallyDepth: this.finallyStack.length, - }); - this.emitBlock(s.body); - this.jumpTargets.pop(); - B.br(lp); - if (lp !== lc) { - B.startBlock(lp); - this.emitWorkerLoopCheck(workerBudget); + this.emitLoopCopies(s, true, le, (exit) => { + const lc = B.newLabel("loop.c"); + const lb = B.newLabel("loop.b"); + const lp = workerBudget !== null ? B.newLabel("loop.p") : lc; B.br(lc); - } + B.startBlock(lc); + B.condBr(this.emitCondition(s.cond), lb, exit); + B.startBlock(lb); + this.jumpTargets.push({ + kind: "loop", + brkLabel: exit, + contLabel: lp, + labels: s.labels, + frameDepth: this.frames.length, + scopeDepth: this.scopes.length, + finallyDepth: this.finallyStack.length, + }); + this.emitBlock(s.body); + this.jumpTargets.pop(); + B.br(lp); + if (lp !== lc) { + B.startBlock(lp); + this.emitWorkerLoopCheck(workerBudget); + B.br(lc); + } + }); B.startBlock(le); break; } case "doWhile": { const workerBudget = this.workerLoopBudget(s); // Body first (runs at least once); continue jumps to the CONDITION. - const lb = B.newLabel("loop.b"); - const lc = B.newLabel("loop.c"); const le = B.newLabel("loop.e"); - B.br(lb); - B.startBlock(lb); - this.jumpTargets.push({ - kind: "loop", - brkLabel: le, - contLabel: lc, - labels: s.labels, - frameDepth: this.frames.length, - scopeDepth: this.scopes.length, - finallyDepth: this.finallyStack.length, + this.emitLoopCopies(s, true, le, (exit) => { + const lb = B.newLabel("loop.b"); + const lc = B.newLabel("loop.c"); + B.br(lb); + B.startBlock(lb); + this.jumpTargets.push({ + kind: "loop", + brkLabel: exit, + contLabel: lc, + labels: s.labels, + frameDepth: this.frames.length, + scopeDepth: this.scopes.length, + finallyDepth: this.finallyStack.length, + }); + this.emitBlock(s.body); + this.jumpTargets.pop(); + B.br(lc); + B.startBlock(lc); + this.emitWorkerLoopCheck(workerBudget); + B.condBr(this.emitCondition(s.cond), lb, exit); }); - this.emitBlock(s.body); - this.jumpTargets.pop(); - B.br(lc); - B.startBlock(lc); - this.emitWorkerLoopCheck(workerBudget); - B.condBr(this.emitCondition(s.cond), lb, le); B.startBlock(le); break; } @@ -7922,8 +8225,6 @@ export class LlEmitter { const initializers = s.init.kind === "block" ? s.init.body : [s.init]; for (const initializer of initializers) this.emitStmt(initializer); } - const lc = B.newLabel("loop.c"); - const lb = B.newLabel("loop.b"); const le = B.newLabel("loop.e"); // JS `for (let i ...)`: each iteration gets a FRESH binding holding // a copy of the previous one (closures made in iteration k keep @@ -7952,69 +8253,75 @@ export class LlEmitter { } }; const workerBudget = this.workerLoopBudget(s); - const lu = - s.update || capturedLets.length > 0 || workerBudget !== null ? B.newLabel("loop.u") : lc; freshenBindings(); - B.br(lc); - B.startBlock(lc); - if (integerLoop && integerSlot) { - const receiver = this.emitStableReceiver(integerLoop.limitReceiver, []); - const lenPtr = B.tmp(); - const len = B.tmp(); - const index = B.tmp(); - const inBounds = B.tmp(); - B.line( - `${lenPtr} = getelementptr inbounds %${integerArrayLoop ? "ScrArr" : "ScrBytes"}, ptr ${receiver.name}, i64 0, i32 1`, - ); - if (integerArrayLoop) this.markMemoryPointer(lenPtr, "array:header"); - B.line( - `${len} = load ${this.sizeType}, ptr ${lenPtr}${this.fieldAliasAttachment(lenPtr)}`, - ); - B.line(`${index} = load ${this.sizeType}, ptr ${integerSlot}`); - B.line(`${inBounds} = icmp ult ${this.sizeType} ${index}, ${len}`); - B.condBr(inBounds, lb, le); - } else if (countedLoop && integerSlot && countedLimit) { - const index = B.tmp(); - const inBounds = B.tmp(); - B.line(`${index} = load i64, ptr ${integerSlot}`); - B.line( - `${inBounds} = icmp ${countedLoop.step > 0 ? "slt" : "sgt"} i64 ${index}, ${countedLimit}`, - ); - B.condBr(inBounds, lb, le); - } else if (s.cond) B.condBr(this.emitCondition(s.cond), lb, le); - else B.br(lb); - B.startBlock(lb); - this.jumpTargets.push({ - kind: "loop", - brkLabel: le, - contLabel: lu, - labels: s.labels, - frameDepth: this.frames.length, - scopeDepth: this.scopes.length, - finallyDepth: this.finallyStack.length, - }); - this.emitBlock(s.body); - this.jumpTargets.pop(); - B.br(lu); - if (lu !== lc) { - B.startBlock(lu); - this.emitWorkerLoopCheck(workerBudget); - freshenBindings(); - if (integerLoop && integerSlot) { - const old = B.tmp(); - const next = B.tmp(); - B.line(`${old} = load ${this.sizeType}, ptr ${integerSlot}`); - B.line(`${next} = add nuw ${this.sizeType} ${old}, 1`); - B.line(`store ${this.sizeType} ${next}, ptr ${integerSlot}`); - } else if (countedLoop && integerSlot) { - const old = B.tmp(); - const next = B.tmp(); - B.line(`${old} = load i64, ptr ${integerSlot}`); - B.line(`${next} = add nsw i64 ${old}, ${countedLoop.step}`); - B.line(`store i64 ${next}, ptr ${integerSlot}`); - } else if (s.update) this.emitStmt(s.update); + this.emitLoopCopies(s, capturedLets.length === 0, le, (exit) => { + const lc = B.newLabel("loop.c"); + const lb = B.newLabel("loop.b"); + const lu = + s.update || capturedLets.length > 0 || workerBudget !== null + ? B.newLabel("loop.u") + : lc; B.br(lc); - } + B.startBlock(lc); + if (integerLoop && integerSlot) { + const receiver = this.emitStableReceiver(integerLoop.limitReceiver, []); + const lenPtr = B.tmp(); + const len = B.tmp(); + const index = B.tmp(); + const inBounds = B.tmp(); + B.line( + `${lenPtr} = getelementptr inbounds %${integerArrayLoop ? "ScrArr" : "ScrBytes"}, ptr ${receiver.name}, i64 0, i32 1`, + ); + if (integerArrayLoop) this.markMemoryPointer(lenPtr, "array:header"); + B.line( + `${len} = load ${this.sizeType}, ptr ${lenPtr}${this.fieldAliasAttachment(lenPtr)}`, + ); + B.line(`${index} = load ${this.sizeType}, ptr ${integerSlot}`); + B.line(`${inBounds} = icmp ult ${this.sizeType} ${index}, ${len}`); + B.condBr(inBounds, lb, exit); + } else if (countedLoop && integerSlot && countedLimit) { + const index = B.tmp(); + const inBounds = B.tmp(); + B.line(`${index} = load i64, ptr ${integerSlot}`); + B.line( + `${inBounds} = icmp ${countedLoop.step > 0 ? "slt" : "sgt"} i64 ${index}, ${countedLimit}`, + ); + B.condBr(inBounds, lb, exit); + } else if (s.cond) B.condBr(this.emitCondition(s.cond), lb, exit); + else B.br(lb); + B.startBlock(lb); + this.jumpTargets.push({ + kind: "loop", + brkLabel: exit, + contLabel: lu, + labels: s.labels, + frameDepth: this.frames.length, + scopeDepth: this.scopes.length, + finallyDepth: this.finallyStack.length, + }); + this.emitBlock(s.body); + this.jumpTargets.pop(); + B.br(lu); + if (lu !== lc) { + B.startBlock(lu); + this.emitWorkerLoopCheck(workerBudget); + freshenBindings(); + if (integerLoop && integerSlot) { + const old = B.tmp(); + const next = B.tmp(); + B.line(`${old} = load ${this.sizeType}, ptr ${integerSlot}`); + B.line(`${next} = add nuw ${this.sizeType} ${old}, 1`); + B.line(`store ${this.sizeType} ${next}, ptr ${integerSlot}`); + } else if (countedLoop && integerSlot) { + const old = B.tmp(); + const next = B.tmp(); + B.line(`${old} = load i64, ptr ${integerSlot}`); + B.line(`${next} = add nsw i64 ${old}, ${countedLoop.step}`); + B.line(`store i64 ${next}, ptr ${integerSlot}`); + } else if (s.update) this.emitStmt(s.update); + B.br(lc); + } + }); B.startBlock(le); this.releaseScope(this.scopes.pop()!); if (integerLoop) this.integerLoopBindings.delete(integerLoop.localId); diff --git a/packages/compiler/src/backend/llvm/speculative-integers.test.ts b/packages/compiler/src/backend/llvm/speculative-integers.test.ts new file mode 100644 index 000000000..ae9172298 --- /dev/null +++ b/packages/compiler/src/backend/llvm/speculative-integers.test.ts @@ -0,0 +1,196 @@ +import { expect, test } from "vitest"; +import { F64, type IrExpr, type IrStmt } from "../../ir/ir.js"; +import { + isSpeculableValue, + planSpeculativeLoop, + speculativeReads, +} from "./speculative-integers.js"; + +const loc = { file: "speculative.ts", start: 0, end: 0 }; +const num = (value: number): IrExpr => ({ kind: "numLit", value, type: F64, loc }); +const ref = (localId: string): IrExpr => ({ kind: "varRef", localId, type: F64, loc }); +const bin = (op: "+" | "-" | "*" | "/" | "%" | "<" | "|", left: IrExpr, right: IrExpr): IrExpr => ({ + kind: "bin", + op, + left, + right, + type: op === "<" ? { kind: "bool" } : F64, + loc, +}); +const assign = (localId: string, value: IrExpr): IrStmt => ({ + kind: "assign", + localId, + value, + loc, +}); +const increment = (localId: string): IrStmt => ({ + kind: "exprStmt", + expr: { kind: "incDec", op: "+", prefix: false, localId, type: F64, loc }, + loc, +}); +const loop = (body: IrStmt[], cond: IrExpr = bin("<", ref("i"), ref("n"))): IrStmt => ({ + kind: "while", + cond, + body, + loc, +}); +const all = (): boolean => true; + +test("integer writes and their loop-invariant inputs are speculated", () => { + const write = assign( + "v", + bin("%", bin("+", bin("*", ref("v"), num(31)), ref("k")), num(1000003)), + ); + const step = increment("i"); + const plan = planSpeculativeLoop(loop([write, step]) as never, all)!; + expect(plan.writes).toEqual(new Set([write, step])); + // `n` (condition) and `k` (write input) are invariant and checked on entry. + expect([...plan.locals]).toEqual([ + ["v", true], + ["i", true], + ["k", true], + ["n", true], + ]); +}); + +test("locals declared in the body need no entry check", () => { + const declare: IrStmt = { kind: "varDecl", localId: "t", init: bin("*", ref("i"), num(2)), loc }; + const plan = planSpeculativeLoop( + loop([ + declare, + assign("s", bin("%", bin("*", ref("s"), ref("t")), num(65521))), + increment("i"), + ]) as never, + all, + )!; + expect(plan.locals.get("t")).toBe(false); + expect(plan.locals.get("s")).toBe(true); +}); + +test("non-integer, expression-position and ineligible writes keep the double copy", () => { + expect(isSpeculableValue(bin("/", ref("a"), num(2)))).toBe(false); + expect(isSpeculableValue(num(0.5))).toBe(false); + expect(isSpeculableValue(num(-0))).toBe(false); + expect(isSpeculableValue(bin("|", ref("a"), num(0)))).toBe(true); + // A fractional write leaves only the counter, which alone is not worth a copy. + const halves = loop([assign("x", bin("/", ref("x"), num(2))), increment("i")]); + expect(planSpeculativeLoop(halves as never, all)?.locals.has("x")).toBeFalsy(); + // A write inside a sequence expression cannot restart its statement. + const sequence: IrStmt = { + kind: "exprStmt", + expr: { + kind: "seqExpr", + stmts: [assign("v", bin("+", ref("v"), num(1)))], + result: num(0), + type: F64, + loc, + }, + loc, + }; + const plan = planSpeculativeLoop( + loop([sequence, assign("v", bin("+", ref("v"), num(2))), increment("i")]) as never, + all, + ); + expect(plan?.locals.has("v")).toBeFalsy(); + expect( + planSpeculativeLoop( + loop([assign("v", bin("+", ref("v"), num(2)))]) as never, + (id) => id !== "v", + ), + ).toBeNull(); +}); + +test("accumulators that are only added to keep their double", () => { + const sum = loop([assign("s", bin("+", ref("s"), bin("%", ref("i"), num(7)))), increment("i")]); + expect(planSpeculativeLoop(sum as never, all)?.locals.has("s")).toBeFalsy(); +}); + +test("nested loops are left to the inner loop", () => { + const inner = loop([assign("v", bin("+", ref("v"), num(1)))]); + expect(planSpeculativeLoop(loop([inner, increment("i")]) as never, all)).toBeNull(); +}); + +test("statically proven locals are never shadowed and leave nothing to duplicate", () => { + const write = assign( + "v", + bin("%", bin("+", bin("*", ref("v"), num(31)), ref("k")), num(1000003)), + ); + const body = [write, increment("i")]; + // Every read proven: no plan at all. + expect( + planSpeculativeLoop( + loop(body) as never, + all, + () => false, + () => true, + ), + ).toBeNull(); + // Only `k` proven: `v` and the counter are shadowed, `k` is not checked. + const plan = planSpeculativeLoop( + loop(body) as never, + all, + () => false, + (e) => e.kind === "varRef" && e.localId === "k", + )!; + expect(plan.locals.has("v")).toBe(true); + expect(plan.locals.has("k")).toBe(false); +}); + +test("heap reads are speculative leaves, read before any check", () => { + const rows: IrExpr = { + kind: "varRef", + localId: "rows", + type: { kind: "array", elem: F64 }, + loc, + } as never; + const element: IrExpr = { kind: "arrayGet", arr: rows, index: ref("i"), type: F64, loc }; + const text: IrExpr = { kind: "varRef", localId: "s", type: { kind: "string" }, loc } as never; + const code: IrExpr = { + kind: "strIntrinsic", + method: "charCodeAt", + receiver: text, + args: [ref("i")], + type: F64, + loc, + } as never; + const value = bin("%", bin("+", bin("*", ref("h"), num(33)), code), num(2147483647)); + expect(isSpeculableValue(value)).toBe(true); + expect(speculativeReads(bin("+", element, code))).toEqual([element, code]); + // The ordinary emitter evaluates bitwise operators; heap reads stay out. + expect(isSpeculableValue(bin("|", code, num(0)))).toBe(false); + const eligible = (id: string): boolean => id !== "rows" && id !== "s"; + const plan = planSpeculativeLoop(loop([assign("h", value), increment("i")]) as never, eligible)!; + expect(plan.locals.has("h")).toBe(true); +}); + +test("invariants may come from bindings no shadow could be written to", () => { + const write = assign("h", bin("%", bin("+", bin("*", ref("h"), num(33)), ref("i")), ref("m"))); + const body = [write, increment("i")]; + const locals = (id: string): boolean => id !== "m"; + expect(planSpeculativeLoop(loop(body) as never, locals)?.locals.has("h")).toBeFalsy(); + const plan = planSpeculativeLoop( + loop(body) as never, + locals, + () => false, + () => false, + all, + )!; + expect(plan.locals.get("m")).toBe(true); + expect(plan.locals.has("h")).toBe(true); +}); + +test("loops that only count and compare keep one copy", () => { + const counter = loop([increment("i")]); + expect(planSpeculativeLoop(counter as never, all)).toBeNull(); + // A product in the condition alone is not. + const bounded = loop([increment("i")], bin("<", ref("i"), bin("*", ref("n"), ref("scale")))); + expect(planSpeculativeLoop(bounded as never, all)).toBeNull(); + // A variable step (a sieve's `j += i`) is worth a copy. + const sieve = loop([assign("i", bin("+", ref("i"), ref("step")))]); + expect(planSpeculativeLoop(sieve as never, all)?.locals.has("i")).toBe(true); + const scaled = loop([ + assign("j", bin("+", ref("j"), bin("*", ref("i"), ref("n")))), + increment("i"), + ]); + expect(planSpeculativeLoop(scaled as never, all)?.locals.has("n")).toBe(true); +}); diff --git a/packages/compiler/src/backend/llvm/speculative-integers.ts b/packages/compiler/src/backend/llvm/speculative-integers.ts new file mode 100644 index 000000000..cf1295fa2 --- /dev/null +++ b/packages/compiler/src/backend/llvm/speculative-integers.ts @@ -0,0 +1,634 @@ +import type { IrExpr, IrStmt } from "../../ir/ir.js"; +import { everyExprChild, everyStmtChild } from "../../ir/traverse.js"; +import { f64Lit } from "./common.js"; +import type { LlValue, LlvmEmitterContext } from "./expr-context.js"; +import { exactInteger, widenInteger } from "./integer-values.js"; +import { withinInt32 } from "../../ir/integer-ranges.js"; + +/** Speculative integer loops. + * + * An innermost loop whose number locals are only written by statement-level + * integer arithmetic (`x = x * 31 + n`, `x += y % m`, `x++`) runs as two + * copies. The fast copy keeps those locals in i64 shadow slots holding safe + * integers and computes the writes with checked integer arithmetic; every + * write also stores the exact double, so the ordinary slot is always current + * and loop exits, calls and jumps need no write-back. A check that fails + * (overflow past 2^53 - 1, a result of -0, a fractional or non-finite leaf, a + * zero divisor) branches to the same statement in the ordinary double copy, + * which re-executes it and finishes the loop there. Re-execution is exact + * because a speculated statement evaluates nothing but locals, literals and + * arithmetic before it writes. + * + * The loop is entered through the fast copy only when every speculated local + * that is live on entry already holds a safe integer other than -0. */ +export interface SpeculativeLoopPlan { + /** Speculated locals; `true` when the entry value is live and checked. */ + locals: Map; + /** Statement-level writes computed as integers in the fast copy. */ + writes: Set; + /** Nothing in the loop can observe a module global or leave it other + * than through its condition, `break` or a failed check: no calls, no + * throwing operations, no labeled jumps and no `return`. */ + observerFree: boolean; +} + +type SpeculativeLoop = Extract; + +/** The fast copy duplicates the loop; keep the duplicated code small. */ +const MAX_LOOP_NODES = 400; + +/** Statements whose emission is plain control flow or a single write. + * Nested loops, switches and try statements keep the loop on the ordinary + * path (their emission keeps state that the two copies would not share). */ +const SIMPLE_STATEMENTS = new Set([ + "varDecl", + "assign", + "exprStmt", + "if", + "block", + "arraySet", + "bytesSet", + "fieldSet", + "recordSet", + "break", + "continue", + "return", +]); + +/** Statements and expressions that run no other code and cannot throw. */ +const OBSERVER_FREE_STATEMENTS = new Set([ + "varDecl", + "assign", + "exprStmt", + "if", + "block", + "break", + "continue", +]); +const OBSERVER_FREE_EXPRESSIONS = new Set([ + "numLit", + "boolLit", + "varRef", + "bin", + "unary", + "incDec", + "assignExpr", + "toBool", + "logical", + "ternary", +]); + +/** Operators whose double form costs conversions or long latency, and that + * take an integer operand directly. */ +const CONSUMING_OPS = new Set([ + "*", + "/", + "%", + "**", + "&", + "|", + "^", + "<<", + ">>", + ">>>", + "<", + "<=", + ">", + ">=", + "===", + "!==", +]); + +const INTEGER_OPS = new Set(["+", "-", "*", "%", "&", "|", "^", "<<", ">>", ">>>"]); +const ARITHMETIC_OPS = new Set(["+", "-", "*", "%"]); + +/** A number expression that reads only locals and literals through integer + * operators: evaluating it has no effects, so a failed check may restart its + * statement. Division and exponentiation usually leave the integers and + * would only bail. */ +export function isSpeculableValue(e: IrExpr, reads = true): boolean { + if (e.type.kind !== "f64") return false; + switch (e.kind) { + case "numLit": + return Number.isSafeInteger(e.value) && !Object.is(e.value, -0); + case "varRef": + return true; + case "bin": { + if (!INTEGER_OPS.has(e.op)) return false; + // Bitwise operators evaluate through the ordinary emitter, which + // must not read the heap after a check (see speculativeReads). + const inner = reads && ARITHMETIC_OPS.has(e.op); + return isSpeculableValue(e.left, inner) && isSpeculableValue(e.right, inner); + } + case "unary": + return (e.op === "-" || e.op === "~") && isSpeculableValue(e.operand, reads && e.op === "-"); + default: + return reads && isSpeculativeRead(e); + } +} + +/** A number read from a string, an array, a record or an object field that + * has no effect: it may throw (a missing element), but it changes nothing, + * so a failed check may restart its statement. Its value carries no range, + * so it is checked like any other unproven integer. */ +export function isSpeculativeRead(e: IrExpr): boolean { + if (e.type.kind !== "f64") return false; + switch (e.kind) { + case "strIntrinsic": + return ( + e.method === "charCodeAt" && + isEffectFree(e.receiver) && + e.args.length === 1 && + isEffectFree(e.args[0]!) + ); + case "arrayGet": + return isEffectFree(e.arr) && isEffectFree(e.index); + case "recordGet": + case "fieldGet": + return isEffectFree(e.obj); + default: + return false; + } +} + +/** Receivers and indexes of speculative reads: locals, literals, number + * arithmetic and nested reads. */ +function isEffectFree(e: IrExpr): boolean { + switch (e.kind) { + case "numLit": + case "varRef": + return true; + case "bin": + return e.left.type.kind === "f64" && isEffectFree(e.left) && isEffectFree(e.right); + case "arrayGet": + return isEffectFree(e.arr) && isEffectFree(e.index); + case "recordGet": + case "fieldGet": + return isEffectFree(e.obj); + default: + return false; + } +} + +/** The speculative reads of a speculable value in evaluation order. */ +export function speculativeReads(e: IrExpr, out: IrExpr[] = []): IrExpr[] { + if (e.kind === "bin" && ARITHMETIC_OPS.has(e.op)) { + speculativeReads(e.left, out); + speculativeReads(e.right, out); + } else if (e.kind === "unary" && e.op === "-") speculativeReads(e.operand, out); + else if (isSpeculativeRead(e)) out.push(e); + return out; +} + +/** `x = x + y` or `x = x - y` with a step that is not a literal, as in a + * sieve's `j += i`: the double copy pays a conversion of every index and + * the step's own latency, while counters stepping by a literal usually + * have a static proof or too little work to repay a second copy. */ +function isVariableStep(localId: string, e: IrExpr): boolean { + if (e.kind !== "bin" || (e.op !== "+" && e.op !== "-")) return false; + const self = (x: IrExpr): boolean => x.kind === "varRef" && x.localId === localId; + const step = self(e.left) ? e.right : e.op === "+" && self(e.right) ? e.left : null; + return step !== null && step.kind !== "numLit"; +} + +function hasProduct(e: IrExpr): boolean { + if (e.kind === "bin") + return e.op === "*" || e.op === "%" || hasProduct(e.left) || hasProduct(e.right); + if (e.kind === "unary") return hasProduct(e.operand); + return false; +} + +function hasArithmetic(e: IrExpr): boolean { + if (e.kind === "bin") + return ARITHMETIC_OPS.has(e.op) || hasArithmetic(e.left) || hasArithmetic(e.right); + if (e.kind === "unary") return e.op === "-" || hasArithmetic(e.operand); + return false; +} + +/** The written local and value of a statement-level write, or null. A + * value of null with a local means `x++`/`x--`. */ +export function speculativeWriteTarget( + s: IrStmt, +): { localId: string; value: IrExpr | null; op: "+" | "-" } | null { + switch (s.kind) { + case "assign": + return { localId: s.localId, value: s.value, op: "+" }; + case "varDecl": + return s.init ? { localId: s.localId, value: s.init, op: "+" } : null; + case "exprStmt": + if (s.expr.kind === "incDec") return { localId: s.expr.localId, value: null, op: s.expr.op }; + if (s.expr.kind === "assignExpr") + return { localId: s.expr.localId, value: s.expr.value, op: "+" }; + return null; + default: + return null; + } +} + +/** `eligible` names the locals a shadow may stand for; `integral` those the + * emitter already holds as integers (counters, int32 locals); `proven` the + * expressions the static range analysis already proves integral. Static + * proofs come first: a local whose reads are all proven is never shadowed + * or checked, and a loop with nothing unproven is never duplicated. */ +export function planSpeculativeLoop( + loop: SpeculativeLoop, + eligible: (localId: string) => boolean, + integral: (localId: string) => boolean = () => false, + proven: (e: IrExpr) => boolean = () => false, + invariantEligible: (localId: string) => boolean = eligible, +): SpeculativeLoopPlan | null { + const writes = new Map(); + const unspeculable = new Set(); + const reads = new Set(); + // Locals with at least one read the static analysis does not prove. + const unproven = new Set(); + const declared = new Set(); + let nodes = 0; + let rejected = false; + let arithmetic = false; + let observerFree = true; + // Locals the loop reads as integers: operands of multiplication, + // remainder, bitwise operators and comparisons, and indexes. An + // accumulator only ever added to gains little from an integer shadow + // (the double add is cheap), so it keeps its double. + const consumed = new Set(); + const consume = (e: IrExpr): void => { + if (e.kind === "varRef") consumed.add(e.localId); + }; + // Operands of multiplication, division and remainder: the operations whose + // double forms cost the most. A loop whose integers only count and compare + // gains nothing measurable from a second copy, so it keeps one. + const multiplied = new Set(); + // A product in the condition alone (`i < Math.floor(n * scale)`) does not + // make the body's work integer. + let inCondition = false; + + const expr = (e: IrExpr): boolean => { + if (rejected) return false; + nodes++; + if (!OBSERVER_FREE_EXPRESSIONS.has(e.kind) || (e.kind === "bin" && e.left.type.kind !== "f64")) + observerFree = false; + if (e.kind === "varRef") { + reads.add(e.localId); + if (!proven(e)) unproven.add(e.localId); + } else if (e.kind === "assignExpr" || e.kind === "incDec") unspeculable.add(e.localId); + else if (e.kind === "bin" && CONSUMING_OPS.has(e.op)) { + consume(e.left); + consume(e.right); + if (!inCondition && (e.op === "*" || e.op === "/" || e.op === "%")) + for (const operand of [e.left, e.right]) + if (operand.kind === "varRef") multiplied.add(operand.localId); + } else if (e.kind === "arrayGet" || e.kind === "arrayHas" || e.kind === "arrayState") + consume(e.index); + else if (e.kind === "bytesIntrinsic" || e.kind === "strIntrinsic") e.args.forEach(consume); + return everyExprChild(e, expr, nested); + }; + // Statements inside expressions (sequence expressions) run mid-expression, + // where no statement can restart: their writes stay unspeculated. + const nested = (s: IrStmt): boolean => { + nodes++; + observerFree = false; + if (s.kind === "assign" || s.kind === "varDecl") unspeculable.add(s.localId); + return everyStmtChild(s, expr, nested); + }; + const stmt = (s: IrStmt): boolean => { + if (rejected) return false; + nodes++; + if (!SIMPLE_STATEMENTS.has(s.kind)) { + rejected = true; + return false; + } + if ( + !OBSERVER_FREE_STATEMENTS.has(s.kind) || + ((s.kind === "break" || s.kind === "continue") && s.label !== undefined) + ) + observerFree = false; + if (s.kind === "varDecl") declared.add(s.localId); + if (s.kind === "arraySet" || s.kind === "bytesSet") consume(s.index); + const target = speculativeWriteTarget(s); + if (target) { + if (target.value === null || isSpeculableValue(target.value)) { + const list = writes.get(target.localId) ?? []; + list.push(s); + writes.set(target.localId, list); + if (target.value === null) { + reads.add(target.localId); + if (s.kind !== "exprStmt" || !proven(s.expr)) unproven.add(target.localId); + } else expr(target.value); + return !rejected; + } + unspeculable.add(target.localId); + if (target.value) expr(target.value); + return !rejected; + } + return everyStmtChild(s, expr, stmt); + }; + + if (loop.cond) { + inCondition = true; + expr(loop.cond); + inCondition = false; + } + if (loop.kind === "for" && loop.update) stmt(loop.update); + for (const s of loop.body) stmt(s); + if (rejected || nodes > MAX_LOOP_NODES) return null; + + const locals = new Map(); + for (const id of writes.keys()) + if (!unspeculable.has(id) && consumed.has(id) && unproven.has(id) && eligible(id)) + locals.set(id, !declared.has(id)); + // Every local a speculated write reads must be an integer whenever the + // write runs: speculated itself, already an integer, or loop-invariant + // (checked on entry). A value computed from doubles the loop writes + // would only fail its check on every entry. + const grounded = (e: IrExpr): boolean => { + if (e.kind === "varRef") { + const id = e.localId; + return ( + proven(e) || + locals.has(id) || + integral(id) || + (!writes.has(id) && !unspeculable.has(id) && invariantEligible(id)) + ); + } + if (isSpeculativeRead(e)) return true; + return everyExprChild(e, grounded, () => false); + }; + for (let changed = true; changed;) { + changed = false; + for (const id of [...locals.keys()]) { + for (const s of writes.get(id)!) { + const target = speculativeWriteTarget(s)!; + if (target.value !== null && !grounded(target.value)) { + locals.delete(id); + changed = true; + break; + } + } + } + } + const speculated = new Set(); + for (const id of locals.keys()) for (const s of writes.get(id)!) speculated.add(s); + if (speculated.size === 0) return null; + // Loop-invariant integer inputs of the writes and the condition join the + // shadows, so their reads carry integer facts too. + const invariant = (e: IrExpr): boolean => { + if (e.kind === "varRef") { + const id = e.localId; + if ( + !proven(e) && + !locals.has(id) && + !writes.has(id) && + !unspeculable.has(id) && + invariantEligible(id) + ) + locals.set(id, true); + return true; + } + return everyExprChild(e, invariant, () => true); + }; + for (const s of speculated) { + const target = speculativeWriteTarget(s)!; + if (target.value === null) arithmetic = true; + else { + if (hasArithmetic(target.value)) arithmetic = true; + invariant(target.value); + } + } + if (!arithmetic) return null; + if (loop.cond) invariant(loop.cond); + const heavy = + [...locals.keys()].some((id) => multiplied.has(id)) || + [...speculated].some((s) => { + const target = speculativeWriteTarget(s)!; + return ( + target.value !== null && + (hasProduct(target.value) || isVariableStep(target.localId, target.value)) + ); + }); + if (!heavy) return null; + return { locals, writes: speculated, observerFree }; +} + +const SAFE = Number.MAX_SAFE_INTEGER; + +/** The range every speculative shadow holds. */ +export const SPECULATIVE_RANGE = { min: -SAFE, max: SAFE }; + +/** Convert a double to i64 and test that it is a safe integer other than + * -0: the ordered range tests reject NaN and infinities, and the bit + * patterns of the value and its round trip agree only for integers that are + * not -0. `freeze` keeps out-of-range conversions from producing poison. */ +export function emitSafeIntegerCheck(host: LlvmEmitterContext, value: string): [string, string] { + const B = host.B; + const low = B.tmp(), + high = B.tmp(), + inRange = B.tmp(), + raw = B.tmp(), + integer = B.tmp(), + back = B.tmp(), + bits = B.tmp(), + backBits = B.tmp(), + exact = B.tmp(), + ok = B.tmp(); + B.line(`${low} = fcmp oge double ${value}, ${f64Lit(-SAFE)}`); + B.line(`${high} = fcmp ole double ${value}, ${f64Lit(SAFE)}`); + B.line(`${inRange} = and i1 ${low}, ${high}`); + B.line(`${raw} = fptosi double ${value} to i64`); + B.line(`${integer} = freeze i64 ${raw}`); + B.line(`${back} = sitofp i64 ${integer} to double`); + B.line(`${bits} = bitcast double ${value} to i64`); + B.line(`${backBits} = bitcast double ${back} to i64`); + B.line(`${exact} = icmp eq i64 ${bits}, ${backBits}`); + B.line(`${ok} = and i1 ${inRange}, ${exact}`); + return [integer, ok]; +} + +/** A condition that is true when an i64 lies outside the safe range. */ +function unsafeInteger(host: LlvmEmitterContext, value: string): string { + const B = host.B; + const biased = B.tmp(), + outside = B.tmp(); + B.line(`${biased} = add i64 ${value}, ${SAFE}`); + B.line(`${outside} = icmp ugt i64 ${biased}, ${2 * SAFE}`); + return outside; +} + +/** A condition that is true when a safe i64 lies outside [low, high]. Every + * speculative value is within ±(2^53 - 1), so bounds at or past those + * limits need no test, and the remaining bounds are exact integers. */ +function outsideRange(host: LlvmEmitterContext, value: string, low: number, high: number): string { + const B = host.B; + let outside = "false"; + if (low > -SAFE) { + outside = B.tmp(); + B.line(`${outside} = icmp slt i64 ${value}, ${low}`); + } + if (high < SAFE) { + const above = B.tmp(); + B.line(`${above} = icmp sgt i64 ${value}, ${high}`); + outside = outside === "false" ? above : either(host, outside, above); + } + return outside; +} + +/** A literal's value or the range analysis' proof for an expression whose + * range lies well inside the safe integers. */ +function staticRange( + host: LlvmEmitterContext, + e: IrExpr, +): { min: number; max: number } | undefined { + const range = e.kind === "numLit" ? { min: e.value, max: e.value } : host.integerRanges.get(e); + if (!range) return undefined; + const limit = 2 ** 52; + return range.min >= -limit && range.max <= limit ? range : undefined; +} + +function either(host: LlvmEmitterContext, a: string, b: string): string { + const t = host.B.tmp(); + host.B.line(`${t} = or i1 ${a}, ${b}`); + return t; +} + +/** Evaluate a speculable value (isSpeculableValue) as a safe i64. `bail` + * branches away when its condition holds, so the integer result equals the + * JS number exactly whenever control continues: sums and products stay + * within 2^53 - 1, no result is -0, and `%` never divides by zero. Other + * operators evaluate through the ordinary emitter and must yield a safe + * integer. */ +export function emitSpeculativeInteger( + host: LlvmEmitterContext, + e: IrExpr, + bail: (condition: string) => void, + reads: ReadonlyMap = new Map(), +): string { + const B = host.B; + // A value the range analysis already proves takes the ordinary integer + // emission, which needs no checks. + const proven = e.kind === "bin" ? host.integerRanges.get(e) : undefined; + if (proven && proven.min >= -SAFE && proven.max <= SAFE && speculativeReads(e).length === 0) + return speculativeLeaf(host, host.emitExpr(e), e, bail); + switch (e.kind) { + case "numLit": + return String(e.value); + case "varRef": { + const slot = host.integerLoopBindings.get(e.localId); + if (slot !== undefined) { + const value = B.tmp(); + B.line(`${value} = load ${slot.type}, ptr ${slot.slot}`); + if (slot.type === "i64") return value; + const wide = B.tmp(); + B.line(`${wide} = ${slot.signed ? "sext" : "zext"} i32 ${value} to i64`); + return wide; + } + const int32 = host.int32Locals.get(e.localId); + if (int32 !== undefined) { + const bits = B.tmp(), + wide = B.tmp(); + B.line(`${bits} = load i32, ptr ${host.binding(e.localId).slot}`); + B.line(`${wide} = ${int32.signed ? "sext" : "zext"} i32 ${bits} to i64`); + return wide; + } + break; + } + case "bin": { + if (e.op !== "+" && e.op !== "-" && e.op !== "*" && e.op !== "%") break; + const a = emitSpeculativeInteger(host, e.left, bail, reads); + const b = emitSpeculativeInteger(host, e.right, bail, reads); + const result = B.tmp(); + if (e.op === "+" || e.op === "-") { + // With one operand's range known, the other is checked against the + // narrowed bounds before the operation: the check then depends on + // no new value, and LLVM keeps the operation itself simple. + const right = staticRange(host, e.right); + const left = e.op === "+" ? staticRange(host, e.left) : undefined; + if (right) { + // a + b with b in [lo, hi] is safe iff a in [-SAFE - lo, SAFE - hi]; + // a - b iff a in [-SAFE + hi, SAFE + lo]. + const low = e.op === "+" ? -SAFE - right.min : -SAFE + right.max; + const high = e.op === "+" ? SAFE - right.max : SAFE + right.min; + bail(outsideRange(host, a, low, high)); + } else if (left) bail(outsideRange(host, b, -SAFE - left.min, SAFE - left.max)); + // Safe operands cannot overflow i64; an exact zero sum is +0. + B.line(`${result} = ${e.op === "+" ? "add" : "sub"} nsw i64 ${a}, ${b}`); + if (!right && !left) bail(unsafeInteger(host, result)); + return result; + } + if (e.op === "*") { + host.declare("declare { i64, i1 } @llvm.smul.with.overflow.i64(i64, i64)"); + const pair = B.tmp(), + overflow = B.tmp(), + zero = B.tmp(), + signs = B.tmp(), + negative = B.tmp(), + negativeZero = B.tmp(); + B.line(`${pair} = call { i64, i1 } @llvm.smul.with.overflow.i64(i64 ${a}, i64 ${b})`); + B.line(`${result} = extractvalue { i64, i1 } ${pair}, 0`); + B.line(`${overflow} = extractvalue { i64, i1 } ${pair}, 1`); + // A zero product with a negative factor is -0. + B.line(`${zero} = icmp eq i64 ${result}, 0`); + B.line(`${signs} = or i64 ${a}, ${b}`); + B.line(`${negative} = icmp slt i64 ${signs}, 0`); + B.line(`${negativeZero} = and i1 ${zero}, ${negative}`); + bail(either(host, either(host, overflow, unsafeInteger(host, result)), negativeZero)); + return result; + } + // `%`: a zero divisor yields NaN, and a zero remainder of a negative + // dividend is -0. Safe operands cannot hit INT64_MIN % -1. + const zeroDivisor = B.tmp(); + B.line(`${zeroDivisor} = icmp eq i64 ${b}, 0`); + bail(zeroDivisor); + const zero = B.tmp(), + negative = B.tmp(), + negativeZero = B.tmp(); + B.line(`${result} = srem i64 ${a}, ${b}`); + B.line(`${zero} = icmp eq i64 ${result}, 0`); + B.line(`${negative} = icmp slt i64 ${a}, 0`); + B.line(`${negativeZero} = and i1 ${zero}, ${negative}`); + bail(negativeZero); + return result; + } + case "unary": { + if (e.op !== "-") break; + const a = emitSpeculativeInteger(host, e.operand, bail, reads); + const zero = B.tmp(), + result = B.tmp(); + // -0 is not an integer result. + B.line(`${zero} = icmp eq i64 ${a}, 0`); + bail(zero); + B.line(`${result} = sub nsw i64 0, ${a}`); + return result; + } + default: + break; + } + return speculativeLeaf(host, reads.get(e) ?? host.emitExpr(e), e, bail); +} + +/** The safe i64 of an evaluated number: its proven integer, or a checked + * conversion. */ +function speculativeLeaf( + host: LlvmEmitterContext, + value: LlValue, + e: IrExpr, + bail: (condition: string) => void, +): string { + const integer = exactInteger(host, value, e); + if (integer && integer.type === "i64" && withinInt32(integer.range)) { + // Only the low half is used, as by a double conversion of an int32: + // LLVM then narrows the operation that produced it (a 64-bit remainder + // by a constant costs a wide multiply). + const narrow = host.B.tmp(), + wide = host.B.tmp(); + host.B.line(`${narrow} = trunc i64 ${integer.name} to i32`); + host.B.line(`${wide} = sext i32 ${narrow} to i64`); + return wide; + } + if (integer && integer.range.min >= -SAFE && integer.range.max <= SAFE) + return widenInteger(host, integer); + const [wide, ok] = emitSafeIntegerCheck(host, value.name); + const fail = host.B.tmp(); + host.B.line(`${fail} = xor i1 ${ok}, true`); + bail(fail); + return wide; +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/speculative-integer-globals.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/speculative-integer-globals.ts.json new file mode 100644 index 000000000..f09db682d --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/speculative-integer-globals.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/speculative-integer-globals.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/speculative-integer-loops.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/speculative-integer-loops.ts.json new file mode 100644 index 000000000..f79d2bfa2 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/speculative-integer-loops.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/speculative-integer-loops.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/speculative-integer-reads.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/speculative-integer-reads.ts.json new file mode 100644 index 000000000..b05283134 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/speculative-integer-reads.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/speculative-integer-reads.ts" + ], + "diags": [] +} diff --git a/tests/corpus/speculative-integer-globals.ts b/tests/corpus/speculative-integer-globals.ts new file mode 100644 index 000000000..fcce1ee5e --- /dev/null +++ b/tests/corpus/speculative-integer-globals.ts @@ -0,0 +1,66 @@ +// Module-level numbers accumulated by top-level loops: calls in the loop +// read them while the loop keeps them as integers, and sums that leave +// the safe integers continue as doubles. + +let total: number = 0; +let calls: number = 0; +let big: number = 9007199254740000; + +function peek(): number { + calls++; + return total; +} + +for (let i: number = 0; i < 1000; i++) { + total = total + (i % 7); + if (total < 0) break; + if (i % 250 === 0) console.log("peek", peek()); +} +console.log("total", total, "calls", calls); + +for (let i: number = 0; i < 3000; i++) { + big += i; + if (big % 7 === -1) break; +} +console.log("big", big, Number.isSafeInteger(big)); + +let product: number = 1; +let rounds: number = 0; +while (rounds < 70) { + product = product * 3; + rounds++; +} +console.log("product", product); + +let zero: number = 1; +for (let i: number = 0; i < 2; i++) zero = zero * -0; +console.log("zero", Object.is(zero, -0)); + +let mixed: number = 10; +for (let i: number = 0; i < 6; i++) { + mixed = mixed - 3; + mixed = mixed * 2; +} +console.log("mixed", mixed); + +let found: number = 0; +for (let i: number = 0; i < 100; i++) { + found = found + i * i; + if (found > 1000) break; +} +console.log("found", found); + +let acc: number = 1; +let steps: number = 0; +while (true) { + acc = acc * 7; + steps++; + if (steps > 30) break; +} +console.log("acc", acc, steps); + +let countdown: number = 5; +do { + countdown = countdown - 2; +} while (countdown > -3); +console.log("countdown", countdown); diff --git a/tests/corpus/speculative-integer-loops.ts b/tests/corpus/speculative-integer-loops.ts new file mode 100644 index 000000000..1397a3357 --- /dev/null +++ b/tests/corpus/speculative-integer-loops.ts @@ -0,0 +1,253 @@ +// Loops whose number locals hold integers until they do not: every check +// that leaves the integer copy (overflow past 2^53 - 1, -0, fractions, +// NaN, zero divisors) must continue with exactly Node's doubles. + +function show(label: string, value: number): void { + console.log(label, Object.is(value, -0) ? "-0" : String(value)); +} + +// A remainder chain that stays small. +function chain(start: number, n: number): number { + let v: number = start; + for (let i: number = 0; i < 100000; i++) v = (v * 31 + n) % 1000003; + return v; +} +show("chain", chain(7, 3)); +show("chain negative", chain(-7, -3)); +show("chain fraction start", chain(7.5, 3)); +show("chain fraction step", chain(7, 0.25)); +show("chain NaN", chain(NaN, 1)); +show("chain -0", chain(-0, 0)); + +// Products that leave the safe integers part-way and keep growing. +function grow(start: number, factor: number): number { + let x: number = start; + let steps: number = 0; + while (steps < 80) { + x = x * factor + 1; + steps++; + } + return x; +} +show("grow 3", grow(1, 3)); +show("grow -3", grow(1, -3)); +show("grow 2", grow(0, 2)); + +// Sums that cross 2^53 - 1 on the way. +function crossing(): number { + let s: number = 9007199254740900; + for (let i: number = 0; i < 200; i++) { + s += i; + if (s % 1000 === -1) break; + } + return s; +} +show("crossing", crossing()); + +// -0 from a product, a remainder and a negation, each in its own loop. +function negativeZeroProduct(k: number): number { + let p: number = 5; + for (let i: number = 0; i < 3; i++) p = p * k; + return p; +} +show("product -0", negativeZeroProduct(0) * -1); +show("product zero", negativeZeroProduct(0)); +function negativeZeroTimes(): number { + let p: number = 0; + let q: number = 1; + for (let i: number = 0; i < 4; i++) { + p = q * (i - 2); + q = p * 0; + } + return q; +} +show("times -0", negativeZeroTimes()); +function negativeZeroRemainder(): number { + let r: number = -9; + let out: number = 1; + for (let i: number = 1; i < 5; i++) { + out = r % 3; + r = r + i; + if (out === 7) break; + } + return out; +} +show("remainder", negativeZeroRemainder()); +function remainderNegZero(): number { + let r: number = 0; + for (let i: number = 0; i < 2; i++) { + r = -6 % (3 + i * 0); + if (r > 100) break; + } + return r; +} +show("remainder -0", remainderNegZero()); +function negation(): number { + let a: number = 3; + let b: number = 1; + for (let i: number = 0; i < 4; i++) { + a = a - 1; + b = -a; + if (a * b === 999) break; + } + return b; +} +show("negation", negation()); +function negationZero(): number { + let a: number = 2; + let b: number = 1; + for (let i: number = 0; i < 2; i++) { + a = a - 1; + b = -a; + if (a * b === 999) break; + } + return b; +} +show("negation -0", negationZero()); + +// A zero divisor part-way through. +function zeroDivisor(): number { + let d: number = 3; + let acc: number = 100; + for (let i: number = 0; i < 6; i++) { + acc = acc % d; + d = d - 1; + acc = acc + 50; + } + return acc; +} +show("zero divisor", zeroDivisor()); + +// Jumps out of the integer copy keep the locals current. +function jumps(limit: number): number { + let total: number = 0; + let count: number = 0; + for (let i: number = 0; i < 1000; i++) { + if (i % 3 === 0) continue; + total = total + i * i; + count++; + if (total > limit) break; + } + return total * 1000 + count; +} +show("jumps", jumps(5000)); +show("jumps huge", jumps(1e300)); +function early(values: number[]): number { + let sum: number = 0; + let i: number = 0; + while (i < values.length) { + sum = sum + i; + if (sum > 10) return sum; + i++; + } + return -sum; +} +show("early", early([1, 2, 3, 4, 5, 6, 7])); +show("early short", early([1, 2])); +function labeled(): number { + let hits: number = 0; + outer: for (let a: number = 0; a < 5; a++) { + let b: number = 0; + while (b < 5) { + b = b + 1; + hits = hits + a * b; + if (b === a) continue outer; + } + } + return hits; +} +show("labeled", labeled()); + +// do-while with a bail in the condition-adjacent update. +function collatz(start: number): number { + let n: number = start; + let steps: number = 0; + do { + n = n % 2 === 0 ? n / 2 : 3 * n + 1; + steps = steps + 1; + } while (n !== 1 && steps < 1000); + return steps; +} +show("collatz", collatz(27)); +function doubling(): number { + let x: number = 1; + let n: number = 0; + do { + x = x * 2; + n++; + } while (n < 60); + return x; +} +show("doubling", doubling()); + +// The update statement itself bails. +function bigStep(): number { + let last: number = 0; + for (let j: number = 1; j < 2e16; j = j * 1000 + 1) last = j; + return last; +} +show("big step", bigStep()); + +// Speculated indexes into typed arrays and arrays. +const bytes: Uint8Array = new Uint8Array(64); +let k: number = 0; +for (let j: number = 3; j < 64; j += 3) { + bytes[j] = j; + k = k + j; +} +show("bytes", bytes[63] + bytes[3] + k); +const squares: number[] = []; +for (let j: number = 0; j < 10; j++) { + let sq: number = j * j; + sq = sq - j; + squares.push(sq); +} +console.log(squares.join(",")); + +// Bitwise leaves and conversions. +function lcg(seed: number): number { + let s: number = seed; + let mixed: number = 0; + for (let i: number = 0; i < 1000; i++) { + s = (s * 1103515245 + 12345) | 0; + mixed = mixed + (s >>> 16); + mixed = mixed % 4294967296; + } + return mixed; +} +show("lcg", lcg(42)); +function unsigned(): number { + let u: number = 0; + for (let i: number = 0; i < 40; i++) u = (u * 2 + 1) >>> 0; + return u; +} +show("unsigned", unsigned()); +function complement(): number { + let c: number = 5; + for (let i: number = 0; i < 3; i++) c = ~c + c * 2; + return c; +} +show("complement", complement()); + +// Inputs that are not integers on entry, and loop-invariant inputs. +function scaled(step: number): number { + let x: number = 0; + for (let i: number = 0; i < 10; i++) { + x = x + step; + if (x % 3 === -1) break; + } + return x; +} +show("scaled int", scaled(3)); +show("scaled fraction", scaled(0.1)); +show("scaled huge", scaled(1e300)); +show("scaled Infinity", scaled(Infinity)); +function invariantNegZero(z: number): number { + let x: number = 0; + for (let i: number = 0; i < 3; i++) { + x = z * 1; + if (x === 5) break; + } + return x; +} +show("invariant -0", invariantNegZero(-0)); diff --git a/tests/corpus/speculative-integer-reads.ts b/tests/corpus/speculative-integer-reads.ts new file mode 100644 index 000000000..86b1eec9f --- /dev/null +++ b/tests/corpus/speculative-integer-reads.ts @@ -0,0 +1,90 @@ +// Speculative integer loops over values read from strings, arrays, records +// and module constants: reads that are not integers (NaN past a string's +// end, fractional elements and fields, a fractional divisor) must continue +// with exactly Node's doubles. + +function show(label: string, value: number): void { + console.log(label, Object.is(value, -0) ? "-0" : String(value)); +} + +function hashText(s: string, extra: number): number { + let h: number = 0; + for (let i: number = 0; i < s.length + extra; i++) h = (h * 33 + s.charCodeAt(i)) % 2147483647; + return h; +} +show("hash", hashText("speculative integer loops", 0)); +show("hash past end", hashText("speculative integer loops", 1)); +show("hash empty", hashText("", 0)); +show("hash unicode", hashText("héllo wörld \u{1F600}", 0)); + +function mixElements(xs: number[]): number { + let a: number = 1; + for (let i: number = 0; i < xs.length; i++) a = (a * 7 + (xs[i]! % 100)) % 65521; + return a; +} +show("elements", mixElements([1, 2, 3, 400, 5000, 60000])); +show("elements fraction", mixElements([1, 2, 3.5, 4, 5])); +show("elements negative", mixElements([1, -2, -300, 4])); +show("elements huge", mixElements([1, 2 ** 60, 3])); +show("elements NaN", mixElements([1, NaN, 3])); + +interface Item { + id: number; + weight: number; +} +function checksum(items: Item[]): number { + let c: number = 0; + for (let i: number = 0; i < items.length; i++) c = (c * 31 + items[i]!.id) % 1000000007; + return c; +} +const items: Item[] = []; +for (let i: number = 0; i < 50; i++) items.push({ id: (i * 7919) % 1000003, weight: i / 4 }); +show("checksum", checksum(items)); +items[25] = { id: 12.25, weight: 0 }; +show("checksum fraction", checksum(items)); +items[25] = { id: -0, weight: 0 }; +show("checksum -0", checksum(items)); + +class Cell { + value: number; + constructor(value: number) { + this.value = value; + } +} +function sumCells(cells: Cell[]): number { + let s: number = 0; + for (let i: number = 0; i < cells.length; i++) s = (s * 3 + cells[i]!.value) % 1000003; + return s; +} +show("cells", sumCells([new Cell(1), new Cell(2), new Cell(3)])); +show("cells fraction", sumCells([new Cell(1), new Cell(2.5), new Cell(3)])); + +// Module constants written only by top-level code act as loop invariants. +const modulus: number = Number("1048573"); +const fractional: number = Number("2.5"); +const zero: number = Number("0"); +function hashWith(n: number, m: number): number { + let h: number = 7; + for (let i: number = 0; i < n; i++) h = (h * 33 + i) % m; + return h; +} +function hashModule(n: number): number { + let h: number = 7; + for (let i: number = 0; i < n; i++) h = (h * 33 + i) % modulus; + return h; +} +function hashFractional(n: number): number { + let h: number = 7; + for (let i: number = 0; i < n; i++) h = (h * 33 + i) % fractional; + return h; +} +function hashZero(n: number): number { + let h: number = 7; + for (let i: number = 0; i < n; i++) h = (h * 33 + i) % zero; + return h; +} +show("module modulus", hashModule(10000)); +show("module fractional", hashFractional(1000)); +show("module zero", hashZero(10)); +show("parameter modulus", hashWith(10000, modulus)); +show("parameter fractional", hashWith(100, fractional));