diff --git a/packages/compiler/src/backend/llvm/byte-numbers.ts b/packages/compiler/src/backend/llvm/byte-numbers.ts index c51d03744a..9001e015dc 100644 --- a/packages/compiler/src/backend/llvm/byte-numbers.ts +++ b/packages/compiler/src/backend/llvm/byte-numbers.ts @@ -4,6 +4,7 @@ import type { LlValue, LlvmEmitterContext } from "./expr-context.js"; import { f64Lit } from "./common.js"; import { exactInteger, widenInteger } from "./integer-values.js"; import { emitTruncF64 } from "./trunc.js"; +import { bytesElementAlias, bytesHeaderAlias } from "./bytes-alias.js"; /** Supported LLVM targets are little-endian. All memory accesses use align * 1: Buffer fields and DataView windows can start at arbitrary byte offsets. */ @@ -79,7 +80,7 @@ export function emitByteNumber( capacity = B.tmp(); room = B.tmp(); B.line(`${lengthPtr} = getelementptr inbounds %ScrBytes, ptr ${receiver.name}, i64 0, i32 1`); - B.line(`${length} = load ${size}, ptr ${lengthPtr}`); + B.line(`${length} = load ${size}, ptr ${lengthPtr}${bytesHeaderAlias(host)}`); B.line(`${room} = icmp uge ${size} ${length}, ${spec.width}`); B.line(`${capacity} = sub ${size} ${length}, ${spec.width}`); if (integer) { @@ -221,14 +222,14 @@ export function emitByteNumber( } } const stored = endianBits(host, raw, spec.width, little); - B.line(`store i${bits} ${stored}, ptr ${pointer}, align 1`); + B.line(`store i${bits} ${stored}, ptr ${pointer}, align 1${bytesElementAlias(host)}`); if (!spec.dataView) { fastResult = B.tmp(); B.line(`${fastResult} = fadd double ${offset.name}, ${f64Lit(spec.width)}`); } } else { const loaded = B.tmp(); - B.line(`${loaded} = load i${bits}, ptr ${pointer}, align 1`); + B.line(`${loaded} = load i${bits}, ptr ${pointer}, align 1${bytesElementAlias(host)}`); const raw = endianBits(host, loaded, spec.width, little); fastResult = B.tmp(); if (spec.floating) { diff --git a/packages/compiler/src/backend/llvm/bytes-alias.ts b/packages/compiler/src/backend/llvm/bytes-alias.ts new file mode 100644 index 0000000000..9aaeaa6c77 --- /dev/null +++ b/packages/compiler/src/backend/llvm/bytes-alias.ts @@ -0,0 +1,23 @@ +import type { LlvmEmitterContext } from "./expr-context.js"; + +/** Alias categories of typed-array storage. A ScrBytes header (length, + * data pointer, shared backing) is fixed at construction and only ever + * written by runtime code, never by emitted code. Element storage is one + * category for every element type: views of different types may share a + * buffer. Neither overlaps the other, a native field, array storage or a + * module global's slot, so element stores leave header loads and global + * receiver slots loop-invariant. Untagged accesses and calls still alias + * every category. */ +export function bytesHeaderAlias(host: LlvmEmitterContext): string { + return host.memoryAliasAttachment("bytes:header"); +} + +export function bytesElementAlias(host: LlvmEmitterContext): string { + return host.memoryAliasAttachment("bytes:elements"); +} + +/** A module global's slot is its own category for whole-slot loads of a + * scalar or typed-array binding. Stores stay untagged (and so alias it). */ +export function globalSlotAlias(host: LlvmEmitterContext, id: string, type: string): string { + return type === "bytes" || type === "f64" ? host.memoryAliasAttachment(`global:${id}`) : ""; +} diff --git a/packages/compiler/src/backend/llvm/bytes-loop-versions.test.ts b/packages/compiler/src/backend/llvm/bytes-loop-versions.test.ts new file mode 100644 index 0000000000..d8ec036fff --- /dev/null +++ b/packages/compiler/src/backend/llvm/bytes-loop-versions.test.ts @@ -0,0 +1,210 @@ +import { expect, test } from "vitest"; +import { + BOOL, + F64, + VOID, + type IrExpr, + type IrFunction, + type IrLocal, + type IrModule, + type IrStmt, + type IrType, +} from "../../ir/ir.js"; +import { analyzeIntegerRanges } from "../../ir/integer-ranges.js"; +import { matchIntegerCountedForLoop } from "../../ir/integer-loops.js"; +import { findInitializedByteLoopBindings } from "./byte-windows.js"; +import { matchBytesLoopVersion, type BytesLoopVersion } from "./bytes-loop-versions.js"; +import { emitLlvmModule } from "./emitter.js"; + +const loc = { file: "versions.ts", start: 0, end: 0 }; +const f64s: IrType = { kind: "bytes", elem: "f64" }; +const i32s: IrType = { kind: "bytes", elem: "i32" }; +const num = (value: number): IrExpr => ({ kind: "numLit", value, type: F64, loc }); +const ref = (localId: string, type: IrType = F64): IrExpr => ({ + kind: "varRef", + localId, + type, + loc, +}); +const bin = (op: string, left: IrExpr, right: IrExpr): IrExpr => + ({ kind: "bin", op, left, right, type: F64, loc }) as IrExpr; +const get = (receiver: IrExpr, index: IrExpr): IrExpr => ({ + kind: "bytesIntrinsic", + method: "get", + receiver, + args: [index], + invalidNaN: true, + type: F64, + loc, +}); +const set = (arr: IrExpr, index: IrExpr, value: IrExpr): IrStmt => ({ + kind: "bytesSet", + arr, + index, + value, + loc, +}); +const counted = ( + body: IrStmt[], + limit: IrExpr = ref("n"), + counter = "i", +): IrStmt & { kind: "for" } => ({ + kind: "for", + init: { kind: "varDecl", localId: counter, init: num(0), loc }, + cond: { kind: "bin", op: "<", left: ref(counter), right: limit, type: BOOL, loc }, + update: { kind: "assign", localId: counter, value: bin("+", ref(counter), num(1)), loc }, + body, + loc, +}); +const local = ( + id: string, + type: IrType, + mutable = false, + extra: Partial = {}, +): IrLocal => ({ + id, + name: id, + type, + mutable, + ...extra, +}); + +function fn(body: IrStmt[], extra: IrLocal[] = []): IrFunction { + return { + name: "kernel", + params: [ + { localId: "out", name: "out", type: f64s }, + { localId: "src", name: "src", type: f64s }, + { localId: "table", name: "table", type: i32s }, + { localId: "n", name: "n", type: F64 }, + { localId: "k", name: "k", type: F64 }, + ], + returnType: VOID, + locals: [ + local("out", f64s), + local("src", f64s), + local("table", i32s), + local("n", F64), + local("k", F64), + local("i", F64, true), + local("j", F64, true), + ...extra, + ], + body, + loc, + }; +} + +function match( + f: IrFunction, + loop: IrStmt & { kind: "for" }, + captured: string[] = [], +): BytesLoopVersion | null { + const ranges = analyzeIntegerRanges(f); + const locals = new Map(f.locals.map((l) => [l.id, l])); + return matchBytesLoopVersion(loop, matchIntegerCountedForLoop(loop, locals, ranges), { + locals, + invariantGlobals: new Set(), + captured: new Set(captured), + ranges, + initialized: findInitializedByteLoopBindings(f).get(loop) ?? new Set(), + proven: new Set(), + }); +} + +test("a counted loop's affine accesses get one guard and exact operand ranges", () => { + const shifted = bin("+", ref("i"), num(1)); + const loop = counted([set(ref("out", f64s), ref("i"), get(ref("src", f64s), shifted))]); + const version = match(fn([loop]), loop)!; + expect(version).not.toBeNull(); + expect(version.bounds.size).toBe(2); + expect(version.receivers.map((r) => r.ref.kind === "varRef" && r.ref.localId)).toEqual([ + "out", + "src", + ]); + expect(version.affine.map((c) => [c.scale, c.minOffset, c.maxEnd])).toEqual([ + [1, 0, 0], + [1, 1, 1], + ]); + // `n` is unproven: the guard bounds the counter so `i + 1` stays exact. + expect(version.counted?.limitGuarded).toBe(true); + expect(version.counterBound).toBe(Number.MAX_SAFE_INTEGER - 1); + expect(version.ranges.get(shifted)).toEqual({ min: 1, max: Number.MAX_SAFE_INTEGER }); +}); + +test("statically ranged indices compare one constant with the length", () => { + const masked = bin("&", bin("^", ref("i"), ref("k")), num(255)); + const loop = counted([set(ref("out", f64s), num(0), get(ref("table", i32s), masked))]); + const version = match(fn([loop]), loop)!; + expect(version.fixed.map((c) => [version.receivers[c.receiver]!.ref, c.end])).toEqual([ + [ref("out", f64s), 0], + [ref("table", i32s), 255], + ]); + expect(version.affine).toEqual([]); + expect(version.counted).toBeNull(); +}); + +test("scaled counters and body temps resolve to their receiver and index", () => { + const temps = [local("view", f64s), local("at", F64)]; + const at = bin("+", bin("*", ref("i"), num(4)), num(3)); + const loop = counted([ + { kind: "varDecl", localId: "view", init: ref("src", f64s), loc }, + { kind: "varDecl", localId: "at", init: at, loc }, + set(ref("view", f64s), ref("at"), num(1)), + // (The integer counted loop needs one direct counter index.) + set(ref("out", f64s), ref("i"), num(0)), + ]); + const version = match(fn([loop], temps), loop)!; + expect(version.receivers.map((r) => r.ref)).toEqual([ref("src", f64s), ref("out", f64s)]); + expect(version.affine.map((c) => [c.scale, c.minOffset, c.maxEnd])).toEqual([ + [4, 3, 3], + [1, 0, 0], + ]); + expect(version.ranges.has(at)).toBe(true); +}); + +test("negative factors, rebinding, captures and outer loops are not versioned", () => { + // `i * -1` is -0 at zero, so it is not treated as an affine operand. + const negative = counted([set(ref("out", f64s), bin("*", ref("i"), num(-1)), num(1))]); + expect(match(fn([negative]), negative)).toBeNull(); + const rebound = counted([ + set(ref("out", f64s), ref("i"), num(1)), + { kind: "assign", localId: "out", value: ref("src", f64s), loc }, + ]); + expect(match(fn([rebound]), rebound)).toBeNull(); + const plain = counted([set(ref("out", f64s), ref("i"), num(1))]); + expect(match(fn([plain]), plain, ["out"])).toBeNull(); + const inner = counted([set(ref("out", f64s), ref("j"), num(1))], ref("n"), "j"); + const outer = counted([inner]); + expect(match(fn([outer]), outer)).toBeNull(); + expect(match(fn([outer]), inner)).not.toBeNull(); +}); + +test("emitted loops version the body and tag typed-array memory", () => { + const loop = counted([ + set(ref("out", f64s), ref("i"), bin("*", get(ref("src", f64s), ref("i")), num(2))), + ]); + const kernel = fn([loop]); + const mod: IrModule = { + irVersion: 16, + sourceFile: loc.file, + entry: "__main", + functions: [ + { name: "__main", params: [], returnType: VOID, locals: [], body: [], loc }, + kernel, + ], + }; + const llvm = emitLlvmModule(mod); + const body = /^define internal [^\n]*@sc_bf_kernel[^]*?^}/m.exec(llvm)![0]; + expect(body).toContain("bytes.version.fast"); + expect(body).toContain("bytes.version.slow"); + // The checked copy keeps its per-element tests; the guard reads lengths. + expect(body).toContain("bytes.index.invalid"); + expect(body).toMatch(/store double [^\n]*, align 1, !tbaa !\d+/); + expect(body).toMatch(/load double, ptr [^\n]*, align 1, !tbaa !\d+/); + expect(body).toMatch(/load i64, ptr [^\n]*, !tbaa !\d+/); + expect(body).toMatch(/load ptr, ptr [^\n]*, !tbaa !\d+/); + const elements = /store double [^\n]*!tbaa !(\d+)/.exec(body)![1]; + const header = /load ptr, ptr [^\n]*!tbaa !(\d+)/.exec(body)![1]; + expect(elements).not.toBe(header); +}); diff --git a/packages/compiler/src/backend/llvm/bytes-loop-versions.ts b/packages/compiler/src/backend/llvm/bytes-loop-versions.ts new file mode 100644 index 0000000000..d3a9b20429 --- /dev/null +++ b/packages/compiler/src/backend/llvm/bytes-loop-versions.ts @@ -0,0 +1,561 @@ +import type { IrExpr, IrLocal, IrStmt } from "../../ir/ir.js"; +import type { IntegerRange, IntegerRanges } from "../../ir/integer-ranges.js"; +import type { IntegerCountedForLoop } from "../../ir/integer-loops.js"; +import type { LlvmEmitterContext } from "./expr-context.js"; +import { byteNumberAccess } from "../../ir/byte-numbers.js"; +import { everyExprChild, everyStmtChild } from "../../ir/traverse.js"; +import { exactInteger, widenInteger } from "./integer-values.js"; +import { emitCountedLoopLimit, emitCountedLoopSafety } from "./counted-loops.js"; +import { emitBytesHeaderLength } from "./expr-bytes.js"; + +type Node = IrExpr | IrStmt; +type VarRef = IrExpr & { kind: "varRef" }; + +/** One loop-invariant integer operand of an affine index. */ +interface Term { + expr: IrExpr; + sign: 1 | -1; +} + +/** Every access of `receiver` at a static index range ends below `end + 1`. */ +interface FixedCheck { + receiver: number; + end: number; +} + +/** Every access of `receiver` at `scale * counter + Σ terms + offset` has an + * offset in [minOffset, maxEnd - width + 1]; `maxEnd` includes the width. */ +interface AffineCheck { + receiver: number; + scale: number; + terms: Term[]; + minOffset: number; + maxEnd: number; +} + +export interface BytesLoopVersion { + /** Typed-array accesses that are in bounds whenever the guard holds. */ + bounds: ReadonlySet; + /** Root receiver bindings, read once before the loop. */ + receivers: { ref: VarRef; global: boolean }[]; + fixed: FixedCheck[]; + affine: AffineCheck[]; + /** The counted loop whose iteration space the affine checks cover. */ + counted: IntegerCountedForLoop | null; + /** The guard also requires every counter value within [-bound, bound]. */ + counterBound: number | null; + /** Integer ranges of affine index operands that hold in the unchecked + * loop (where the guard bounds the counter). */ + ranges: ReadonlyMap; +} + +export interface BytesLoopFacts { + /** The function's own locals (not module globals). */ + locals: ReadonlyMap; + /** Module globals whose slot no call can replace: stable constants and + * the private bindings of a guarded initializer. */ + invariantGlobals: ReadonlySet; + captured: ReadonlySet; + ranges: IntegerRanges; + /** Locals definitely initialized when the loop is entered. */ + initialized: ReadonlySet; + /** Accesses already proved in bounds without a guard. */ + proven: ReadonlySet; +} + +const MAX_COST = 400; +const MAX_CHECKS = 8; +const MAX_OFFSET = 2 ** 31; +const SAFE = Number.MAX_SAFE_INTEGER; +const INVARIANT_OPS = new Set(["+", "-", "*", "&", "|", "^", "<<", ">>", ">>>"]); + +/** Loop versioning for typed-array bounds. A typed array's length and data + * pointer are fixed at construction (ScrBytes has no resize, transfer or + * detach), so an access is in bounds for the whole loop when its receiver + * binding and index operands cannot change while it runs. One test before + * the loop then covers every iteration: + * + * - a statically ranged integer index `[lo, hi]` (lo >= 0) needs + * `hi < length`, such as `table[x & 0xff]` or a constant-trip counter; + * - an index `k * counter + invariant + c` of a counted loop needs its first + * and last values in bounds, such as `out[base + i]` for `i < n` or + * `view.getUint32(i * 4)`. + * + * Only innermost synchronous loops are versioned, so the body is emitted + * twice at most. The checked loop runs whenever the guard fails, and keeps + * every out-of-bounds read, ignored write and NaN-index behavior. */ +export function matchBytesLoopVersion( + loop: IrStmt & { kind: "for" }, + counted: IntegerCountedForLoop | null, + facts: BytesLoopFacts, +): BytesLoopVersion | null { + const written = new Set(); + const temps = new Map(); + let cost = 0; + let innermost = true; + const scanExpr = (e: IrExpr): boolean => { + if (++cost > MAX_COST) return false; + if (e.kind === "assignExpr" || e.kind === "incDec") written.add(e.localId); + return everyExprChild(e, scanExpr, scanStmt); + }; + const scanStmt = (s: IrStmt): boolean => { + if (++cost > MAX_COST) return false; + switch (s.kind) { + case "for": + case "forOf": + case "while": + case "doWhile": + innermost = false; + return false; + case "varDecl": { + if (temps.has(s.localId) || written.has(s.localId)) temps.delete(s.localId); + else if (s.init) temps.set(s.localId, s.init); + written.add(s.localId); + break; + } + case "assign": + written.add(s.localId); + break; + case "tryCatch": + if (s.catchLocalId) written.add(s.catchLocalId); + break; + default: + break; + } + return everyStmtChild(s, scanExpr, scanStmt); + }; + if ( + !loop.body.every(scanStmt) || + !innermost || + (loop.cond !== null && !scanExpr(loop.cond)) || + (loop.update !== null && !scanStmt(loop.update)) + ) + return null; + // A body temp stands for its initializer only when it is a constant + // binding assigned exactly once, by that declaration. + const declared = new Map(); + const countDecls = (s: IrStmt): boolean => { + if (s.kind === "varDecl") declared.set(s.localId, (declared.get(s.localId) ?? 0) + 1); + return everyStmtChild(s, countExprDecls, countDecls); + }; + const countExprDecls = (e: IrExpr): boolean => everyExprChild(e, countExprDecls, countDecls); + loop.body.every(countDecls); + const temp = (id: string): IrExpr | null => { + const local = facts.locals.get(id); + const init = temps.get(id); + if ( + !init || + !local || + local.mutable || + local.boxed || + local.tdz || + facts.captured.has(id) || + declared.get(id) !== 1 + ) + return null; + return init; + }; + const counter = counted?.localId ?? null; + // A binding keeps its loop-entry value for the whole loop. + const invariant = (id: string): boolean => { + if (written.has(id) || id === counter) return false; + const local = facts.locals.get(id); + if (local) + return !local.boxed && !local.tdz && !facts.captured.has(id) && facts.initialized.has(id); + return facts.invariantGlobals.has(id); + }; + const resolving = new Set(); + const root = (e: IrExpr): VarRef | null => { + if (e.kind !== "varRef" || e.type.kind !== "bytes") return null; + const init = temp(e.localId); + if (init) { + if (resolving.has(e.localId)) return null; + resolving.add(e.localId); + const result = root(init); + resolving.delete(e.localId); + return result; + } + return invariant(e.localId) ? e : null; + }; + // A pure integer operand evaluated again before the loop. + const invariantKey = (e: IrExpr): string | null => { + if (!facts.ranges.get(e)) return null; + if (e.kind === "numLit") return Number.isSafeInteger(e.value) ? `n${e.value}` : null; + if (e.kind === "varRef") + return invariant(e.localId) ? `v${e.localId.length}:${e.localId}` : null; + if ( + e.kind === "bin" && + INVARIANT_OPS.has(e.op) && + e.left.type.kind === "f64" && + e.right.type.kind === "f64" + ) { + const left = invariantKey(e.left), + right = invariantKey(e.right); + return left !== null && right !== null ? `(${e.op} ${left} ${right})` : null; + } + return null; + }; + interface Affine { + /** The counter's coefficient; 0 when the value is loop-invariant. */ + counter: number; + terms: (Term & { key: string })[]; + offset: number; + } + // Every operand of an affine index is `counter * k + Σ terms + offset`. + // Invariant terms need proven ranges. Operands that involve the counter + // need none: the guard bounds the counter so that each of them is a safe + // integer, which makes the index's double arithmetic exact. + const shapes = new Map(); + const affine = (e: IrExpr): Affine | null => { + if (shapes.has(e)) return shapes.get(e)!; + const result = affineShape(e); + shapes.set(e, result); + return result; + }; + const affineShape = (e: IrExpr): Affine | null => { + if (e.kind === "numLit") + return Number.isSafeInteger(e.value) && Math.abs(e.value) <= MAX_OFFSET + ? { counter: 0, terms: [], offset: e.value } + : null; + if (e.kind === "varRef") { + if (e.localId === counter) return { counter: 1, terms: [], offset: 0 }; + const init = temp(e.localId); + if (init) { + if (resolving.has(e.localId)) return null; + resolving.add(e.localId); + const result = affine(init); + resolving.delete(e.localId); + return result; + } + } + if ( + e.kind === "bin" && + (e.op === "+" || e.op === "-") && + e.left.type.kind === "f64" && + e.right.type.kind === "f64" + ) { + const left = affine(e.left), + right = affine(e.right); + if (left && right && (left.counter === 0 || right.counter === 0)) { + const sign = e.op === "+" ? 1 : -1; + const offset = left.offset + sign * right.offset; + if (Math.abs(offset) > MAX_OFFSET) return null; + const terms = [ + ...left.terms, + ...right.terms.map((t) => ({ ...t, sign: (sign * t.sign) as 1 | -1 })), + ]; + const result = { counter: left.counter + sign * right.counter, terms, offset }; + if (result.counter !== 0 || facts.ranges.get(e)) return result; + } + } + // A positive constant multiple of the counter: `i * 4`, `4 * i`, + // `(i + 1) * 8`. (A negative factor turns a zero into -0.) + if (e.kind === "bin" && e.op === "*") { + const literal = + e.right.kind === "numLit" ? e.right : e.left.kind === "numLit" ? e.left : null; + const scaled = literal === e.right ? e.left : e.right; + const factor = literal?.value ?? 0; + if ( + literal && + scaled.type.kind === "f64" && + Number.isSafeInteger(factor) && + factor > 0 && + factor <= MAX_OFFSET + ) { + const inner = affine(scaled); + if (inner && inner.counter !== 0 && inner.terms.length === 0) { + const counterScale = inner.counter * factor, + offset = inner.offset * factor; + if (Math.abs(counterScale) <= MAX_OFFSET && Math.abs(offset) <= MAX_OFFSET) + return { counter: counterScale, terms: [], offset }; + } + } + } + const key = invariantKey(e); + return key !== null ? { counter: 0, terms: [{ expr: e, sign: 1, key }], offset: 0 } : null; + }; + // The operands of an index that involve the counter. + const chain = (e: IrExpr, out: Set): void => { + if (!shapes.get(e)?.counter) return; + out.add(e); + if (e.kind === "varRef") { + const init = temp(e.localId); + if (init) chain(init, out); + } else if (e.kind === "bin") { + chain(e.left, out); + chain(e.right, out); + } + }; + const termMagnitude = (terms: Term[]): number => + terms.reduce((sum, t) => { + const r = facts.ranges.get(t.expr)!; + return sum + Math.max(Math.abs(r.min), Math.abs(r.max)); + }, 0); + + const receivers: { ref: VarRef; global: boolean }[] = []; + const receiverIndex = new Map(); + const fixed = new Map(); + const affineChecks = new Map< + string, + { check: AffineCheck; nodes: Node[]; operands: Set } + >(); + const receiverOf = (ref: VarRef): number => { + let index = receiverIndex.get(ref.localId); + if (index === undefined) { + index = receivers.length; + receiverIndex.set(ref.localId, index); + receivers.push({ ref, global: !facts.locals.has(ref.localId) }); + } + return index; + }; + const access = (node: Node, array: IrExpr, index: IrExpr, width: number): void => { + if (facts.proven.has(node)) return; + const ref = root(array); + if (!ref) return; + const shape = counter !== null ? affine(index) : null; + if (shape && shape.counter !== 0) { + const operands = new Set(); + chain(index, operands); + const terms = [...shape.terms].sort((a, b) => + a.key < b.key ? -1 : a.key > b.key ? 1 : a.sign - b.sign, + ); + const receiver = receiverOf(ref); + const key = `${receiver}|${shape.counter}|${terms.map((t) => `${t.sign}${t.key}`).join(",")}`; + const end = shape.offset + width - 1; + const group = affineChecks.get(key); + if (group) { + group.check.minOffset = Math.min(group.check.minOffset, shape.offset); + group.check.maxEnd = Math.max(group.check.maxEnd, end); + group.nodes.push(node); + for (const operand of operands) group.operands.add(operand); + } else + affineChecks.set(key, { + check: { receiver, scale: shape.counter, terms, minOffset: shape.offset, maxEnd: end }, + nodes: [node], + operands, + }); + return; + } + const range = facts.ranges.get(index); + if (!range) return; + const end = range.max + width - 1; + if (range.min < 0 || !Number.isSafeInteger(end)) return; + const receiver = receiverOf(ref); + const group = fixed.get(receiver); + if (group) { + group.check.end = Math.max(group.check.end, end); + group.nodes.push(node); + } else fixed.set(receiver, { check: { receiver, end }, nodes: [node] }); + }; + const visitExpr = (e: IrExpr): boolean => { + if (e.kind === "bytesIntrinsic") { + const numeric = byteNumberAccess(e); + if (numeric) access(e, e.receiver, e.args[numeric.offsetArg]!, numeric.width); + else if (e.method === "get" && e.args[0]) access(e, e.receiver, e.args[0], 1); + } + return everyExprChild(e, visitExpr, visitStmt); + }; + const visitStmt = (s: IrStmt): boolean => { + if (s.kind === "bytesSet") access(s, s.arr, s.index, 1); + return everyStmtChild(s, visitExpr, visitStmt); + }; + loop.body.every(visitStmt); + // Bound the counter so every counter operand stays a safe integer. + let bound = SAFE; + const affineGroups = [...affineChecks.values()].filter((group) => { + let limit = SAFE; + for (const operand of group.operands) { + const shape = shapes.get(operand)!; + const room = SAFE - termMagnitude(shape.terms) - Math.abs(shape.offset); + limit = Math.min(limit, Math.floor(room / Math.abs(shape.counter))); + } + if (limit < 0 || !counted) return false; + bound = Math.min(bound, limit); + return true; + }); + const groups = [...fixed.values(), ...affineGroups].slice(0, MAX_CHECKS); + if (groups.length === 0) return null; + const bounds = new Set(); + const used = new Set(); + const ranges = new Map(); + const counterLow = Math.max(counted?.range.min ?? 0, -bound), + counterHigh = Math.min(counted?.range.max ?? 0, bound); + for (const group of groups) { + used.add(group.check.receiver); + for (const node of group.nodes) bounds.add(node); + if (!("operands" in group)) continue; + for (const operand of group.operands) { + if (facts.ranges.get(operand)) continue; + const shape = shapes.get(operand)!; + const a = shape.counter * counterLow, + b = shape.counter * counterHigh; + let min = Math.min(a, b) + shape.offset, + max = Math.max(a, b) + shape.offset; + for (const term of shape.terms) { + const r = facts.ranges.get(term.expr)!; + min += term.sign > 0 ? r.min : -r.max; + max += term.sign > 0 ? r.max : -r.min; + } + if (counterLow <= counterHigh) ranges.set(operand, { min, max }); + } + } + const kept = groups.map((g) => g.check); + const affineKept = kept.filter((c): c is AffineCheck => "terms" in c); + // Renumber the receivers the kept checks use. + const order = [...used].sort((a, b) => a - b); + const renumber = new Map(order.map((old, index) => [old, index])); + const needsBound = + affineKept.length > 0 && + counted !== null && + (counted.range.min < -bound || counted.range.max > bound); + return { + bounds, + receivers: order.map((index) => receivers[index]!), + fixed: kept + .filter((c): c is FixedCheck => !("terms" in c)) + .map((c) => ({ ...c, receiver: renumber.get(c.receiver)! })), + affine: affineKept.map((c) => ({ ...c, receiver: renumber.get(c.receiver)! })), + counted: affineKept.length > 0 ? counted : null, + counterBound: needsBound ? bound : null, + ranges, + }; +} + +/** Branches to `fast` when every access the version proves is in bounds for + * every iteration, and to `slow` otherwise (including a counted loop whose + * integer form is not established, or a module receiver not yet set). */ +export function emitBytesLoopVersionGuard( + host: LlvmEmitterContext, + version: BytesLoopVersion, + fast: string, + slow: string, +): void { + const B = host.B; + const next = (cond: string): void => { + if (cond === "true") return; + const ok = B.newLabel("bytes.version.check"); + B.condBr(cond, ok, slow); + B.startBlock(ok); + }; + const counted = version.counted; + if (counted?.guarded) next(emitCountedLoopSafety(host, counted)); + const receivers = version.receivers.map(({ ref, global }) => { + const value = host.emitReadReceiver(ref).name; + if (global) { + const set = B.tmp(); + B.line(`${set} = icmp ne ptr ${value}, null`); + next(set); + } + return value; + }); + const lengths = receivers.map((receiver) => { + const len = emitBytesHeaderLength(host, receiver); + if (host.sizeType === "i64") return len; + const wide = B.tmp(); + B.line(`${wide} = zext ${host.sizeType} ${len} to i64`); + return wide; + }); + const conditions: string[] = []; + for (const check of version.fixed) { + const ok = B.tmp(); + B.line(`${ok} = icmp ugt i64 ${lengths[check.receiver]}, ${check.end}`); + conditions.push(ok); + } + if (counted && version.affine.length > 0) { + // The first and last counter values, in ascending order. The last value + // of a non-unit step is exact: `start + ⌊(limit ∓ 1 - start) / step⌋ * + // step`. A loop that runs no iteration may compute anything here. + const startValue = host.emitExpr(counted.start); + const startInteger = exactInteger(host, startValue, counted.start); + let start: string; + if (startInteger) start = widenInteger(host, startInteger); + else { + // The counted-loop safety test above proved an exact integer. + start = B.tmp(); + B.line(`${start} = fptosi double ${startValue.name} to i64`); + } + const limit = emitCountedLoopLimit(host, counted); + const step = Math.abs(counted.step); + const edge = B.tmp(); + B.line(`${edge} = add i64 ${limit}, ${counted.step > 0 ? -1 : 1}`); + let last = edge; + if (step !== 1) { + const span = B.tmp(), + whole = B.tmp(), + covered = B.tmp(); + last = B.tmp(); + B.line( + counted.step > 0 + ? `${span} = sub i64 ${edge}, ${start}` + : `${span} = sub i64 ${start}, ${edge}`, + ); + B.line(`${whole} = sdiv i64 ${span}, ${step}`); + B.line(`${covered} = mul i64 ${whole}, ${step}`); + B.line(`${last} = ${counted.step > 0 ? "add" : "sub"} i64 ${start}, ${covered}`); + } + const low = counted.step > 0 ? start : last; + const high = counted.step > 0 ? last : start; + if (version.counterBound !== null) { + const above = B.tmp(), + below = B.tmp(); + B.line(`${above} = icmp sge i64 ${low}, ${-version.counterBound}`); + B.line(`${below} = icmp sle i64 ${high}, ${version.counterBound}`); + conditions.push(above, below); + } + for (const check of version.affine) { + let base = "0"; + for (const term of check.terms) { + const value = host.emitExpr(term.expr); + const integer = exactInteger(host, value, term.expr); + if (!integer) { + // Unreachable: every term has a proven range. Stay checked. + B.br(slow); + B.startBlock(B.newLabel("bytes.version.dead")); + return; + } + const wide = widenInteger(host, integer); + const sum = B.tmp(); + B.line(`${sum} = ${term.sign > 0 ? "add" : "sub"} i64 ${base}, ${wide}`); + base = sum; + } + // Products wrap only for a loop with no iteration: within the + // iteration space every scaled counter is a proven safe integer. + const scaledLow = scaleCounter(host, low, check.scale), + scaledHigh = scaleCounter(host, high, check.scale); + const first = B.tmp(), + final = B.tmp(), + nonNegative = B.tmp(), + below = B.tmp(), + both = B.tmp(); + B.line(`${first} = add i64 ${check.scale > 0 ? scaledLow : scaledHigh}, ${base}`); + B.line(`${final} = add i64 ${check.scale > 0 ? scaledHigh : scaledLow}, ${base}`); + const firstOffset = B.tmp(), + lastEnd = B.tmp(); + B.line(`${firstOffset} = add i64 ${first}, ${check.minOffset}`); + B.line(`${lastEnd} = add i64 ${final}, ${check.maxEnd}`); + B.line(`${nonNegative} = icmp sge i64 ${firstOffset}, 0`); + // Lengths are at most 2^53 elements: a signed comparison is exact. + B.line(`${below} = icmp slt i64 ${lastEnd}, ${lengths[check.receiver]}`); + B.line(`${both} = and i1 ${nonNegative}, ${below}`); + conditions.push(both); + } + } + let valid = "true"; + for (const cond of conditions) { + if (valid === "true") valid = cond; + else { + const all = B.tmp(); + B.line(`${all} = and i1 ${valid}, ${cond}`); + valid = all; + } + } + if (valid === "true") B.br(fast); + else B.condBr(valid, fast, slow); +} + +function scaleCounter(host: LlvmEmitterContext, value: string, scale: number): string { + if (scale === 1) return value; + const scaled = host.B.tmp(); + host.B.line(`${scaled} = mul i64 ${value}, ${scale}`); + return scaled; +} diff --git a/packages/compiler/src/backend/llvm/counted-loops.ts b/packages/compiler/src/backend/llvm/counted-loops.ts index 03e0ed4f3a..36fc63e1ff 100644 --- a/packages/compiler/src/backend/llvm/counted-loops.ts +++ b/packages/compiler/src/backend/llvm/counted-loops.ts @@ -42,3 +42,54 @@ export function emitCountedLoopLimit( B.line(`${limit} = add nsw i64 ${integer}, ${ascending ? 1 : -1}`); return limit; } + +/** The run-time test of a guarded counted loop: an unproven limit is within + * the integer form's finite range and an unproven start is an exact safe + * integer. Returns `true` when nothing needs testing. */ +export function emitCountedLoopSafety( + host: LlvmEmitterContext, + loop: IntegerCountedForLoop, +): string { + const B = host.B; + let safe = "true"; + if (loop.limitGuarded) { + const bound = host.emitExpr(loop.limit); + safe = B.tmp(); + B.line( + `${safe} = fcmp ${loop.step > 0 ? "ole" : "oge"} double ${bound.name}, ${f64Lit(loop.guardLimit)}`, + ); + } + if (loop.startGuarded) { + // An exact integer of at most 2^53 - 1 in magnitude, and not -0 + // (the bit patterns of the value and its round trip agree). + // NaN and infinities fail the ordered range tests. + const start = host.emitExpr(loop.start); + const low = B.tmp(), + high = B.tmp(), + raw = B.tmp(), + wide = B.tmp(), + back = B.tmp(), + bits = B.tmp(), + backBits = B.tmp(), + exact = B.tmp(), + inRange = B.tmp(), + ok = B.tmp(); + B.line(`${low} = fcmp oge double ${start.name}, ${f64Lit(-Number.MAX_SAFE_INTEGER)}`); + B.line(`${high} = fcmp ole double ${start.name}, ${f64Lit(Number.MAX_SAFE_INTEGER)}`); + B.line(`${inRange} = and i1 ${low}, ${high}`); + B.line(`${raw} = fptosi double ${start.name} to i64`); + B.line(`${wide} = freeze i64 ${raw}`); + B.line(`${back} = sitofp i64 ${wide} to double`); + B.line(`${bits} = bitcast double ${start.name} to i64`); + B.line(`${backBits} = bitcast double ${back} to i64`); + B.line(`${exact} = icmp eq i64 ${bits}, ${backBits}`); + B.line(`${ok} = and i1 ${inRange}, ${exact}`); + if (safe === "true") safe = ok; + else { + const both = B.tmp(); + B.line(`${both} = and i1 ${safe}, ${ok}`); + safe = both; + } + } + return safe; +} diff --git a/packages/compiler/src/backend/llvm/emitter.ts b/packages/compiler/src/backend/llvm/emitter.ts index 2cccda19ce..8c83638a53 100644 --- a/packages/compiler/src/backend/llvm/emitter.ts +++ b/packages/compiler/src/backend/llvm/emitter.ts @@ -54,8 +54,10 @@ import { matchIntegerArrayForLoop, matchIntegerBytesForLoop, matchIntegerCountedForLoop, + type IntegerCountedForLoop, } from "../../ir/integer-loops.js"; -import { emitCountedLoopLimit } from "./counted-loops.js"; +import { emitCountedLoopLimit, emitCountedLoopSafety } from "./counted-loops.js"; +import { emitBytesLoopVersionGuard, matchBytesLoopVersion } from "./bytes-loop-versions.js"; import { emitSafeIntegerCheck, emitSpeculativeInteger, @@ -66,6 +68,7 @@ import { } from "./speculative-integers.js"; import { emitLiteralSwitch } from "./switch-dispatch.js"; import { findBytesBounds } from "./bytes-bounds.js"; +import { globalSlotAlias } from "./bytes-alias.js"; import { emitByteWindowGuard, findInitializedByteLoopBindings, @@ -1065,6 +1068,14 @@ export class LlEmitter { wrappingSums = new Set(); private byteWindowsEnabled = true; private byteWindowEntries: ReturnType = new Map(); + /** Typed-array loop versioning (bytes-loop-versions.ts) for the current + * function: off in debug and suspending functions and inside a copy. */ + private bytesLoopVersionsEnabled = false; + private bytesLoopEntries: ReturnType | null = null; + private bytesLoopFunction: IrFunction | null = null; + /** Counted loops whose integer-form safety test an enclosing version + * guard already passed. */ + private readonly provenCountedGuards = new Set(); bytesBounds: ReadonlySet = new Set(); integerViews = new Map(); /** Module globals the current function owns outright (initializer @@ -6161,6 +6172,34 @@ export class LlEmitter { }; } + private matchBytesLoopVersion( + s: IrStmt & { kind: "for" }, + counted: IntegerCountedForLoop | null, + ): ReturnType { + const fn = this.bytesLoopFunction; + if (!fn) return null; + this.bytesLoopEntries ??= findInitializedByteLoopBindings(fn); + const initialized = this.bytesLoopEntries.get(s); + if (!initialized) return null; + const invariantGlobals = new Set(this.stableGlobals); + for (const global of this.initializerBindings.get(fn.name) ?? []) + invariantGlobals.add(global.id); + return matchBytesLoopVersion(s, counted, { + locals: this.currentLocals, + invariantGlobals, + captured: this.captureIds, + ranges: this.integerRanges, + initialized, + proven: this.bytesBounds, + }); + } + + /** The alias tag of a whole-slot load of a binding (module globals only). */ + slotAlias(id: string): string { + const binding = this.binding(id); + return binding.kind === "global" ? globalSlotAlias(this, id, binding.type.kind) : ""; + } + private emitDebugLocal(local: IrLocal, slot: string, arg = 0, captured = false): void { const debug = this.debug?.local(local, this.debugScope, arg, captured); if (!debug) return; @@ -6712,6 +6751,9 @@ export class LlEmitter { this.finalFieldBorrows = !fn.async && !fn.generator && constructorClass(fn.name) === null && this.finalFields.size > 0; this.byteWindowsEnabled = this.countedLoopsEnabled; + this.bytesLoopVersionsEnabled = this.countedLoopsEnabled; + this.bytesLoopEntries = null; + this.bytesLoopFunction = fn; this.byteWindowEntries = this.byteWindowsEnabled && numericFn.locals.some((l) => l.type.kind === "bytes") ? findInitializedByteLoopBindings(fn) @@ -8670,48 +8712,40 @@ export class LlEmitter { B.startBlock(done); break; } + const version = this.bytesLoopVersionsEnabled + ? this.matchBytesLoopVersion(s, countedLoop) + : null; + if (version) { + const fast = B.newLabel("bytes.version.fast"), + slow = B.newLabel("bytes.version.slow"), + done = B.newLabel("bytes.version.done"); + emitBytesLoopVersionGuard(this, version, fast, slow); + const originalBounds = this.bytesBounds, + originalRanges = this.integerRanges; + this.bytesLoopVersionsEnabled = false; + try { + B.startBlock(slow); + this.emitStmt(s); + B.br(done); + B.startBlock(fast); + this.bytesBounds = new Set([...originalBounds, ...version.bounds]); + if (version.ranges.size > 0) + this.integerRanges = new Map([...originalRanges, ...version.ranges]); + if (version.counted?.guarded) this.provenCountedGuards.add(s); + this.emitStmt(s); + B.br(done); + } finally { + this.bytesBounds = originalBounds; + this.integerRanges = originalRanges; + this.provenCountedGuards.delete(s); + this.bytesLoopVersionsEnabled = true; + } + B.startBlock(done); + break; + } let countedJoin: string | null = null; - if (countedLoop?.guarded) { - let safe = "true"; - if (countedLoop.limitGuarded) { - const bound = this.emitExpr(countedLoop.limit); - safe = B.tmp(); - B.line( - `${safe} = fcmp ${countedLoop.step > 0 ? "ole" : "oge"} double ${bound.name}, ${f64Lit(countedLoop.guardLimit)}`, - ); - } - if (countedLoop.startGuarded) { - // An exact integer of at most 2^53 - 1 in magnitude, and not -0 - // (the bit patterns of the value and its round trip agree). - // NaN and infinities fail the ordered range tests. - const start = this.emitExpr(countedLoop.start); - const low = B.tmp(), - high = B.tmp(), - raw = B.tmp(), - wide = B.tmp(), - back = B.tmp(), - bits = B.tmp(), - backBits = B.tmp(), - exact = B.tmp(), - inRange = B.tmp(), - ok = B.tmp(); - B.line(`${low} = fcmp oge double ${start.name}, ${f64Lit(-Number.MAX_SAFE_INTEGER)}`); - B.line(`${high} = fcmp ole double ${start.name}, ${f64Lit(Number.MAX_SAFE_INTEGER)}`); - B.line(`${inRange} = and i1 ${low}, ${high}`); - B.line(`${raw} = fptosi double ${start.name} to i64`); - B.line(`${wide} = freeze i64 ${raw}`); - B.line(`${back} = sitofp i64 ${wide} to double`); - B.line(`${bits} = bitcast double ${start.name} to i64`); - B.line(`${backBits} = bitcast double ${back} to i64`); - B.line(`${exact} = icmp eq i64 ${bits}, ${backBits}`); - B.line(`${ok} = and i1 ${inRange}, ${exact}`); - if (safe === "true") safe = ok; - else { - const both = B.tmp(); - B.line(`${both} = and i1 ${safe}, ${ok}`); - safe = both; - } - } + if (countedLoop?.guarded && !this.provenCountedGuards.has(s)) { + const safe = emitCountedLoopSafety(this, countedLoop); const fast = B.newLabel("counted.fast"); const slow = B.newLabel("counted.slow"); countedJoin = B.newLabel("counted.done"); @@ -8818,7 +8852,7 @@ export class LlEmitter { B.line( `${lenPtr} = getelementptr inbounds %${integerArrayLoop ? "ScrArr" : "ScrBytes"}, ptr ${receiver.name}, i64 0, i32 1`, ); - if (integerArrayLoop) this.markMemoryPointer(lenPtr, "array:header"); + this.markMemoryPointer(lenPtr, integerArrayLoop ? "array:header" : "bytes:header"); B.line( `${len} = load ${this.sizeType}, ptr ${lenPtr}${this.fieldAliasAttachment(lenPtr)}`, ); @@ -10309,7 +10343,7 @@ export class LlEmitter { } if (binding.kind === "global") this.checkGlobalTdz(e.localId); const value = this.B.tmp(); - this.B.line(`${value} = load ptr, ptr ${binding.slot}`); + this.B.line(`${value} = load ptr, ptr ${binding.slot}${this.slotAlias(e.localId)}`); return { name: value, type: e.type }; } if (e.kind === "dynCheck" && e.exact) return this.emitExactCapsuleObject(e.value, e.type); diff --git a/packages/compiler/src/backend/llvm/expr-bytes.ts b/packages/compiler/src/backend/llvm/expr-bytes.ts index ac025df099..485b7a6b33 100644 --- a/packages/compiler/src/backend/llvm/expr-bytes.ts +++ b/packages/compiler/src/backend/llvm/expr-bytes.ts @@ -8,6 +8,7 @@ import { exactInteger, widenInteger } from "./integer-values.js"; import { byteNumberAccess } from "../../ir/byte-numbers.js"; import { emitByteNumber } from "./byte-numbers.js"; import { integerOrNaNRange } from "./wrapping-sums.js"; +import { bytesElementAlias, bytesHeaderAlias } from "./bytes-alias.js"; const BYTES_NUM_KIND: Record = { u8: { kind: 0, le: false }, @@ -93,10 +94,7 @@ export function emitBytesIndex( B.line(`${narrow} = trunc i64 ${wide} to ${host.sizeType}`); return narrow; } - const lenPtr = B.tmp(); - const len = B.tmp(); - B.line(`${lenPtr} = getelementptr inbounds %ScrBytes, ptr ${receiver}, i64 0, i32 1`); - B.line(`${len} = load ${host.sizeType}, ptr ${lenPtr}`); + const len = emitBytesHeaderLength(host, receiver); if (integer) { const wideIndex = widenInteger(host, integer); const wideLength = host.sizeType === "i64" ? len : B.tmp(); @@ -145,11 +143,20 @@ export function emitBytesIndex( return idx; } +/** The element count of a typed array, in the target's size type. */ +export function emitBytesHeaderLength(host: LlvmEmitterContext, receiver: string): string { + const p = host.B.tmp(); + const len = host.B.tmp(); + host.B.line(`${p} = getelementptr inbounds %ScrBytes, ptr ${receiver}, i64 0, i32 1`); + host.B.line(`${len} = load ${host.sizeType}, ptr ${p}${bytesHeaderAlias(host)}`); + return len; +} + export function emitBytesData(host: LlvmEmitterContext, receiver: string): string { const p = host.B.tmp(); const data = host.B.tmp(); host.B.line(`${p} = getelementptr inbounds %ScrBytes, ptr ${receiver}, i64 0, i32 3`); - host.B.line(`${data} = load ptr, ptr ${p}`); + host.B.line(`${data} = load ptr, ptr ${p}${bytesHeaderAlias(host)}`); return data; } @@ -160,10 +167,7 @@ export function emitBytesLength( bytes: boolean, ): LlValue { const B = host.B; - const p = B.tmp(); - const len = B.tmp(); - B.line(`${p} = getelementptr inbounds %ScrBytes, ptr ${receiver}, i64 0, i32 1`); - B.line(`${len} = load ${host.sizeType}, ptr ${p}`); + const len = emitBytesHeaderLength(host, receiver); const size = BYTES_ELEMENT_SIZE[elem]; const count = bytes && size !== 1 ? B.tmp() : len; if (count !== len) B.line(`${count} = shl ${host.sizeType} ${len}, ${Math.log2(size)}`); @@ -187,7 +191,7 @@ function emitSharedBytesSplit(host: LlvmEmitterContext, receiver: string): strin isShared = B.tmp(), expected = B.tmp(); B.line(`${slot} = getelementptr inbounds %ScrBytes, ptr ${receiver}, i64 0, i32 9`); - B.line(`${storage} = load ptr, ptr ${slot}`); + B.line(`${storage} = load ptr, ptr ${slot}${bytesHeaderAlias(host)}`); B.line(`${isShared} = icmp ne ptr ${storage}, null`); B.line(`${expected} = call i1 @llvm.expect.i1(i1 ${isShared}, i1 false)`); const shared = B.newLabel("bytes.shared"), @@ -257,9 +261,15 @@ function emitBytesLoad( inBounds: boolean, invalid: string | undefined, ): LlValue { + if (host.mod.workers) { + const idx = emitBytesIndex(host, receiver, index, expr, invalid, inBounds); + return emitSharedOrLocalLoad(host, elem, receiver, idx); + } + // The header is immutable, so the data pointer loads beside the length, + // ahead of the bounds branch: an unconditional load LICM can hoist. + const data = host.emitBytesData(receiver); const idx = emitBytesIndex(host, receiver, index, expr, invalid, inBounds); - if (host.mod.workers) return emitSharedOrLocalLoad(host, elem, receiver, idx); - return emitBytesLoadAt(host, elem, receiver, idx); + return emitBytesLoadAt(host, elem, data, idx); } /** Worker programs: the element at a checked index of a view that may be @@ -285,7 +295,7 @@ function emitSharedOrLocalLoad( const sharedDone = B.newLabel("bytes.shared.done"); B.br(sharedDone); B.startBlock(local); - const own = emitBytesLoadAt(host, elem, receiver, idx); + const own = emitBytesLoadAt(host, elem, host.emitBytesData(receiver), idx); const localDone = B.newLabel("bytes.local.done"); B.br(localDone); B.startBlock(localDone); @@ -306,15 +316,15 @@ function emitSharedOrLocalLoad( return { name: out, type: F64, uint32: u32 }; } -/** The element at a checked index of a thread-owned view. */ +/** The element at a checked index of a thread-owned view, given its data + * pointer. */ function emitBytesLoadAt( host: LlvmEmitterContext, elem: IrBytesElem, - receiver: string, + data: string, idx: string, ): LlValue { const B = host.B; - const data = host.emitBytesData(receiver); const p = B.tmp(); if (elem === "f16") { // Half-precision bits widen exactly; the runtime converts them. @@ -322,7 +332,7 @@ function emitBytesLoadAt( const raw = B.tmp(); const out = B.tmp(); B.line(`${p} = getelementptr inbounds i16, ptr ${data}, ${host.sizeType} ${idx}`); - B.line(`${raw} = load i16, ptr ${p}, align 1`); + B.line(`${raw} = load i16, ptr ${p}, align 1${bytesElementAlias(host)}`); B.line(`${out} = call double @scr_f16_to_f64(i16 zeroext ${raw})`); return { name: out, type: F64 }; } @@ -333,7 +343,7 @@ function emitBytesLoadAt( const wide = B.tmp(); const out = B.tmp(); B.line(`${p} = getelementptr inbounds i${bits}, ptr ${data}, ${host.sizeType} ${idx}`); - B.line(`${raw} = load i${bits}, ptr ${p}, align 1`); + B.line(`${raw} = load i${bits}, ptr ${p}, align 1${bytesElementAlias(host)}`); B.line(`${wide} = ${signed ? "sext" : "zext"} i${bits} ${raw} to i32`); B.line(`${out} = ${signed ? "sitofp" : "uitofp"} i32 ${wide} to double`); return { name: out, type: F64, uint32: wide }; @@ -342,20 +352,20 @@ function emitBytesLoadAt( const raw = B.tmp(); const out = B.tmp(); B.line(`${p} = getelementptr inbounds float, ptr ${data}, ${host.sizeType} ${idx}`); - B.line(`${raw} = load float, ptr ${p}, align 1`); + B.line(`${raw} = load float, ptr ${p}, align 1${bytesElementAlias(host)}`); B.line(`${out} = fpext float ${raw} to double`); return { name: out, type: F64 }; } if (elem === "f64") { const out = B.tmp(); B.line(`${p} = getelementptr inbounds double, ptr ${data}, ${host.sizeType} ${idx}`); - B.line(`${out} = load double, ptr ${p}, align 1`); + B.line(`${out} = load double, ptr ${p}, align 1${bytesElementAlias(host)}`); return { name: out, type: F64 }; } const raw = B.tmp(); const out = B.tmp(); B.line(`${p} = getelementptr inbounds i32, ptr ${data}, ${host.sizeType} ${idx}`); - B.line(`${raw} = load i32, ptr ${p}, align 1`); + B.line(`${raw} = load i32, ptr ${p}, align 1${bytesElementAlias(host)}`); B.line(`${out} = ${elem === "i32" ? "sitofp" : "uitofp"} i32 ${raw} to double`); return { name: out, type: F64, uint32: raw }; } @@ -465,6 +475,7 @@ export function emitBytesSet( const B = host.B; // Invalid integer-indexed writes are ignored after evaluating the RHS. const done = B.newLabel("bytes.store.done"); + const data = host.emitBytesData(receiver); const idx = emitBytesIndex(host, receiver, index, indexExpr, done, inBounds); if (host.mod.workers) { host.declare(`declare void @scr_bytes_shared_set(ptr, ${host.sizeType}, double)`); @@ -487,7 +498,6 @@ export function emitBytesSet( elem === "f16" || elem === "f32" || elem === "f64" ? null : host.emitToUint32(value.name, expr, value.uint32); - const data = host.emitBytesData(receiver); const p = B.tmp(); if (elem === "f16") { // One round-to-nearest-even step from the double, in the runtime. @@ -495,24 +505,24 @@ export function emitBytesSet( const narrowed = B.tmp(); B.line(`${narrowed} = call zeroext i16 @scr_f16_from_f64(double ${value.name})`); B.line(`${p} = getelementptr inbounds i16, ptr ${data}, ${host.sizeType} ${idx}`); - B.line(`store i16 ${narrowed}, ptr ${p}, align 1`); + B.line(`store i16 ${narrowed}, ptr ${p}, align 1${bytesElementAlias(host)}`); } else if (BYTES_ELEMENT_SIZE[elem] < 4) { const bits = BYTES_ELEMENT_SIZE[elem] * 8; const byte = B.tmp(); B.line(`${byte} = trunc i32 ${stored!} to i${bits}`); B.line(`${p} = getelementptr inbounds i${bits}, ptr ${data}, ${host.sizeType} ${idx}`); - B.line(`store i${bits} ${byte}, ptr ${p}, align 1`); + B.line(`store i${bits} ${byte}, ptr ${p}, align 1${bytesElementAlias(host)}`); } else if (elem === "f32") { const narrowed = B.tmp(); B.line(`${narrowed} = fptrunc double ${value.name} to float`); B.line(`${p} = getelementptr inbounds float, ptr ${data}, ${host.sizeType} ${idx}`); - B.line(`store float ${narrowed}, ptr ${p}, align 1`); + B.line(`store float ${narrowed}, ptr ${p}, align 1${bytesElementAlias(host)}`); } else if (elem === "f64") { B.line(`${p} = getelementptr inbounds double, ptr ${data}, ${host.sizeType} ${idx}`); - B.line(`store double ${value.name}, ptr ${p}, align 1`); + B.line(`store double ${value.name}, ptr ${p}, align 1${bytesElementAlias(host)}`); } else { B.line(`${p} = getelementptr inbounds i32, ptr ${data}, ${host.sizeType} ${idx}`); - B.line(`store i32 ${stored!}, ptr ${p}, align 1`); + B.line(`store i32 ${stored!}, ptr ${p}, align 1${bytesElementAlias(host)}`); } B.br(done); B.startBlock(done); diff --git a/packages/compiler/src/backend/llvm/expr-containers.ts b/packages/compiler/src/backend/llvm/expr-containers.ts index 9c77690f8b..470477d44e 100644 --- a/packages/compiler/src/backend/llvm/expr-containers.ts +++ b/packages/compiler/src/backend/llvm/expr-containers.ts @@ -526,7 +526,7 @@ export function emitStableReceiver( if (b.kind !== "boxed") { if (b.kind === "global") host.checkGlobalTdz(receiver.localId); const value = host.B.tmp(); - host.B.line(`${value} = load ptr, ptr ${b.slot}`); + host.B.line(`${value} = load ptr, ptr ${b.slot}${host.slotAlias(receiver.localId)}`); return { name: value, type: receiver.type }; } } diff --git a/packages/compiler/src/backend/llvm/expr-primitives.ts b/packages/compiler/src/backend/llvm/expr-primitives.ts index aaae4c35cc..7ad649f2d8 100644 --- a/packages/compiler/src/backend/llvm/expr-primitives.ts +++ b/packages/compiler/src/backend/llvm/expr-primitives.ts @@ -107,7 +107,7 @@ export function emitLiteralExpr( return host.own({ name: v, type: e.type }); } const t = B.tmp(); - B.line(`${t} = load ${host.llType(b.type)}, ptr ${b.slot}`); + B.line(`${t} = load ${host.llType(b.type)}, ptr ${b.slot}${host.slotAlias(e.localId)}`); const integerView = host.integerViews.get(e.localId); if (integerView !== undefined) { // Snapshot both representations before later operands can write diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5467-typed-array-loop-versions.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5467-typed-array-loop-versions.ts.json new file mode 100644 index 0000000000..a4be596965 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5467-typed-array-loop-versions.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/5467-typed-array-loop-versions.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5468-typed-array-loop-aliasing.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5468-typed-array-loop-aliasing.ts.json new file mode 100644 index 0000000000..517cbab4c6 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5468-typed-array-loop-aliasing.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/5468-typed-array-loop-aliasing.ts" + ], + "diags": [] +} diff --git a/tests/corpus/5467-typed-array-loop-versions.ts b/tests/corpus/5467-typed-array-loop-versions.ts new file mode 100644 index 0000000000..62848ae432 --- /dev/null +++ b/tests/corpus/5467-typed-array-loop-versions.ts @@ -0,0 +1,238 @@ +// Typed-array loops whose accesses are proved in bounds by one test before +// the loop run without per-element checks; every other entry runs the +// checked loop. Both must keep JavaScript's exact out-of-bounds behavior: +// reads of missing elements are undefined (NaN in arithmetic), writes are +// ignored, DataView accesses throw at the failing iteration, and +// fractional, negative and NaN indices never touch storage. + +const table: Int32Array = new Int32Array(256); +for (let n = 0; n < 256; n++) table[n] = (n * 2654435761) | 0; +const shortTable: Int32Array = new Int32Array(100); +for (let n = 0; n < 100; n++) shortTable[n] = n + 1; + +function masked(t: Int32Array, data: Uint8Array): number { + let h = 0; + for (let i = 0; i < data.length; i++) h = (h ^ t[(h ^ data[i]) & 0xff]) | 0; + return h; +} + +function maskedWrite(t: Int32Array, count: number): void { + for (let i = 0; i < count; i++) t[(i * 7) & 0xff] = i; +} + +const data = new Uint8Array(64); +for (let i = 0; i < data.length; i++) data[i] = (i * 37 + 11) & 0xff; +console.log("masked full", masked(table, data)); +console.log("masked short", masked(shortTable, data)); +const writable = new Int32Array(200); +maskedWrite(writable, 300); +console.log("masked write", writable[0], writable[7], writable[199], writable.length); + +// Counter plus an invariant base, in bounds and running off either end. +function shifted(src: Float64Array, out: Float64Array, base: number, n: number): number { + let sum = 0; + for (let i = 0; i < n; i++) { + out[base + i] = src[i] * 2; + sum += src[base + i - 1]; + } + return sum; +} + +const src = new Float64Array(8); +for (let i = 0; i < 8; i++) src[i] = i + 0.25; +for (const [base, n] of [ + [1, 4], + [0, 8], + [5, 8], + [-3, 6], + [2, 0], + [3, 2.5], +]) { + const out = new Float64Array(8); + const sum = shifted(src, out, base, n); + console.log("shifted", base, n, sum, Array.from(out).join(",")); +} + +// Descending loops, including ones that walk below zero. +function descending(a: Uint16Array, low: number): number { + let sum = 0; + for (let i = a.length - 1; i >= low; i--) { + a[i] = a[i] + 1000; + sum += a[i]; + } + return sum; +} +const u16 = new Uint16Array([65000, 1, 2, 3]); +console.log("descending", descending(u16, 0), Array.from(u16).join(",")); +console.log("descending below", descending(u16, -2), Array.from(u16).join(",")); + +// Fractional starts and offsets never name an element. +function fractional(a: Float32Array): string { + const seen: string[] = []; + for (let i = 0.5; i < 3; i++) seen.push(String(a[i])); + for (let i = 0; i < 3; i++) { + a[i + 0.5] = 99; + seen.push(String(a[i] + a[i - 0.5])); + } + return seen.join(" "); +} +console.log("fractional", fractional(new Float32Array([1.5, 2.5, 3.5]))); + +function nanIndex(a: Int8Array, offset: number): number { + let sum = 0; + for (let i = 0; i < a.length; i++) { + a[i + offset] = 120; + sum += a[i + offset] | 0; + } + return sum; +} +const i8 = new Int8Array([1, 2, 3]); +console.log("nan index", nanIndex(i8, NaN), nanIndex(i8, -0), Array.from(i8).join(",")); + +// Unproven limits keep the integer loop's guard: huge, infinite and NaN. +function limits(a: Uint8Array, limit: number): number { + let sum = 0; + for (let i = 0; i < limit; i++) { + if (i >= a.length + 2) break; + sum += a[i] | 0; + a[i] = i; + } + return sum; +} +const lim = new Uint8Array([5, 6, 7]); +console.log("limits", limits(lim, 2), limits(lim, 1e300), limits(lim, Infinity), limits(lim, NaN)); + +// A DataView access past the end throws after the earlier iterations ran. +const viewProgress: number[] = [0]; +function viewLoop(view: DataView, n: number): number { + let sum = 0; + for (let i = 0; i < n; i++) { + sum += view.getUint32(i * 4, true) + view.getUint8(i * 4 + 3); + viewProgress[0] = sum; + } + return sum; +} +function viewSum(view: DataView, n: number): number { + viewProgress[0] = -1; + try { + return viewLoop(view, n); + } catch (e) { + console.log("view throw", (e as Error).name, viewProgress[0]); + } + return viewProgress[0]; +} +const dvBytes = new Uint8Array(16); +for (let i = 0; i < 16; i++) dvBytes[i] = i * 3; +const view = new DataView(dvBytes.buffer); +console.log("view ok", viewSum(view, 4)); +console.log("view partial", viewSum(view, 6)); + +// Strided loops: the last counter value of a non-unit step is exact, so a +// record that ends at the last element still takes the unchecked loop and +// a partial trailing record does not. +function quads(words: Uint16Array): number { + let sum = 0; + for (let i = 0; i < words.length; i += 4) sum += words[i] + words[i + 3] * 2; + return sum; +} +console.log("quads", quads(new Uint16Array([1, 2, 3, 4, 5, 6, 7, 8]))); +console.log("quads partial", quads(new Uint16Array([1, 2, 3, 4, 5, 6]))); +function backTriples(words: Uint16Array): number { + let sum = 0; + for (let i = words.length - 1; i >= 0; i -= 3) sum = sum * 3 + words[i] + (words[i - 2] | 0); + return sum; +} +console.log("back triples", backTriples(new Uint16Array([1, 2, 3, 4, 5, 6]))); +console.log("back triples partial", backTriples(new Uint16Array([1, 2, 3, 4, 5, 6, 7]))); +function scaled(words: Uint16Array, n: number): number { + let sum = 0; + for (let i = 0; i < n; i++) sum += words[i * 2] - (words[(i + 1) * 2 - 1] | 0) + (words[9 - i] | 0); + return sum; +} +const scaledWords = new Uint16Array(10); +for (let i = 0; i < 10; i++) scaledWords[i] = i * i; +console.log("scaled", scaled(scaledWords, 5), scaled(scaledWords, 6), scaled(scaledWords, 11)); +// A negative zero index names element 0 and stays -0 as a number. +function negativeZero(words: Uint16Array, n: number): string { + const seen: string[] = []; + for (let i = 0; i < n; i++) { + const k = i * -1; + const m = (i - 1) * 2; + seen.push(`${words[k]}:${Object.is(k, -0)}:${words[m] | 0}:${Object.is(m, -0)}`); + } + return seen.join(" "); +} +console.log("negative zero", negativeZero(scaledWords, 3)); + +// Clamped and wrapping element types on the unchecked path. +function convert(c: Uint8ClampedArray, s: Int16Array): string { + for (let i = 0; i < 4; i++) { + c[i] = i * 100 - 50.5; + s[i] = i * 20000; + } + return Array.from(c).join(",") + " " + Array.from(s).join(","); +} +console.log("convert", convert(new Uint8ClampedArray(4), new Int16Array(4))); + +// Module receivers stay correct across calls in the loop body, including a +// call that writes the same arrays. +const left: Float64Array = new Float64Array(16); +const right: Float64Array = new Float64Array(16); +function poke(k: number): number { + right[k] = right[k] + 1; + return k; +} +for (let i = 0; i < 16; i++) left[i] = i; +let crossed = 0; +for (let i = 0; i < 16; i++) { + right[i] = left[i] * 3; + crossed += right[poke(i)]; +} +console.log("module receivers", crossed, right[0], right[15]); + +// A body that reassigns its receiver keeps reading the current binding. +function rebinding(first: Uint8Array, second: Uint8Array): number { + let current = first; + let sum = 0; + for (let i = 0; i < 4; i++) { + sum += current[i] | 0; + if (i === 1) current = second; + } + return sum; +} +console.log("rebinding", rebinding(new Uint8Array([1, 2, 3, 4]), new Uint8Array([10, 20]))); + +// A throwing body leaves the writes of earlier iterations in place. +function failing(a: Uint32Array, at: number): void { + for (let i = 0; i < a.length; i++) { + if (i === at) throw new Error("stop at " + i); + a[i] = 4294967295 - i; + } +} +const u32 = new Uint32Array(5); +try { + failing(u32, 3); +} catch (e) { + console.log((e as Error).message, Array.from(u32).join(",")); +} + +// A matrix product over module-level constant arrays. +const N = 6; +const ma: Float64Array = new Float64Array(N * N); +const mb: Float64Array = new Float64Array(N * N); +const mc: Float64Array = new Float64Array(N * N); +for (let i = 0; i < N * N; i++) { + ma[i] = (i % 7) - 3; + mb[i] = (i % 5) * 0.5; +} +for (let i = 0; i < N; i++) { + for (let k = 0; k < N; k++) { + const aik = ma[i * N + k]; + const kOff = k * N; + const iOff = i * N; + for (let j = 0; j < N; j++) mc[iOff + j] += aik * mb[kOff + j]; + } +} +let trace = 0; +for (let i = 0; i < N; i++) trace += mc[i * N + i]; +console.log("matmul", trace, mc[1], mc[N * N - 1]); diff --git a/tests/corpus/5468-typed-array-loop-aliasing.ts b/tests/corpus/5468-typed-array-loop-aliasing.ts new file mode 100644 index 0000000000..befcc8227b --- /dev/null +++ b/tests/corpus/5468-typed-array-loop-aliasing.ts @@ -0,0 +1,87 @@ +// Element stores may alias any other view of the same buffer, whatever its +// element type, offset or length, but never a typed array's own length and +// data pointer. Loops that write through one view and read through another +// must observe every store in program order, also when the loop runs +// without per-element bounds checks. + +const buffer = new ArrayBuffer(64); +const f64 = new Float64Array(buffer); +const u8 = new Uint8Array(buffer); +const i32 = new Int32Array(buffer, 8, 8); + +// Writes through Float64Array observed through a Uint8Array in the same +// iteration. +let topBytes = 0; +for (let i = 0; i < f64.length; i++) { + f64[i] = -(i + 1) * 1.5; + topBytes += u8[i * 8 + 7]; +} +console.log("f64 then u8", topBytes, u8[7], u8[63]); + +// Byte stores observed through an overlapping Int32Array window. +let words = 0; +for (let i = 0; i < 32; i++) { + u8[8 + i] = i + 1; + words = (words + i32[i >> 2]) | 0; +} +console.log("u8 then i32", words, i32[0], i32[7]); + +// A shifted subarray reads what the previous iteration wrote: a prefix +// recurrence that any reordering or vector widening would break. +const chain = new Uint8Array(40); +chain[0] = 1; +const ahead = chain.subarray(1); +for (let i = 0; i < ahead.length; i++) ahead[i] = chain[i] + 2; +console.log("recurrence", chain[0], chain[1], chain[20], chain[39]); + +// Overlapping copies in both directions. +const values = new Int16Array(12); +for (let i = 0; i < values.length; i++) values[i] = i * 10; +const lower = values.subarray(0, 11); +const upper = values.subarray(1); +for (let i = 0; i < lower.length; i++) lower[i] = upper[i]; +console.log("copy down", Array.from(values).join(",")); +for (let i = 0; i < values.length; i++) values[i] = i * 10; +for (let i = 0; i < lower.length; i++) upper[i] = lower[i]; +console.log("copy up", Array.from(values).join(",")); + +// Same-array accumulation with a lagging read. +function lagged(a: Float64Array, lag: number): number { + for (let i = lag; i < a.length; i++) a[i] = a[i] + a[i - lag]; + return a[a.length - 1]; +} +const fib = new Float64Array(30); +fib[0] = 1; +fib[1] = 1; +console.log("lagged", lagged(fib, 1), lagged(fib, 2)); + +// Two parameters that are the same array, and a destination that aliases +// part of its own source. +function axpy(alpha: number, x: Float64Array, y: Float64Array, n: number): void { + for (let i = 0; i < n; i++) y[i] = y[i] + alpha * x[i]; +} +const same = new Float64Array([1, 2, 3, 4, 5, 6, 7, 8]); +axpy(2, same, same, same.length); +console.log("axpy same", Array.from(same).join(",")); +const whole = new Float64Array([1, 2, 3, 4, 5, 6, 7, 8]); +axpy(1, whole, whole.subarray(2), 6); +console.log("axpy shifted", Array.from(whole).join(",")); + +// Element stores never change a length or a module binding, even when a +// float store writes the bit pattern of a small length. +const meta: Float64Array = new Float64Array(4); +const lengths: number[] = []; +for (let i = 0; i < 4; i++) { + meta[i] = 5e-324 * (i + 1); + lengths.push(meta.length); +} +console.log("lengths", lengths.join(","), meta[3] > 0); + +// Global numbers next to typed-array stores keep their values. +let scale = 3; +const scaled: Float32Array = new Float32Array(6); +for (let i = 0; i < scaled.length; i++) { + scaled[i] = scale * i; + if (i === 2) scale = 0.5; +} +console.log("scaled", Array.from(scaled).join(","));