From de782764c2039b5652dda4626383047316c1024a Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Wed, 14 Oct 2026 16:55:54 -0700 Subject: [PATCH] Version typed-array loops on one bounds guard and tag typed-array memory for alias analysis Typedarray element stores could alias every header and module-global load, so each access reloaded the length, data pointer and global receiver slot and kept its own bounds branch. Element accesses now share one TBAA category (views of any element type may share a buffer), header loads another, and whole-slot loads of bytes/number globals their own, so LICM keeps them in registers. The data pointer loads beside the length, ahead of the branch. ScrBytes lengths and data pointers are fixed at construction, so innermost synchronous for loops whose accesses use a statically ranged index or an affine counter index (k * counter + invariant + c) are versioned: one test before the loop proves every access in bounds (bounding the counter so every affine operand stays an exact safe integer), and the copy behind it runs without per-element checks. The checked loop keeps exact out-of-bounds, NaN, fractional and DataView-throw behavior. --- .../compiler/src/backend/llvm/byte-numbers.ts | 7 +- .../compiler/src/backend/llvm/bytes-alias.ts | 23 + .../backend/llvm/bytes-loop-versions.test.ts | 210 +++++++ .../src/backend/llvm/bytes-loop-versions.ts | 561 ++++++++++++++++++ .../src/backend/llvm/counted-loops.ts | 51 ++ packages/compiler/src/backend/llvm/emitter.ts | 122 ++-- .../compiler/src/backend/llvm/expr-bytes.ts | 64 +- .../src/backend/llvm/expr-containers.ts | 2 +- .../src/backend/llvm/expr-primitives.ts | 2 +- .../5467-typed-array-loop-versions.ts.json | 6 + .../5468-typed-array-loop-aliasing.ts.json | 6 + .../corpus/5467-typed-array-loop-versions.ts | 238 ++++++++ .../corpus/5468-typed-array-loop-aliasing.ts | 87 +++ 13 files changed, 1303 insertions(+), 76 deletions(-) create mode 100644 packages/compiler/src/backend/llvm/bytes-alias.ts create mode 100644 packages/compiler/src/backend/llvm/bytes-loop-versions.test.ts create mode 100644 packages/compiler/src/backend/llvm/bytes-loop-versions.ts create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5467-typed-array-loop-versions.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5468-typed-array-loop-aliasing.ts.json create mode 100644 tests/corpus/5467-typed-array-loop-versions.ts create mode 100644 tests/corpus/5468-typed-array-loop-aliasing.ts diff --git a/packages/compiler/src/backend/llvm/byte-numbers.ts b/packages/compiler/src/backend/llvm/byte-numbers.ts index c51d03744a..9001e015dc 100644 --- a/packages/compiler/src/backend/llvm/byte-numbers.ts +++ b/packages/compiler/src/backend/llvm/byte-numbers.ts @@ -4,6 +4,7 @@ import type { LlValue, LlvmEmitterContext } from "./expr-context.js"; import { f64Lit } from "./common.js"; import { exactInteger, widenInteger } from "./integer-values.js"; import { emitTruncF64 } from "./trunc.js"; +import { bytesElementAlias, bytesHeaderAlias } from "./bytes-alias.js"; /** Supported LLVM targets are little-endian. All memory accesses use align * 1: Buffer fields and DataView windows can start at arbitrary byte offsets. */ @@ -79,7 +80,7 @@ export function emitByteNumber( capacity = B.tmp(); room = B.tmp(); B.line(`${lengthPtr} = getelementptr inbounds %ScrBytes, ptr ${receiver.name}, i64 0, i32 1`); - B.line(`${length} = load ${size}, ptr ${lengthPtr}`); + B.line(`${length} = load ${size}, ptr ${lengthPtr}${bytesHeaderAlias(host)}`); B.line(`${room} = icmp uge ${size} ${length}, ${spec.width}`); B.line(`${capacity} = sub ${size} ${length}, ${spec.width}`); if (integer) { @@ -221,14 +222,14 @@ export function emitByteNumber( } } const stored = endianBits(host, raw, spec.width, little); - B.line(`store i${bits} ${stored}, ptr ${pointer}, align 1`); + B.line(`store i${bits} ${stored}, ptr ${pointer}, align 1${bytesElementAlias(host)}`); if (!spec.dataView) { fastResult = B.tmp(); B.line(`${fastResult} = fadd double ${offset.name}, ${f64Lit(spec.width)}`); } } else { const loaded = B.tmp(); - B.line(`${loaded} = load i${bits}, ptr ${pointer}, align 1`); + B.line(`${loaded} = load i${bits}, ptr ${pointer}, align 1${bytesElementAlias(host)}`); const raw = endianBits(host, loaded, spec.width, little); fastResult = B.tmp(); if (spec.floating) { diff --git a/packages/compiler/src/backend/llvm/bytes-alias.ts b/packages/compiler/src/backend/llvm/bytes-alias.ts new file mode 100644 index 0000000000..9aaeaa6c77 --- /dev/null +++ b/packages/compiler/src/backend/llvm/bytes-alias.ts @@ -0,0 +1,23 @@ +import type { LlvmEmitterContext } from "./expr-context.js"; + +/** Alias categories of typed-array storage. A ScrBytes header (length, + * data pointer, shared backing) is fixed at construction and only ever + * written by runtime code, never by emitted code. Element storage is one + * category for every element type: views of different types may share a + * buffer. Neither overlaps the other, a native field, array storage or a + * module global's slot, so element stores leave header loads and global + * receiver slots loop-invariant. Untagged accesses and calls still alias + * every category. */ +export function bytesHeaderAlias(host: LlvmEmitterContext): string { + return host.memoryAliasAttachment("bytes:header"); +} + +export function bytesElementAlias(host: LlvmEmitterContext): string { + return host.memoryAliasAttachment("bytes:elements"); +} + +/** A module global's slot is its own category for whole-slot loads of a + * scalar or typed-array binding. Stores stay untagged (and so alias it). */ +export function globalSlotAlias(host: LlvmEmitterContext, id: string, type: string): string { + return type === "bytes" || type === "f64" ? host.memoryAliasAttachment(`global:${id}`) : ""; +} diff --git a/packages/compiler/src/backend/llvm/bytes-loop-versions.test.ts b/packages/compiler/src/backend/llvm/bytes-loop-versions.test.ts new file mode 100644 index 0000000000..d8ec036fff --- /dev/null +++ b/packages/compiler/src/backend/llvm/bytes-loop-versions.test.ts @@ -0,0 +1,210 @@ +import { expect, test } from "vitest"; +import { + BOOL, + F64, + VOID, + type IrExpr, + type IrFunction, + type IrLocal, + type IrModule, + type IrStmt, + type IrType, +} from "../../ir/ir.js"; +import { analyzeIntegerRanges } from "../../ir/integer-ranges.js"; +import { matchIntegerCountedForLoop } from "../../ir/integer-loops.js"; +import { findInitializedByteLoopBindings } from "./byte-windows.js"; +import { matchBytesLoopVersion, type BytesLoopVersion } from "./bytes-loop-versions.js"; +import { emitLlvmModule } from "./emitter.js"; + +const loc = { file: "versions.ts", start: 0, end: 0 }; +const f64s: IrType = { kind: "bytes", elem: "f64" }; +const i32s: IrType = { kind: "bytes", elem: "i32" }; +const num = (value: number): IrExpr => ({ kind: "numLit", value, type: F64, loc }); +const ref = (localId: string, type: IrType = F64): IrExpr => ({ + kind: "varRef", + localId, + type, + loc, +}); +const bin = (op: string, left: IrExpr, right: IrExpr): IrExpr => + ({ kind: "bin", op, left, right, type: F64, loc }) as IrExpr; +const get = (receiver: IrExpr, index: IrExpr): IrExpr => ({ + kind: "bytesIntrinsic", + method: "get", + receiver, + args: [index], + invalidNaN: true, + type: F64, + loc, +}); +const set = (arr: IrExpr, index: IrExpr, value: IrExpr): IrStmt => ({ + kind: "bytesSet", + arr, + index, + value, + loc, +}); +const counted = ( + body: IrStmt[], + limit: IrExpr = ref("n"), + counter = "i", +): IrStmt & { kind: "for" } => ({ + kind: "for", + init: { kind: "varDecl", localId: counter, init: num(0), loc }, + cond: { kind: "bin", op: "<", left: ref(counter), right: limit, type: BOOL, loc }, + update: { kind: "assign", localId: counter, value: bin("+", ref(counter), num(1)), loc }, + body, + loc, +}); +const local = ( + id: string, + type: IrType, + mutable = false, + extra: Partial = {}, +): IrLocal => ({ + id, + name: id, + type, + mutable, + ...extra, +}); + +function fn(body: IrStmt[], extra: IrLocal[] = []): IrFunction { + return { + name: "kernel", + params: [ + { localId: "out", name: "out", type: f64s }, + { localId: "src", name: "src", type: f64s }, + { localId: "table", name: "table", type: i32s }, + { localId: "n", name: "n", type: F64 }, + { localId: "k", name: "k", type: F64 }, + ], + returnType: VOID, + locals: [ + local("out", f64s), + local("src", f64s), + local("table", i32s), + local("n", F64), + local("k", F64), + local("i", F64, true), + local("j", F64, true), + ...extra, + ], + body, + loc, + }; +} + +function match( + f: IrFunction, + loop: IrStmt & { kind: "for" }, + captured: string[] = [], +): BytesLoopVersion | null { + const ranges = analyzeIntegerRanges(f); + const locals = new Map(f.locals.map((l) => [l.id, l])); + return matchBytesLoopVersion(loop, matchIntegerCountedForLoop(loop, locals, ranges), { + locals, + invariantGlobals: new Set(), + captured: new Set(captured), + ranges, + initialized: findInitializedByteLoopBindings(f).get(loop) ?? new Set(), + proven: new Set(), + }); +} + +test("a counted loop's affine accesses get one guard and exact operand ranges", () => { + const shifted = bin("+", ref("i"), num(1)); + const loop = counted([set(ref("out", f64s), ref("i"), get(ref("src", f64s), shifted))]); + const version = match(fn([loop]), loop)!; + expect(version).not.toBeNull(); + expect(version.bounds.size).toBe(2); + expect(version.receivers.map((r) => r.ref.kind === "varRef" && r.ref.localId)).toEqual([ + "out", + "src", + ]); + expect(version.affine.map((c) => [c.scale, c.minOffset, c.maxEnd])).toEqual([ + [1, 0, 0], + [1, 1, 1], + ]); + // `n` is unproven: the guard bounds the counter so `i + 1` stays exact. + expect(version.counted?.limitGuarded).toBe(true); + expect(version.counterBound).toBe(Number.MAX_SAFE_INTEGER - 1); + expect(version.ranges.get(shifted)).toEqual({ min: 1, max: Number.MAX_SAFE_INTEGER }); +}); + +test("statically ranged indices compare one constant with the length", () => { + const masked = bin("&", bin("^", ref("i"), ref("k")), num(255)); + const loop = counted([set(ref("out", f64s), num(0), get(ref("table", i32s), masked))]); + const version = match(fn([loop]), loop)!; + expect(version.fixed.map((c) => [version.receivers[c.receiver]!.ref, c.end])).toEqual([ + [ref("out", f64s), 0], + [ref("table", i32s), 255], + ]); + expect(version.affine).toEqual([]); + expect(version.counted).toBeNull(); +}); + +test("scaled counters and body temps resolve to their receiver and index", () => { + const temps = [local("view", f64s), local("at", F64)]; + const at = bin("+", bin("*", ref("i"), num(4)), num(3)); + const loop = counted([ + { kind: "varDecl", localId: "view", init: ref("src", f64s), loc }, + { kind: "varDecl", localId: "at", init: at, loc }, + set(ref("view", f64s), ref("at"), num(1)), + // (The integer counted loop needs one direct counter index.) + set(ref("out", f64s), ref("i"), num(0)), + ]); + const version = match(fn([loop], temps), loop)!; + expect(version.receivers.map((r) => r.ref)).toEqual([ref("src", f64s), ref("out", f64s)]); + expect(version.affine.map((c) => [c.scale, c.minOffset, c.maxEnd])).toEqual([ + [4, 3, 3], + [1, 0, 0], + ]); + expect(version.ranges.has(at)).toBe(true); +}); + +test("negative factors, rebinding, captures and outer loops are not versioned", () => { + // `i * -1` is -0 at zero, so it is not treated as an affine operand. + const negative = counted([set(ref("out", f64s), bin("*", ref("i"), num(-1)), num(1))]); + expect(match(fn([negative]), negative)).toBeNull(); + const rebound = counted([ + set(ref("out", f64s), ref("i"), num(1)), + { kind: "assign", localId: "out", value: ref("src", f64s), loc }, + ]); + expect(match(fn([rebound]), rebound)).toBeNull(); + const plain = counted([set(ref("out", f64s), ref("i"), num(1))]); + expect(match(fn([plain]), plain, ["out"])).toBeNull(); + const inner = counted([set(ref("out", f64s), ref("j"), num(1))], ref("n"), "j"); + const outer = counted([inner]); + expect(match(fn([outer]), outer)).toBeNull(); + expect(match(fn([outer]), inner)).not.toBeNull(); +}); + +test("emitted loops version the body and tag typed-array memory", () => { + const loop = counted([ + set(ref("out", f64s), ref("i"), bin("*", get(ref("src", f64s), ref("i")), num(2))), + ]); + const kernel = fn([loop]); + const mod: IrModule = { + irVersion: 16, + sourceFile: loc.file, + entry: "__main", + functions: [ + { name: "__main", params: [], returnType: VOID, locals: [], body: [], loc }, + kernel, + ], + }; + const llvm = emitLlvmModule(mod); + const body = /^define internal [^\n]*@sc_bf_kernel[^]*?^}/m.exec(llvm)![0]; + expect(body).toContain("bytes.version.fast"); + expect(body).toContain("bytes.version.slow"); + // The checked copy keeps its per-element tests; the guard reads lengths. + expect(body).toContain("bytes.index.invalid"); + expect(body).toMatch(/store double [^\n]*, align 1, !tbaa !\d+/); + expect(body).toMatch(/load double, ptr [^\n]*, align 1, !tbaa !\d+/); + expect(body).toMatch(/load i64, ptr [^\n]*, !tbaa !\d+/); + expect(body).toMatch(/load ptr, ptr [^\n]*, !tbaa !\d+/); + const elements = /store double [^\n]*!tbaa !(\d+)/.exec(body)![1]; + const header = /load ptr, ptr [^\n]*!tbaa !(\d+)/.exec(body)![1]; + expect(elements).not.toBe(header); +}); diff --git a/packages/compiler/src/backend/llvm/bytes-loop-versions.ts b/packages/compiler/src/backend/llvm/bytes-loop-versions.ts new file mode 100644 index 0000000000..d3a9b20429 --- /dev/null +++ b/packages/compiler/src/backend/llvm/bytes-loop-versions.ts @@ -0,0 +1,561 @@ +import type { IrExpr, IrLocal, IrStmt } from "../../ir/ir.js"; +import type { IntegerRange, IntegerRanges } from "../../ir/integer-ranges.js"; +import type { IntegerCountedForLoop } from "../../ir/integer-loops.js"; +import type { LlvmEmitterContext } from "./expr-context.js"; +import { byteNumberAccess } from "../../ir/byte-numbers.js"; +import { everyExprChild, everyStmtChild } from "../../ir/traverse.js"; +import { exactInteger, widenInteger } from "./integer-values.js"; +import { emitCountedLoopLimit, emitCountedLoopSafety } from "./counted-loops.js"; +import { emitBytesHeaderLength } from "./expr-bytes.js"; + +type Node = IrExpr | IrStmt; +type VarRef = IrExpr & { kind: "varRef" }; + +/** One loop-invariant integer operand of an affine index. */ +interface Term { + expr: IrExpr; + sign: 1 | -1; +} + +/** Every access of `receiver` at a static index range ends below `end + 1`. */ +interface FixedCheck { + receiver: number; + end: number; +} + +/** Every access of `receiver` at `scale * counter + Σ terms + offset` has an + * offset in [minOffset, maxEnd - width + 1]; `maxEnd` includes the width. */ +interface AffineCheck { + receiver: number; + scale: number; + terms: Term[]; + minOffset: number; + maxEnd: number; +} + +export interface BytesLoopVersion { + /** Typed-array accesses that are in bounds whenever the guard holds. */ + bounds: ReadonlySet; + /** Root receiver bindings, read once before the loop. */ + receivers: { ref: VarRef; global: boolean }[]; + fixed: FixedCheck[]; + affine: AffineCheck[]; + /** The counted loop whose iteration space the affine checks cover. */ + counted: IntegerCountedForLoop | null; + /** The guard also requires every counter value within [-bound, bound]. */ + counterBound: number | null; + /** Integer ranges of affine index operands that hold in the unchecked + * loop (where the guard bounds the counter). */ + ranges: ReadonlyMap; +} + +export interface BytesLoopFacts { + /** The function's own locals (not module globals). */ + locals: ReadonlyMap; + /** Module globals whose slot no call can replace: stable constants and + * the private bindings of a guarded initializer. */ + invariantGlobals: ReadonlySet; + captured: ReadonlySet; + ranges: IntegerRanges; + /** Locals definitely initialized when the loop is entered. */ + initialized: ReadonlySet; + /** Accesses already proved in bounds without a guard. */ + proven: ReadonlySet; +} + +const MAX_COST = 400; +const MAX_CHECKS = 8; +const MAX_OFFSET = 2 ** 31; +const SAFE = Number.MAX_SAFE_INTEGER; +const INVARIANT_OPS = new Set(["+", "-", "*", "&", "|", "^", "<<", ">>", ">>>"]); + +/** Loop versioning for typed-array bounds. A typed array's length and data + * pointer are fixed at construction (ScrBytes has no resize, transfer or + * detach), so an access is in bounds for the whole loop when its receiver + * binding and index operands cannot change while it runs. One test before + * the loop then covers every iteration: + * + * - a statically ranged integer index `[lo, hi]` (lo >= 0) needs + * `hi < length`, such as `table[x & 0xff]` or a constant-trip counter; + * - an index `k * counter + invariant + c` of a counted loop needs its first + * and last values in bounds, such as `out[base + i]` for `i < n` or + * `view.getUint32(i * 4)`. + * + * Only innermost synchronous loops are versioned, so the body is emitted + * twice at most. The checked loop runs whenever the guard fails, and keeps + * every out-of-bounds read, ignored write and NaN-index behavior. */ +export function matchBytesLoopVersion( + loop: IrStmt & { kind: "for" }, + counted: IntegerCountedForLoop | null, + facts: BytesLoopFacts, +): BytesLoopVersion | null { + const written = new Set(); + const temps = new Map(); + let cost = 0; + let innermost = true; + const scanExpr = (e: IrExpr): boolean => { + if (++cost > MAX_COST) return false; + if (e.kind === "assignExpr" || e.kind === "incDec") written.add(e.localId); + return everyExprChild(e, scanExpr, scanStmt); + }; + const scanStmt = (s: IrStmt): boolean => { + if (++cost > MAX_COST) return false; + switch (s.kind) { + case "for": + case "forOf": + case "while": + case "doWhile": + innermost = false; + return false; + case "varDecl": { + if (temps.has(s.localId) || written.has(s.localId)) temps.delete(s.localId); + else if (s.init) temps.set(s.localId, s.init); + written.add(s.localId); + break; + } + case "assign": + written.add(s.localId); + break; + case "tryCatch": + if (s.catchLocalId) written.add(s.catchLocalId); + break; + default: + break; + } + return everyStmtChild(s, scanExpr, scanStmt); + }; + if ( + !loop.body.every(scanStmt) || + !innermost || + (loop.cond !== null && !scanExpr(loop.cond)) || + (loop.update !== null && !scanStmt(loop.update)) + ) + return null; + // A body temp stands for its initializer only when it is a constant + // binding assigned exactly once, by that declaration. + const declared = new Map(); + const countDecls = (s: IrStmt): boolean => { + if (s.kind === "varDecl") declared.set(s.localId, (declared.get(s.localId) ?? 0) + 1); + return everyStmtChild(s, countExprDecls, countDecls); + }; + const countExprDecls = (e: IrExpr): boolean => everyExprChild(e, countExprDecls, countDecls); + loop.body.every(countDecls); + const temp = (id: string): IrExpr | null => { + const local = facts.locals.get(id); + const init = temps.get(id); + if ( + !init || + !local || + local.mutable || + local.boxed || + local.tdz || + facts.captured.has(id) || + declared.get(id) !== 1 + ) + return null; + return init; + }; + const counter = counted?.localId ?? null; + // A binding keeps its loop-entry value for the whole loop. + const invariant = (id: string): boolean => { + if (written.has(id) || id === counter) return false; + const local = facts.locals.get(id); + if (local) + return !local.boxed && !local.tdz && !facts.captured.has(id) && facts.initialized.has(id); + return facts.invariantGlobals.has(id); + }; + const resolving = new Set(); + const root = (e: IrExpr): VarRef | null => { + if (e.kind !== "varRef" || e.type.kind !== "bytes") return null; + const init = temp(e.localId); + if (init) { + if (resolving.has(e.localId)) return null; + resolving.add(e.localId); + const result = root(init); + resolving.delete(e.localId); + return result; + } + return invariant(e.localId) ? e : null; + }; + // A pure integer operand evaluated again before the loop. + const invariantKey = (e: IrExpr): string | null => { + if (!facts.ranges.get(e)) return null; + if (e.kind === "numLit") return Number.isSafeInteger(e.value) ? `n${e.value}` : null; + if (e.kind === "varRef") + return invariant(e.localId) ? `v${e.localId.length}:${e.localId}` : null; + if ( + e.kind === "bin" && + INVARIANT_OPS.has(e.op) && + e.left.type.kind === "f64" && + e.right.type.kind === "f64" + ) { + const left = invariantKey(e.left), + right = invariantKey(e.right); + return left !== null && right !== null ? `(${e.op} ${left} ${right})` : null; + } + return null; + }; + interface Affine { + /** The counter's coefficient; 0 when the value is loop-invariant. */ + counter: number; + terms: (Term & { key: string })[]; + offset: number; + } + // Every operand of an affine index is `counter * k + Σ terms + offset`. + // Invariant terms need proven ranges. Operands that involve the counter + // need none: the guard bounds the counter so that each of them is a safe + // integer, which makes the index's double arithmetic exact. + const shapes = new Map(); + const affine = (e: IrExpr): Affine | null => { + if (shapes.has(e)) return shapes.get(e)!; + const result = affineShape(e); + shapes.set(e, result); + return result; + }; + const affineShape = (e: IrExpr): Affine | null => { + if (e.kind === "numLit") + return Number.isSafeInteger(e.value) && Math.abs(e.value) <= MAX_OFFSET + ? { counter: 0, terms: [], offset: e.value } + : null; + if (e.kind === "varRef") { + if (e.localId === counter) return { counter: 1, terms: [], offset: 0 }; + const init = temp(e.localId); + if (init) { + if (resolving.has(e.localId)) return null; + resolving.add(e.localId); + const result = affine(init); + resolving.delete(e.localId); + return result; + } + } + if ( + e.kind === "bin" && + (e.op === "+" || e.op === "-") && + e.left.type.kind === "f64" && + e.right.type.kind === "f64" + ) { + const left = affine(e.left), + right = affine(e.right); + if (left && right && (left.counter === 0 || right.counter === 0)) { + const sign = e.op === "+" ? 1 : -1; + const offset = left.offset + sign * right.offset; + if (Math.abs(offset) > MAX_OFFSET) return null; + const terms = [ + ...left.terms, + ...right.terms.map((t) => ({ ...t, sign: (sign * t.sign) as 1 | -1 })), + ]; + const result = { counter: left.counter + sign * right.counter, terms, offset }; + if (result.counter !== 0 || facts.ranges.get(e)) return result; + } + } + // A positive constant multiple of the counter: `i * 4`, `4 * i`, + // `(i + 1) * 8`. (A negative factor turns a zero into -0.) + if (e.kind === "bin" && e.op === "*") { + const literal = + e.right.kind === "numLit" ? e.right : e.left.kind === "numLit" ? e.left : null; + const scaled = literal === e.right ? e.left : e.right; + const factor = literal?.value ?? 0; + if ( + literal && + scaled.type.kind === "f64" && + Number.isSafeInteger(factor) && + factor > 0 && + factor <= MAX_OFFSET + ) { + const inner = affine(scaled); + if (inner && inner.counter !== 0 && inner.terms.length === 0) { + const counterScale = inner.counter * factor, + offset = inner.offset * factor; + if (Math.abs(counterScale) <= MAX_OFFSET && Math.abs(offset) <= MAX_OFFSET) + return { counter: counterScale, terms: [], offset }; + } + } + } + const key = invariantKey(e); + return key !== null ? { counter: 0, terms: [{ expr: e, sign: 1, key }], offset: 0 } : null; + }; + // The operands of an index that involve the counter. + const chain = (e: IrExpr, out: Set): void => { + if (!shapes.get(e)?.counter) return; + out.add(e); + if (e.kind === "varRef") { + const init = temp(e.localId); + if (init) chain(init, out); + } else if (e.kind === "bin") { + chain(e.left, out); + chain(e.right, out); + } + }; + const termMagnitude = (terms: Term[]): number => + terms.reduce((sum, t) => { + const r = facts.ranges.get(t.expr)!; + return sum + Math.max(Math.abs(r.min), Math.abs(r.max)); + }, 0); + + const receivers: { ref: VarRef; global: boolean }[] = []; + const receiverIndex = new Map(); + const fixed = new Map(); + const affineChecks = new Map< + string, + { check: AffineCheck; nodes: Node[]; operands: Set } + >(); + const receiverOf = (ref: VarRef): number => { + let index = receiverIndex.get(ref.localId); + if (index === undefined) { + index = receivers.length; + receiverIndex.set(ref.localId, index); + receivers.push({ ref, global: !facts.locals.has(ref.localId) }); + } + return index; + }; + const access = (node: Node, array: IrExpr, index: IrExpr, width: number): void => { + if (facts.proven.has(node)) return; + const ref = root(array); + if (!ref) return; + const shape = counter !== null ? affine(index) : null; + if (shape && shape.counter !== 0) { + const operands = new Set(); + chain(index, operands); + const terms = [...shape.terms].sort((a, b) => + a.key < b.key ? -1 : a.key > b.key ? 1 : a.sign - b.sign, + ); + const receiver = receiverOf(ref); + const key = `${receiver}|${shape.counter}|${terms.map((t) => `${t.sign}${t.key}`).join(",")}`; + const end = shape.offset + width - 1; + const group = affineChecks.get(key); + if (group) { + group.check.minOffset = Math.min(group.check.minOffset, shape.offset); + group.check.maxEnd = Math.max(group.check.maxEnd, end); + group.nodes.push(node); + for (const operand of operands) group.operands.add(operand); + } else + affineChecks.set(key, { + check: { receiver, scale: shape.counter, terms, minOffset: shape.offset, maxEnd: end }, + nodes: [node], + operands, + }); + return; + } + const range = facts.ranges.get(index); + if (!range) return; + const end = range.max + width - 1; + if (range.min < 0 || !Number.isSafeInteger(end)) return; + const receiver = receiverOf(ref); + const group = fixed.get(receiver); + if (group) { + group.check.end = Math.max(group.check.end, end); + group.nodes.push(node); + } else fixed.set(receiver, { check: { receiver, end }, nodes: [node] }); + }; + const visitExpr = (e: IrExpr): boolean => { + if (e.kind === "bytesIntrinsic") { + const numeric = byteNumberAccess(e); + if (numeric) access(e, e.receiver, e.args[numeric.offsetArg]!, numeric.width); + else if (e.method === "get" && e.args[0]) access(e, e.receiver, e.args[0], 1); + } + return everyExprChild(e, visitExpr, visitStmt); + }; + const visitStmt = (s: IrStmt): boolean => { + if (s.kind === "bytesSet") access(s, s.arr, s.index, 1); + return everyStmtChild(s, visitExpr, visitStmt); + }; + loop.body.every(visitStmt); + // Bound the counter so every counter operand stays a safe integer. + let bound = SAFE; + const affineGroups = [...affineChecks.values()].filter((group) => { + let limit = SAFE; + for (const operand of group.operands) { + const shape = shapes.get(operand)!; + const room = SAFE - termMagnitude(shape.terms) - Math.abs(shape.offset); + limit = Math.min(limit, Math.floor(room / Math.abs(shape.counter))); + } + if (limit < 0 || !counted) return false; + bound = Math.min(bound, limit); + return true; + }); + const groups = [...fixed.values(), ...affineGroups].slice(0, MAX_CHECKS); + if (groups.length === 0) return null; + const bounds = new Set(); + const used = new Set(); + const ranges = new Map(); + const counterLow = Math.max(counted?.range.min ?? 0, -bound), + counterHigh = Math.min(counted?.range.max ?? 0, bound); + for (const group of groups) { + used.add(group.check.receiver); + for (const node of group.nodes) bounds.add(node); + if (!("operands" in group)) continue; + for (const operand of group.operands) { + if (facts.ranges.get(operand)) continue; + const shape = shapes.get(operand)!; + const a = shape.counter * counterLow, + b = shape.counter * counterHigh; + let min = Math.min(a, b) + shape.offset, + max = Math.max(a, b) + shape.offset; + for (const term of shape.terms) { + const r = facts.ranges.get(term.expr)!; + min += term.sign > 0 ? r.min : -r.max; + max += term.sign > 0 ? r.max : -r.min; + } + if (counterLow <= counterHigh) ranges.set(operand, { min, max }); + } + } + const kept = groups.map((g) => g.check); + const affineKept = kept.filter((c): c is AffineCheck => "terms" in c); + // Renumber the receivers the kept checks use. + const order = [...used].sort((a, b) => a - b); + const renumber = new Map(order.map((old, index) => [old, index])); + const needsBound = + affineKept.length > 0 && + counted !== null && + (counted.range.min < -bound || counted.range.max > bound); + return { + bounds, + receivers: order.map((index) => receivers[index]!), + fixed: kept + .filter((c): c is FixedCheck => !("terms" in c)) + .map((c) => ({ ...c, receiver: renumber.get(c.receiver)! })), + affine: affineKept.map((c) => ({ ...c, receiver: renumber.get(c.receiver)! })), + counted: affineKept.length > 0 ? counted : null, + counterBound: needsBound ? bound : null, + ranges, + }; +} + +/** Branches to `fast` when every access the version proves is in bounds for + * every iteration, and to `slow` otherwise (including a counted loop whose + * integer form is not established, or a module receiver not yet set). */ +export function emitBytesLoopVersionGuard( + host: LlvmEmitterContext, + version: BytesLoopVersion, + fast: string, + slow: string, +): void { + const B = host.B; + const next = (cond: string): void => { + if (cond === "true") return; + const ok = B.newLabel("bytes.version.check"); + B.condBr(cond, ok, slow); + B.startBlock(ok); + }; + const counted = version.counted; + if (counted?.guarded) next(emitCountedLoopSafety(host, counted)); + const receivers = version.receivers.map(({ ref, global }) => { + const value = host.emitReadReceiver(ref).name; + if (global) { + const set = B.tmp(); + B.line(`${set} = icmp ne ptr ${value}, null`); + next(set); + } + return value; + }); + const lengths = receivers.map((receiver) => { + const len = emitBytesHeaderLength(host, receiver); + if (host.sizeType === "i64") return len; + const wide = B.tmp(); + B.line(`${wide} = zext ${host.sizeType} ${len} to i64`); + return wide; + }); + const conditions: string[] = []; + for (const check of version.fixed) { + const ok = B.tmp(); + B.line(`${ok} = icmp ugt i64 ${lengths[check.receiver]}, ${check.end}`); + conditions.push(ok); + } + if (counted && version.affine.length > 0) { + // The first and last counter values, in ascending order. The last value + // of a non-unit step is exact: `start + ⌊(limit ∓ 1 - start) / step⌋ * + // step`. A loop that runs no iteration may compute anything here. + const startValue = host.emitExpr(counted.start); + const startInteger = exactInteger(host, startValue, counted.start); + let start: string; + if (startInteger) start = widenInteger(host, startInteger); + else { + // The counted-loop safety test above proved an exact integer. + start = B.tmp(); + B.line(`${start} = fptosi double ${startValue.name} to i64`); + } + const limit = emitCountedLoopLimit(host, counted); + const step = Math.abs(counted.step); + const edge = B.tmp(); + B.line(`${edge} = add i64 ${limit}, ${counted.step > 0 ? -1 : 1}`); + let last = edge; + if (step !== 1) { + const span = B.tmp(), + whole = B.tmp(), + covered = B.tmp(); + last = B.tmp(); + B.line( + counted.step > 0 + ? `${span} = sub i64 ${edge}, ${start}` + : `${span} = sub i64 ${start}, ${edge}`, + ); + B.line(`${whole} = sdiv i64 ${span}, ${step}`); + B.line(`${covered} = mul i64 ${whole}, ${step}`); + B.line(`${last} = ${counted.step > 0 ? "add" : "sub"} i64 ${start}, ${covered}`); + } + const low = counted.step > 0 ? start : last; + const high = counted.step > 0 ? last : start; + if (version.counterBound !== null) { + const above = B.tmp(), + below = B.tmp(); + B.line(`${above} = icmp sge i64 ${low}, ${-version.counterBound}`); + B.line(`${below} = icmp sle i64 ${high}, ${version.counterBound}`); + conditions.push(above, below); + } + for (const check of version.affine) { + let base = "0"; + for (const term of check.terms) { + const value = host.emitExpr(term.expr); + const integer = exactInteger(host, value, term.expr); + if (!integer) { + // Unreachable: every term has a proven range. Stay checked. + B.br(slow); + B.startBlock(B.newLabel("bytes.version.dead")); + return; + } + const wide = widenInteger(host, integer); + const sum = B.tmp(); + B.line(`${sum} = ${term.sign > 0 ? "add" : "sub"} i64 ${base}, ${wide}`); + base = sum; + } + // Products wrap only for a loop with no iteration: within the + // iteration space every scaled counter is a proven safe integer. + const scaledLow = scaleCounter(host, low, check.scale), + scaledHigh = scaleCounter(host, high, check.scale); + const first = B.tmp(), + final = B.tmp(), + nonNegative = B.tmp(), + below = B.tmp(), + both = B.tmp(); + B.line(`${first} = add i64 ${check.scale > 0 ? scaledLow : scaledHigh}, ${base}`); + B.line(`${final} = add i64 ${check.scale > 0 ? scaledHigh : scaledLow}, ${base}`); + const firstOffset = B.tmp(), + lastEnd = B.tmp(); + B.line(`${firstOffset} = add i64 ${first}, ${check.minOffset}`); + B.line(`${lastEnd} = add i64 ${final}, ${check.maxEnd}`); + B.line(`${nonNegative} = icmp sge i64 ${firstOffset}, 0`); + // Lengths are at most 2^53 elements: a signed comparison is exact. + B.line(`${below} = icmp slt i64 ${lastEnd}, ${lengths[check.receiver]}`); + B.line(`${both} = and i1 ${nonNegative}, ${below}`); + conditions.push(both); + } + } + let valid = "true"; + for (const cond of conditions) { + if (valid === "true") valid = cond; + else { + const all = B.tmp(); + B.line(`${all} = and i1 ${valid}, ${cond}`); + valid = all; + } + } + if (valid === "true") B.br(fast); + else B.condBr(valid, fast, slow); +} + +function scaleCounter(host: LlvmEmitterContext, value: string, scale: number): string { + if (scale === 1) return value; + const scaled = host.B.tmp(); + host.B.line(`${scaled} = mul i64 ${value}, ${scale}`); + return scaled; +} diff --git a/packages/compiler/src/backend/llvm/counted-loops.ts b/packages/compiler/src/backend/llvm/counted-loops.ts index 03e0ed4f3a..36fc63e1ff 100644 --- a/packages/compiler/src/backend/llvm/counted-loops.ts +++ b/packages/compiler/src/backend/llvm/counted-loops.ts @@ -42,3 +42,54 @@ export function emitCountedLoopLimit( B.line(`${limit} = add nsw i64 ${integer}, ${ascending ? 1 : -1}`); return limit; } + +/** The run-time test of a guarded counted loop: an unproven limit is within + * the integer form's finite range and an unproven start is an exact safe + * integer. Returns `true` when nothing needs testing. */ +export function emitCountedLoopSafety( + host: LlvmEmitterContext, + loop: IntegerCountedForLoop, +): string { + const B = host.B; + let safe = "true"; + if (loop.limitGuarded) { + const bound = host.emitExpr(loop.limit); + safe = B.tmp(); + B.line( + `${safe} = fcmp ${loop.step > 0 ? "ole" : "oge"} double ${bound.name}, ${f64Lit(loop.guardLimit)}`, + ); + } + if (loop.startGuarded) { + // An exact integer of at most 2^53 - 1 in magnitude, and not -0 + // (the bit patterns of the value and its round trip agree). + // NaN and infinities fail the ordered range tests. + const start = host.emitExpr(loop.start); + const low = B.tmp(), + high = B.tmp(), + raw = B.tmp(), + wide = B.tmp(), + back = B.tmp(), + bits = B.tmp(), + backBits = B.tmp(), + exact = B.tmp(), + inRange = B.tmp(), + ok = B.tmp(); + B.line(`${low} = fcmp oge double ${start.name}, ${f64Lit(-Number.MAX_SAFE_INTEGER)}`); + B.line(`${high} = fcmp ole double ${start.name}, ${f64Lit(Number.MAX_SAFE_INTEGER)}`); + B.line(`${inRange} = and i1 ${low}, ${high}`); + B.line(`${raw} = fptosi double ${start.name} to i64`); + B.line(`${wide} = freeze i64 ${raw}`); + B.line(`${back} = sitofp i64 ${wide} to double`); + B.line(`${bits} = bitcast double ${start.name} to i64`); + B.line(`${backBits} = bitcast double ${back} to i64`); + B.line(`${exact} = icmp eq i64 ${bits}, ${backBits}`); + B.line(`${ok} = and i1 ${inRange}, ${exact}`); + if (safe === "true") safe = ok; + else { + const both = B.tmp(); + B.line(`${both} = and i1 ${safe}, ${ok}`); + safe = both; + } + } + return safe; +} diff --git a/packages/compiler/src/backend/llvm/emitter.ts b/packages/compiler/src/backend/llvm/emitter.ts index 2cccda19ce..8c83638a53 100644 --- a/packages/compiler/src/backend/llvm/emitter.ts +++ b/packages/compiler/src/backend/llvm/emitter.ts @@ -54,8 +54,10 @@ import { matchIntegerArrayForLoop, matchIntegerBytesForLoop, matchIntegerCountedForLoop, + type IntegerCountedForLoop, } from "../../ir/integer-loops.js"; -import { emitCountedLoopLimit } from "./counted-loops.js"; +import { emitCountedLoopLimit, emitCountedLoopSafety } from "./counted-loops.js"; +import { emitBytesLoopVersionGuard, matchBytesLoopVersion } from "./bytes-loop-versions.js"; import { emitSafeIntegerCheck, emitSpeculativeInteger, @@ -66,6 +68,7 @@ import { } from "./speculative-integers.js"; import { emitLiteralSwitch } from "./switch-dispatch.js"; import { findBytesBounds } from "./bytes-bounds.js"; +import { globalSlotAlias } from "./bytes-alias.js"; import { emitByteWindowGuard, findInitializedByteLoopBindings, @@ -1065,6 +1068,14 @@ export class LlEmitter { wrappingSums = new Set(); private byteWindowsEnabled = true; private byteWindowEntries: ReturnType = new Map(); + /** Typed-array loop versioning (bytes-loop-versions.ts) for the current + * function: off in debug and suspending functions and inside a copy. */ + private bytesLoopVersionsEnabled = false; + private bytesLoopEntries: ReturnType | null = null; + private bytesLoopFunction: IrFunction | null = null; + /** Counted loops whose integer-form safety test an enclosing version + * guard already passed. */ + private readonly provenCountedGuards = new Set(); bytesBounds: ReadonlySet = new Set(); integerViews = new Map(); /** Module globals the current function owns outright (initializer @@ -6161,6 +6172,34 @@ export class LlEmitter { }; } + private matchBytesLoopVersion( + s: IrStmt & { kind: "for" }, + counted: IntegerCountedForLoop | null, + ): ReturnType { + const fn = this.bytesLoopFunction; + if (!fn) return null; + this.bytesLoopEntries ??= findInitializedByteLoopBindings(fn); + const initialized = this.bytesLoopEntries.get(s); + if (!initialized) return null; + const invariantGlobals = new Set(this.stableGlobals); + for (const global of this.initializerBindings.get(fn.name) ?? []) + invariantGlobals.add(global.id); + return matchBytesLoopVersion(s, counted, { + locals: this.currentLocals, + invariantGlobals, + captured: this.captureIds, + ranges: this.integerRanges, + initialized, + proven: this.bytesBounds, + }); + } + + /** The alias tag of a whole-slot load of a binding (module globals only). */ + slotAlias(id: string): string { + const binding = this.binding(id); + return binding.kind === "global" ? globalSlotAlias(this, id, binding.type.kind) : ""; + } + private emitDebugLocal(local: IrLocal, slot: string, arg = 0, captured = false): void { const debug = this.debug?.local(local, this.debugScope, arg, captured); if (!debug) return; @@ -6712,6 +6751,9 @@ export class LlEmitter { this.finalFieldBorrows = !fn.async && !fn.generator && constructorClass(fn.name) === null && this.finalFields.size > 0; this.byteWindowsEnabled = this.countedLoopsEnabled; + this.bytesLoopVersionsEnabled = this.countedLoopsEnabled; + this.bytesLoopEntries = null; + this.bytesLoopFunction = fn; this.byteWindowEntries = this.byteWindowsEnabled && numericFn.locals.some((l) => l.type.kind === "bytes") ? findInitializedByteLoopBindings(fn) @@ -8670,48 +8712,40 @@ export class LlEmitter { B.startBlock(done); break; } + const version = this.bytesLoopVersionsEnabled + ? this.matchBytesLoopVersion(s, countedLoop) + : null; + if (version) { + const fast = B.newLabel("bytes.version.fast"), + slow = B.newLabel("bytes.version.slow"), + done = B.newLabel("bytes.version.done"); + emitBytesLoopVersionGuard(this, version, fast, slow); + const originalBounds = this.bytesBounds, + originalRanges = this.integerRanges; + this.bytesLoopVersionsEnabled = false; + try { + B.startBlock(slow); + this.emitStmt(s); + B.br(done); + B.startBlock(fast); + this.bytesBounds = new Set([...originalBounds, ...version.bounds]); + if (version.ranges.size > 0) + this.integerRanges = new Map([...originalRanges, ...version.ranges]); + if (version.counted?.guarded) this.provenCountedGuards.add(s); + this.emitStmt(s); + B.br(done); + } finally { + this.bytesBounds = originalBounds; + this.integerRanges = originalRanges; + this.provenCountedGuards.delete(s); + this.bytesLoopVersionsEnabled = true; + } + B.startBlock(done); + break; + } let countedJoin: string | null = null; - if (countedLoop?.guarded) { - let safe = "true"; - if (countedLoop.limitGuarded) { - const bound = this.emitExpr(countedLoop.limit); - safe = B.tmp(); - B.line( - `${safe} = fcmp ${countedLoop.step > 0 ? "ole" : "oge"} double ${bound.name}, ${f64Lit(countedLoop.guardLimit)}`, - ); - } - if (countedLoop.startGuarded) { - // An exact integer of at most 2^53 - 1 in magnitude, and not -0 - // (the bit patterns of the value and its round trip agree). - // NaN and infinities fail the ordered range tests. - const start = this.emitExpr(countedLoop.start); - const low = B.tmp(), - high = B.tmp(), - raw = B.tmp(), - wide = B.tmp(), - back = B.tmp(), - bits = B.tmp(), - backBits = B.tmp(), - exact = B.tmp(), - inRange = B.tmp(), - ok = B.tmp(); - B.line(`${low} = fcmp oge double ${start.name}, ${f64Lit(-Number.MAX_SAFE_INTEGER)}`); - B.line(`${high} = fcmp ole double ${start.name}, ${f64Lit(Number.MAX_SAFE_INTEGER)}`); - B.line(`${inRange} = and i1 ${low}, ${high}`); - B.line(`${raw} = fptosi double ${start.name} to i64`); - B.line(`${wide} = freeze i64 ${raw}`); - B.line(`${back} = sitofp i64 ${wide} to double`); - B.line(`${bits} = bitcast double ${start.name} to i64`); - B.line(`${backBits} = bitcast double ${back} to i64`); - B.line(`${exact} = icmp eq i64 ${bits}, ${backBits}`); - B.line(`${ok} = and i1 ${inRange}, ${exact}`); - if (safe === "true") safe = ok; - else { - const both = B.tmp(); - B.line(`${both} = and i1 ${safe}, ${ok}`); - safe = both; - } - } + if (countedLoop?.guarded && !this.provenCountedGuards.has(s)) { + const safe = emitCountedLoopSafety(this, countedLoop); const fast = B.newLabel("counted.fast"); const slow = B.newLabel("counted.slow"); countedJoin = B.newLabel("counted.done"); @@ -8818,7 +8852,7 @@ export class LlEmitter { B.line( `${lenPtr} = getelementptr inbounds %${integerArrayLoop ? "ScrArr" : "ScrBytes"}, ptr ${receiver.name}, i64 0, i32 1`, ); - if (integerArrayLoop) this.markMemoryPointer(lenPtr, "array:header"); + this.markMemoryPointer(lenPtr, integerArrayLoop ? "array:header" : "bytes:header"); B.line( `${len} = load ${this.sizeType}, ptr ${lenPtr}${this.fieldAliasAttachment(lenPtr)}`, ); @@ -10309,7 +10343,7 @@ export class LlEmitter { } if (binding.kind === "global") this.checkGlobalTdz(e.localId); const value = this.B.tmp(); - this.B.line(`${value} = load ptr, ptr ${binding.slot}`); + this.B.line(`${value} = load ptr, ptr ${binding.slot}${this.slotAlias(e.localId)}`); return { name: value, type: e.type }; } if (e.kind === "dynCheck" && e.exact) return this.emitExactCapsuleObject(e.value, e.type); diff --git a/packages/compiler/src/backend/llvm/expr-bytes.ts b/packages/compiler/src/backend/llvm/expr-bytes.ts index ac025df099..485b7a6b33 100644 --- a/packages/compiler/src/backend/llvm/expr-bytes.ts +++ b/packages/compiler/src/backend/llvm/expr-bytes.ts @@ -8,6 +8,7 @@ import { exactInteger, widenInteger } from "./integer-values.js"; import { byteNumberAccess } from "../../ir/byte-numbers.js"; import { emitByteNumber } from "./byte-numbers.js"; import { integerOrNaNRange } from "./wrapping-sums.js"; +import { bytesElementAlias, bytesHeaderAlias } from "./bytes-alias.js"; const BYTES_NUM_KIND: Record = { u8: { kind: 0, le: false }, @@ -93,10 +94,7 @@ export function emitBytesIndex( B.line(`${narrow} = trunc i64 ${wide} to ${host.sizeType}`); return narrow; } - const lenPtr = B.tmp(); - const len = B.tmp(); - B.line(`${lenPtr} = getelementptr inbounds %ScrBytes, ptr ${receiver}, i64 0, i32 1`); - B.line(`${len} = load ${host.sizeType}, ptr ${lenPtr}`); + const len = emitBytesHeaderLength(host, receiver); if (integer) { const wideIndex = widenInteger(host, integer); const wideLength = host.sizeType === "i64" ? len : B.tmp(); @@ -145,11 +143,20 @@ export function emitBytesIndex( return idx; } +/** The element count of a typed array, in the target's size type. */ +export function emitBytesHeaderLength(host: LlvmEmitterContext, receiver: string): string { + const p = host.B.tmp(); + const len = host.B.tmp(); + host.B.line(`${p} = getelementptr inbounds %ScrBytes, ptr ${receiver}, i64 0, i32 1`); + host.B.line(`${len} = load ${host.sizeType}, ptr ${p}${bytesHeaderAlias(host)}`); + return len; +} + export function emitBytesData(host: LlvmEmitterContext, receiver: string): string { const p = host.B.tmp(); const data = host.B.tmp(); host.B.line(`${p} = getelementptr inbounds %ScrBytes, ptr ${receiver}, i64 0, i32 3`); - host.B.line(`${data} = load ptr, ptr ${p}`); + host.B.line(`${data} = load ptr, ptr ${p}${bytesHeaderAlias(host)}`); return data; } @@ -160,10 +167,7 @@ export function emitBytesLength( bytes: boolean, ): LlValue { const B = host.B; - const p = B.tmp(); - const len = B.tmp(); - B.line(`${p} = getelementptr inbounds %ScrBytes, ptr ${receiver}, i64 0, i32 1`); - B.line(`${len} = load ${host.sizeType}, ptr ${p}`); + const len = emitBytesHeaderLength(host, receiver); const size = BYTES_ELEMENT_SIZE[elem]; const count = bytes && size !== 1 ? B.tmp() : len; if (count !== len) B.line(`${count} = shl ${host.sizeType} ${len}, ${Math.log2(size)}`); @@ -187,7 +191,7 @@ function emitSharedBytesSplit(host: LlvmEmitterContext, receiver: string): strin isShared = B.tmp(), expected = B.tmp(); B.line(`${slot} = getelementptr inbounds %ScrBytes, ptr ${receiver}, i64 0, i32 9`); - B.line(`${storage} = load ptr, ptr ${slot}`); + B.line(`${storage} = load ptr, ptr ${slot}${bytesHeaderAlias(host)}`); B.line(`${isShared} = icmp ne ptr ${storage}, null`); B.line(`${expected} = call i1 @llvm.expect.i1(i1 ${isShared}, i1 false)`); const shared = B.newLabel("bytes.shared"), @@ -257,9 +261,15 @@ function emitBytesLoad( inBounds: boolean, invalid: string | undefined, ): LlValue { + if (host.mod.workers) { + const idx = emitBytesIndex(host, receiver, index, expr, invalid, inBounds); + return emitSharedOrLocalLoad(host, elem, receiver, idx); + } + // The header is immutable, so the data pointer loads beside the length, + // ahead of the bounds branch: an unconditional load LICM can hoist. + const data = host.emitBytesData(receiver); const idx = emitBytesIndex(host, receiver, index, expr, invalid, inBounds); - if (host.mod.workers) return emitSharedOrLocalLoad(host, elem, receiver, idx); - return emitBytesLoadAt(host, elem, receiver, idx); + return emitBytesLoadAt(host, elem, data, idx); } /** Worker programs: the element at a checked index of a view that may be @@ -285,7 +295,7 @@ function emitSharedOrLocalLoad( const sharedDone = B.newLabel("bytes.shared.done"); B.br(sharedDone); B.startBlock(local); - const own = emitBytesLoadAt(host, elem, receiver, idx); + const own = emitBytesLoadAt(host, elem, host.emitBytesData(receiver), idx); const localDone = B.newLabel("bytes.local.done"); B.br(localDone); B.startBlock(localDone); @@ -306,15 +316,15 @@ function emitSharedOrLocalLoad( return { name: out, type: F64, uint32: u32 }; } -/** The element at a checked index of a thread-owned view. */ +/** The element at a checked index of a thread-owned view, given its data + * pointer. */ function emitBytesLoadAt( host: LlvmEmitterContext, elem: IrBytesElem, - receiver: string, + data: string, idx: string, ): LlValue { const B = host.B; - const data = host.emitBytesData(receiver); const p = B.tmp(); if (elem === "f16") { // Half-precision bits widen exactly; the runtime converts them. @@ -322,7 +332,7 @@ function emitBytesLoadAt( const raw = B.tmp(); const out = B.tmp(); B.line(`${p} = getelementptr inbounds i16, ptr ${data}, ${host.sizeType} ${idx}`); - B.line(`${raw} = load i16, ptr ${p}, align 1`); + B.line(`${raw} = load i16, ptr ${p}, align 1${bytesElementAlias(host)}`); B.line(`${out} = call double @scr_f16_to_f64(i16 zeroext ${raw})`); return { name: out, type: F64 }; } @@ -333,7 +343,7 @@ function emitBytesLoadAt( const wide = B.tmp(); const out = B.tmp(); B.line(`${p} = getelementptr inbounds i${bits}, ptr ${data}, ${host.sizeType} ${idx}`); - B.line(`${raw} = load i${bits}, ptr ${p}, align 1`); + B.line(`${raw} = load i${bits}, ptr ${p}, align 1${bytesElementAlias(host)}`); B.line(`${wide} = ${signed ? "sext" : "zext"} i${bits} ${raw} to i32`); B.line(`${out} = ${signed ? "sitofp" : "uitofp"} i32 ${wide} to double`); return { name: out, type: F64, uint32: wide }; @@ -342,20 +352,20 @@ function emitBytesLoadAt( const raw = B.tmp(); const out = B.tmp(); B.line(`${p} = getelementptr inbounds float, ptr ${data}, ${host.sizeType} ${idx}`); - B.line(`${raw} = load float, ptr ${p}, align 1`); + B.line(`${raw} = load float, ptr ${p}, align 1${bytesElementAlias(host)}`); B.line(`${out} = fpext float ${raw} to double`); return { name: out, type: F64 }; } if (elem === "f64") { const out = B.tmp(); B.line(`${p} = getelementptr inbounds double, ptr ${data}, ${host.sizeType} ${idx}`); - B.line(`${out} = load double, ptr ${p}, align 1`); + B.line(`${out} = load double, ptr ${p}, align 1${bytesElementAlias(host)}`); return { name: out, type: F64 }; } const raw = B.tmp(); const out = B.tmp(); B.line(`${p} = getelementptr inbounds i32, ptr ${data}, ${host.sizeType} ${idx}`); - B.line(`${raw} = load i32, ptr ${p}, align 1`); + B.line(`${raw} = load i32, ptr ${p}, align 1${bytesElementAlias(host)}`); B.line(`${out} = ${elem === "i32" ? "sitofp" : "uitofp"} i32 ${raw} to double`); return { name: out, type: F64, uint32: raw }; } @@ -465,6 +475,7 @@ export function emitBytesSet( const B = host.B; // Invalid integer-indexed writes are ignored after evaluating the RHS. const done = B.newLabel("bytes.store.done"); + const data = host.emitBytesData(receiver); const idx = emitBytesIndex(host, receiver, index, indexExpr, done, inBounds); if (host.mod.workers) { host.declare(`declare void @scr_bytes_shared_set(ptr, ${host.sizeType}, double)`); @@ -487,7 +498,6 @@ export function emitBytesSet( elem === "f16" || elem === "f32" || elem === "f64" ? null : host.emitToUint32(value.name, expr, value.uint32); - const data = host.emitBytesData(receiver); const p = B.tmp(); if (elem === "f16") { // One round-to-nearest-even step from the double, in the runtime. @@ -495,24 +505,24 @@ export function emitBytesSet( const narrowed = B.tmp(); B.line(`${narrowed} = call zeroext i16 @scr_f16_from_f64(double ${value.name})`); B.line(`${p} = getelementptr inbounds i16, ptr ${data}, ${host.sizeType} ${idx}`); - B.line(`store i16 ${narrowed}, ptr ${p}, align 1`); + B.line(`store i16 ${narrowed}, ptr ${p}, align 1${bytesElementAlias(host)}`); } else if (BYTES_ELEMENT_SIZE[elem] < 4) { const bits = BYTES_ELEMENT_SIZE[elem] * 8; const byte = B.tmp(); B.line(`${byte} = trunc i32 ${stored!} to i${bits}`); B.line(`${p} = getelementptr inbounds i${bits}, ptr ${data}, ${host.sizeType} ${idx}`); - B.line(`store i${bits} ${byte}, ptr ${p}, align 1`); + B.line(`store i${bits} ${byte}, ptr ${p}, align 1${bytesElementAlias(host)}`); } else if (elem === "f32") { const narrowed = B.tmp(); B.line(`${narrowed} = fptrunc double ${value.name} to float`); B.line(`${p} = getelementptr inbounds float, ptr ${data}, ${host.sizeType} ${idx}`); - B.line(`store float ${narrowed}, ptr ${p}, align 1`); + B.line(`store float ${narrowed}, ptr ${p}, align 1${bytesElementAlias(host)}`); } else if (elem === "f64") { B.line(`${p} = getelementptr inbounds double, ptr ${data}, ${host.sizeType} ${idx}`); - B.line(`store double ${value.name}, ptr ${p}, align 1`); + B.line(`store double ${value.name}, ptr ${p}, align 1${bytesElementAlias(host)}`); } else { B.line(`${p} = getelementptr inbounds i32, ptr ${data}, ${host.sizeType} ${idx}`); - B.line(`store i32 ${stored!}, ptr ${p}, align 1`); + B.line(`store i32 ${stored!}, ptr ${p}, align 1${bytesElementAlias(host)}`); } B.br(done); B.startBlock(done); diff --git a/packages/compiler/src/backend/llvm/expr-containers.ts b/packages/compiler/src/backend/llvm/expr-containers.ts index 9c77690f8b..470477d44e 100644 --- a/packages/compiler/src/backend/llvm/expr-containers.ts +++ b/packages/compiler/src/backend/llvm/expr-containers.ts @@ -526,7 +526,7 @@ export function emitStableReceiver( if (b.kind !== "boxed") { if (b.kind === "global") host.checkGlobalTdz(receiver.localId); const value = host.B.tmp(); - host.B.line(`${value} = load ptr, ptr ${b.slot}`); + host.B.line(`${value} = load ptr, ptr ${b.slot}${host.slotAlias(receiver.localId)}`); return { name: value, type: receiver.type }; } } diff --git a/packages/compiler/src/backend/llvm/expr-primitives.ts b/packages/compiler/src/backend/llvm/expr-primitives.ts index aaae4c35cc..7ad649f2d8 100644 --- a/packages/compiler/src/backend/llvm/expr-primitives.ts +++ b/packages/compiler/src/backend/llvm/expr-primitives.ts @@ -107,7 +107,7 @@ export function emitLiteralExpr( return host.own({ name: v, type: e.type }); } const t = B.tmp(); - B.line(`${t} = load ${host.llType(b.type)}, ptr ${b.slot}`); + B.line(`${t} = load ${host.llType(b.type)}, ptr ${b.slot}${host.slotAlias(e.localId)}`); const integerView = host.integerViews.get(e.localId); if (integerView !== undefined) { // Snapshot both representations before later operands can write diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5467-typed-array-loop-versions.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5467-typed-array-loop-versions.ts.json new file mode 100644 index 0000000000..a4be596965 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5467-typed-array-loop-versions.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/5467-typed-array-loop-versions.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5468-typed-array-loop-aliasing.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5468-typed-array-loop-aliasing.ts.json new file mode 100644 index 0000000000..517cbab4c6 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5468-typed-array-loop-aliasing.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/5468-typed-array-loop-aliasing.ts" + ], + "diags": [] +} diff --git a/tests/corpus/5467-typed-array-loop-versions.ts b/tests/corpus/5467-typed-array-loop-versions.ts new file mode 100644 index 0000000000..62848ae432 --- /dev/null +++ b/tests/corpus/5467-typed-array-loop-versions.ts @@ -0,0 +1,238 @@ +// Typed-array loops whose accesses are proved in bounds by one test before +// the loop run without per-element checks; every other entry runs the +// checked loop. Both must keep JavaScript's exact out-of-bounds behavior: +// reads of missing elements are undefined (NaN in arithmetic), writes are +// ignored, DataView accesses throw at the failing iteration, and +// fractional, negative and NaN indices never touch storage. + +const table: Int32Array = new Int32Array(256); +for (let n = 0; n < 256; n++) table[n] = (n * 2654435761) | 0; +const shortTable: Int32Array = new Int32Array(100); +for (let n = 0; n < 100; n++) shortTable[n] = n + 1; + +function masked(t: Int32Array, data: Uint8Array): number { + let h = 0; + for (let i = 0; i < data.length; i++) h = (h ^ t[(h ^ data[i]) & 0xff]) | 0; + return h; +} + +function maskedWrite(t: Int32Array, count: number): void { + for (let i = 0; i < count; i++) t[(i * 7) & 0xff] = i; +} + +const data = new Uint8Array(64); +for (let i = 0; i < data.length; i++) data[i] = (i * 37 + 11) & 0xff; +console.log("masked full", masked(table, data)); +console.log("masked short", masked(shortTable, data)); +const writable = new Int32Array(200); +maskedWrite(writable, 300); +console.log("masked write", writable[0], writable[7], writable[199], writable.length); + +// Counter plus an invariant base, in bounds and running off either end. +function shifted(src: Float64Array, out: Float64Array, base: number, n: number): number { + let sum = 0; + for (let i = 0; i < n; i++) { + out[base + i] = src[i] * 2; + sum += src[base + i - 1]; + } + return sum; +} + +const src = new Float64Array(8); +for (let i = 0; i < 8; i++) src[i] = i + 0.25; +for (const [base, n] of [ + [1, 4], + [0, 8], + [5, 8], + [-3, 6], + [2, 0], + [3, 2.5], +]) { + const out = new Float64Array(8); + const sum = shifted(src, out, base, n); + console.log("shifted", base, n, sum, Array.from(out).join(",")); +} + +// Descending loops, including ones that walk below zero. +function descending(a: Uint16Array, low: number): number { + let sum = 0; + for (let i = a.length - 1; i >= low; i--) { + a[i] = a[i] + 1000; + sum += a[i]; + } + return sum; +} +const u16 = new Uint16Array([65000, 1, 2, 3]); +console.log("descending", descending(u16, 0), Array.from(u16).join(",")); +console.log("descending below", descending(u16, -2), Array.from(u16).join(",")); + +// Fractional starts and offsets never name an element. +function fractional(a: Float32Array): string { + const seen: string[] = []; + for (let i = 0.5; i < 3; i++) seen.push(String(a[i])); + for (let i = 0; i < 3; i++) { + a[i + 0.5] = 99; + seen.push(String(a[i] + a[i - 0.5])); + } + return seen.join(" "); +} +console.log("fractional", fractional(new Float32Array([1.5, 2.5, 3.5]))); + +function nanIndex(a: Int8Array, offset: number): number { + let sum = 0; + for (let i = 0; i < a.length; i++) { + a[i + offset] = 120; + sum += a[i + offset] | 0; + } + return sum; +} +const i8 = new Int8Array([1, 2, 3]); +console.log("nan index", nanIndex(i8, NaN), nanIndex(i8, -0), Array.from(i8).join(",")); + +// Unproven limits keep the integer loop's guard: huge, infinite and NaN. +function limits(a: Uint8Array, limit: number): number { + let sum = 0; + for (let i = 0; i < limit; i++) { + if (i >= a.length + 2) break; + sum += a[i] | 0; + a[i] = i; + } + return sum; +} +const lim = new Uint8Array([5, 6, 7]); +console.log("limits", limits(lim, 2), limits(lim, 1e300), limits(lim, Infinity), limits(lim, NaN)); + +// A DataView access past the end throws after the earlier iterations ran. +const viewProgress: number[] = [0]; +function viewLoop(view: DataView, n: number): number { + let sum = 0; + for (let i = 0; i < n; i++) { + sum += view.getUint32(i * 4, true) + view.getUint8(i * 4 + 3); + viewProgress[0] = sum; + } + return sum; +} +function viewSum(view: DataView, n: number): number { + viewProgress[0] = -1; + try { + return viewLoop(view, n); + } catch (e) { + console.log("view throw", (e as Error).name, viewProgress[0]); + } + return viewProgress[0]; +} +const dvBytes = new Uint8Array(16); +for (let i = 0; i < 16; i++) dvBytes[i] = i * 3; +const view = new DataView(dvBytes.buffer); +console.log("view ok", viewSum(view, 4)); +console.log("view partial", viewSum(view, 6)); + +// Strided loops: the last counter value of a non-unit step is exact, so a +// record that ends at the last element still takes the unchecked loop and +// a partial trailing record does not. +function quads(words: Uint16Array): number { + let sum = 0; + for (let i = 0; i < words.length; i += 4) sum += words[i] + words[i + 3] * 2; + return sum; +} +console.log("quads", quads(new Uint16Array([1, 2, 3, 4, 5, 6, 7, 8]))); +console.log("quads partial", quads(new Uint16Array([1, 2, 3, 4, 5, 6]))); +function backTriples(words: Uint16Array): number { + let sum = 0; + for (let i = words.length - 1; i >= 0; i -= 3) sum = sum * 3 + words[i] + (words[i - 2] | 0); + return sum; +} +console.log("back triples", backTriples(new Uint16Array([1, 2, 3, 4, 5, 6]))); +console.log("back triples partial", backTriples(new Uint16Array([1, 2, 3, 4, 5, 6, 7]))); +function scaled(words: Uint16Array, n: number): number { + let sum = 0; + for (let i = 0; i < n; i++) sum += words[i * 2] - (words[(i + 1) * 2 - 1] | 0) + (words[9 - i] | 0); + return sum; +} +const scaledWords = new Uint16Array(10); +for (let i = 0; i < 10; i++) scaledWords[i] = i * i; +console.log("scaled", scaled(scaledWords, 5), scaled(scaledWords, 6), scaled(scaledWords, 11)); +// A negative zero index names element 0 and stays -0 as a number. +function negativeZero(words: Uint16Array, n: number): string { + const seen: string[] = []; + for (let i = 0; i < n; i++) { + const k = i * -1; + const m = (i - 1) * 2; + seen.push(`${words[k]}:${Object.is(k, -0)}:${words[m] | 0}:${Object.is(m, -0)}`); + } + return seen.join(" "); +} +console.log("negative zero", negativeZero(scaledWords, 3)); + +// Clamped and wrapping element types on the unchecked path. +function convert(c: Uint8ClampedArray, s: Int16Array): string { + for (let i = 0; i < 4; i++) { + c[i] = i * 100 - 50.5; + s[i] = i * 20000; + } + return Array.from(c).join(",") + " " + Array.from(s).join(","); +} +console.log("convert", convert(new Uint8ClampedArray(4), new Int16Array(4))); + +// Module receivers stay correct across calls in the loop body, including a +// call that writes the same arrays. +const left: Float64Array = new Float64Array(16); +const right: Float64Array = new Float64Array(16); +function poke(k: number): number { + right[k] = right[k] + 1; + return k; +} +for (let i = 0; i < 16; i++) left[i] = i; +let crossed = 0; +for (let i = 0; i < 16; i++) { + right[i] = left[i] * 3; + crossed += right[poke(i)]; +} +console.log("module receivers", crossed, right[0], right[15]); + +// A body that reassigns its receiver keeps reading the current binding. +function rebinding(first: Uint8Array, second: Uint8Array): number { + let current = first; + let sum = 0; + for (let i = 0; i < 4; i++) { + sum += current[i] | 0; + if (i === 1) current = second; + } + return sum; +} +console.log("rebinding", rebinding(new Uint8Array([1, 2, 3, 4]), new Uint8Array([10, 20]))); + +// A throwing body leaves the writes of earlier iterations in place. +function failing(a: Uint32Array, at: number): void { + for (let i = 0; i < a.length; i++) { + if (i === at) throw new Error("stop at " + i); + a[i] = 4294967295 - i; + } +} +const u32 = new Uint32Array(5); +try { + failing(u32, 3); +} catch (e) { + console.log((e as Error).message, Array.from(u32).join(",")); +} + +// A matrix product over module-level constant arrays. +const N = 6; +const ma: Float64Array = new Float64Array(N * N); +const mb: Float64Array = new Float64Array(N * N); +const mc: Float64Array = new Float64Array(N * N); +for (let i = 0; i < N * N; i++) { + ma[i] = (i % 7) - 3; + mb[i] = (i % 5) * 0.5; +} +for (let i = 0; i < N; i++) { + for (let k = 0; k < N; k++) { + const aik = ma[i * N + k]; + const kOff = k * N; + const iOff = i * N; + for (let j = 0; j < N; j++) mc[iOff + j] += aik * mb[kOff + j]; + } +} +let trace = 0; +for (let i = 0; i < N; i++) trace += mc[i * N + i]; +console.log("matmul", trace, mc[1], mc[N * N - 1]); diff --git a/tests/corpus/5468-typed-array-loop-aliasing.ts b/tests/corpus/5468-typed-array-loop-aliasing.ts new file mode 100644 index 0000000000..befcc8227b --- /dev/null +++ b/tests/corpus/5468-typed-array-loop-aliasing.ts @@ -0,0 +1,87 @@ +// Element stores may alias any other view of the same buffer, whatever its +// element type, offset or length, but never a typed array's own length and +// data pointer. Loops that write through one view and read through another +// must observe every store in program order, also when the loop runs +// without per-element bounds checks. + +const buffer = new ArrayBuffer(64); +const f64 = new Float64Array(buffer); +const u8 = new Uint8Array(buffer); +const i32 = new Int32Array(buffer, 8, 8); + +// Writes through Float64Array observed through a Uint8Array in the same +// iteration. +let topBytes = 0; +for (let i = 0; i < f64.length; i++) { + f64[i] = -(i + 1) * 1.5; + topBytes += u8[i * 8 + 7]; +} +console.log("f64 then u8", topBytes, u8[7], u8[63]); + +// Byte stores observed through an overlapping Int32Array window. +let words = 0; +for (let i = 0; i < 32; i++) { + u8[8 + i] = i + 1; + words = (words + i32[i >> 2]) | 0; +} +console.log("u8 then i32", words, i32[0], i32[7]); + +// A shifted subarray reads what the previous iteration wrote: a prefix +// recurrence that any reordering or vector widening would break. +const chain = new Uint8Array(40); +chain[0] = 1; +const ahead = chain.subarray(1); +for (let i = 0; i < ahead.length; i++) ahead[i] = chain[i] + 2; +console.log("recurrence", chain[0], chain[1], chain[20], chain[39]); + +// Overlapping copies in both directions. +const values = new Int16Array(12); +for (let i = 0; i < values.length; i++) values[i] = i * 10; +const lower = values.subarray(0, 11); +const upper = values.subarray(1); +for (let i = 0; i < lower.length; i++) lower[i] = upper[i]; +console.log("copy down", Array.from(values).join(",")); +for (let i = 0; i < values.length; i++) values[i] = i * 10; +for (let i = 0; i < lower.length; i++) upper[i] = lower[i]; +console.log("copy up", Array.from(values).join(",")); + +// Same-array accumulation with a lagging read. +function lagged(a: Float64Array, lag: number): number { + for (let i = lag; i < a.length; i++) a[i] = a[i] + a[i - lag]; + return a[a.length - 1]; +} +const fib = new Float64Array(30); +fib[0] = 1; +fib[1] = 1; +console.log("lagged", lagged(fib, 1), lagged(fib, 2)); + +// Two parameters that are the same array, and a destination that aliases +// part of its own source. +function axpy(alpha: number, x: Float64Array, y: Float64Array, n: number): void { + for (let i = 0; i < n; i++) y[i] = y[i] + alpha * x[i]; +} +const same = new Float64Array([1, 2, 3, 4, 5, 6, 7, 8]); +axpy(2, same, same, same.length); +console.log("axpy same", Array.from(same).join(",")); +const whole = new Float64Array([1, 2, 3, 4, 5, 6, 7, 8]); +axpy(1, whole, whole.subarray(2), 6); +console.log("axpy shifted", Array.from(whole).join(",")); + +// Element stores never change a length or a module binding, even when a +// float store writes the bit pattern of a small length. +const meta: Float64Array = new Float64Array(4); +const lengths: number[] = []; +for (let i = 0; i < 4; i++) { + meta[i] = 5e-324 * (i + 1); + lengths.push(meta.length); +} +console.log("lengths", lengths.join(","), meta[3] > 0); + +// Global numbers next to typed-array stores keep their values. +let scale = 3; +const scaled: Float32Array = new Float32Array(6); +for (let i = 0; i < scaled.length; i++) { + scaled[i] = scale * i; + if (i === 2) scale = 0.5; +} +console.log("scaled", Array.from(scaled).join(","));