Keep int32 slots for long number arrays with checked integer stores

A local number[] candidate whose stores are exact integers but not all
proven int32s (a distance array updated with dist[v] = dist[u]! + 1), and
that is only ever allocated by new Array(n).fill(v), now holds SCR_ELEM_I32
slots when n is at least 4096. A shorter fill creates doubles, which fit a
typical L1 data cache. The first store of a value that is not an int32
widens the array to doubles in place (scr_arr_widen_i32).

An innermost loop that accesses such arrays tests their kinds once on
entry (backend/llvm/int32-kind-loops.ts). Its fast copy reads and writes
them as int32 arrays; a store that would widen one leaves for a copy that
re-executes it with kind tests and finishes the loop on doubles. When the
arrays already hold doubles, a copy accesses them as double arrays. Other
accesses test the kind each time.

Only these arrays take kind tests: every other array, and every program
without one, emits the same code as before.
This commit is contained in:
Chris Tate
2026-10-22 21:37:12 -05:00
parent 53dc4df990
commit fe406429eb
19 changed files with 1108 additions and 71 deletions
@@ -542,7 +542,9 @@ export function emitDenseArrayState(
* value's reference. A dense index stores the slot, marks it present, and
* extends the length when it lands at or past it; a replaced present
* reference is released after the new edge is published (a release can
* collect cycles). Every other index takes the unchanged runtime call. */
* collect cycles). Every other index takes the unchanged runtime call.
* `int32`, for an int32 array, is the value as an i32 when the caller has
* it at hand. */
export function emitDenseArraySet(
host: LlvmEmitterContext,
array: string,
@@ -552,6 +554,7 @@ export function emitDenseArraySet(
element: IrType,
value: string,
proven = false,
int32: string | null = null,
): void {
const B = host.B;
const ty = valueType(acc);
@@ -566,8 +569,8 @@ export function emitDenseArraySet(
}
// An int32 array's stores are proven int32s (ir/int32-arrays.ts); both
// dense paths below store the converted value.
let int32Value: string | null = null;
if (acc === "i32") {
let int32Value: string | null = int32;
if (acc === "i32" && int32Value === null) {
int32Value = B.tmp();
B.line(`${int32Value} = fptosi double ${value} to i32`);
}
@@ -139,6 +139,8 @@ export function emitFunction(emitter: LlEmitter, fn: IrFunction): string {
emitter.speculativeFunction = fn.name;
emitter.speculation = null;
emitter.speculationResume = null;
emitter.int32Kinds = null;
emitter.int32KindResume = null;
emitter.finalFieldBorrows =
!fn.async &&
!fn.generator &&
@@ -34,9 +34,16 @@ import { dynHoleRead } from "./loop-array-borrows.js";
import { endsWithJump, matchStringSelfConcat } from "../../ir/analysis.js";
import { charAtUnitRead, emitDiscardedSplice } from "./expr-containers.js";
import { InternalCompilerError } from "../../errors.js";
import { emitDenseArrayGet, emitDenseArrayPop, emitDenseArraySet } from "./dense-array-access.js";
import {
emitDenseArrayGet,
emitDenseArrayPop,
emitDenseArraySet,
SCR_ELEM_I32,
} from "./dense-array-access.js";
import { emitBorrowedInput } from "./borrowed-inputs.js";
import { f64Lit, llvmCommentText, loadArrCount, sizeToDouble } from "./common.js";
import { f64Lit, llvmCommentText, loadArrCount, SCR_ARR_FIELD, sizeToDouble } from "./common.js";
import { planInt32KindLoop, type Int32KindLoopPlan } from "./int32-kind-loops.js";
import { withinInt32 } from "../../ir/integer-ranges.js";
import { resultSlotStore } from "../result-slots.js";
import {
matchIntegerArrayForLoop,
@@ -411,7 +418,21 @@ export function emitStmtBody(emitter: LlEmitter, s: IrStmt): void {
const v = emitter.emitExpr(s.value);
if (s.arr.type.kind !== "array")
throw new InternalCompilerError("llvm emitter bug: arraySet on non-array");
if (emitter.checkedInt32Array(s.arr)) {
// An int32 array with checked stores: the int32 arm stores an int32
// value, and widens the array for any other first.
emitter.emitPublishedArrayGuard(arr.name, s.arr.type, 0, idx.name);
const elem = s.arr.type.elem;
const proven = emitter.provenDenseIndex(s.arr, s.index);
emitter.emitByElementKind(arr.name, null, (acc, widened) => {
if (acc === "i32") emitter.emitCheckedInt32Store(arr.name, v.name, s.value, widened);
emitDenseArraySet(emitter, arr.name, idx, s.index, acc, elem, v.name, proven);
return null;
});
break;
}
const acc = emitter.arrayAccess(s.arr, s.arr.type.elem);
const int32 = emitter.emitInt32KindStoreCheck(s.arr, v, s.value);
emitter.emitPublishedArrayGuard(arr.name, s.arr.type, 0, idx.name);
if (acc === "ref") emitter.toHeap(v);
emitDenseArraySet(
@@ -423,6 +444,7 @@ export function emitStmtBody(emitter: LlEmitter, s: IrStmt): void {
s.arr.type.elem,
v.name,
emitter.provenDenseIndex(s.arr, s.index),
int32,
);
break;
}
@@ -478,12 +500,19 @@ export function emitStmtBody(emitter: LlEmitter, s: IrStmt): void {
B.condBr(dropsLast, pop, call);
B.startBlock(pop);
const elem = s.arr.type.elem;
const acc = emitter.arrayAccess(s.arr, elem);
const raw = emitDenseArrayPop(emitter, arr.name, acc, elem, call, call);
if (acc === "ref" && isRefCounted(elem)) {
const old = B.tmp();
B.line(`${old} = inttoptr i64 ${raw} to ptr`);
emitter.releaseHeap(old, elem);
if (emitter.checkedInt32Array(s.arr)) {
emitter.emitByElementKind(arr.name, null, (acc) => {
emitDenseArrayPop(emitter, arr.name, acc, elem, call, call);
return null;
});
} else {
const acc = emitter.arrayAccess(s.arr, elem);
const raw = emitDenseArrayPop(emitter, arr.name, acc, elem, call, call);
if (acc === "ref" && isRefCounted(elem)) {
const old = B.tmp();
B.line(`${old} = inttoptr i64 ${raw} to ptr`);
emitter.releaseHeap(old, elem);
}
}
B.br(done);
B.startBlock(call);
@@ -2098,9 +2127,26 @@ export function emitLoopCopies(
)
: null;
if (!plan) {
const proven = allowed ? planProvenLoop(emitter, loop) : null;
if (proven) emitProvenLoop(emitter, proven, exit, emitCopy);
else emitCopy(exit);
const ordinary = (exit: string): void => {
const proven = allowed ? planProvenLoop(emitter, loop) : null;
if (proven) emitProvenLoop(emitter, proven, exit, emitCopy);
else emitCopy(exit);
};
const kinds =
allowed &&
emitter.speculativeLoopsEnabled &&
emitter.int32Slots.checkedInt32Arrays.size > 0 &&
emitter.int32Kinds === null &&
emitter.speculation === null &&
emitter.speculationResume === null
? planInt32KindLoop(
loop,
(id) => emitter.checkedInt32Local(id),
(e) => withinInt32(emitter.integerRanges.get(e)),
)
: null;
if (kinds) emitInt32KindLoop(emitter, kinds, exit, kinds.writes.size > 0 ? emitCopy : ordinary);
else ordinary(exit);
return;
}
const B = emitter.B;
@@ -2243,6 +2289,124 @@ export function emitLoopCopies(
}
}
/** An int32 kind loop (int32-kind-loops.ts). The entry test reads the kind
* of each of the plan's arrays: when every one holds int32 slots the fast
* copy runs, when every one holds doubles the wide copy, which reads and
* writes them as double arrays, and otherwise (a plan with several arrays)
* the mixed copy, which tests their kinds at each access. A failed store of
* the fast copy re-executes in the third copy, which finishes its
* iteration with kind tests and then enters the mixed copy, or the wide
* copy when the plan has one array (it just widened). Each copy is emitted
* by `ordinary`: the loop's plain copy when a store may fail, since a
* proven integer loop keeps locals in shadows of its own copy that another
* copy cannot resume from. */
function emitInt32KindLoop(
emitter: LlEmitter,
plan: Int32KindLoopPlan,
exit: string,
ordinary: (exit: string) => void,
): void {
const B = emitter.B;
const fn = emitter.currentFunctionName;
const keys = new Set([...plan.arrays.keys()].map((id) => `${fn}\0${id}`));
const several = keys.size > 1;
let allInt32 = "true",
allWide = "true";
const and = (all: string, next: string): string => {
if (all === "true") return next;
const both = B.tmp();
B.line(`${both} = and i1 ${all}, ${next}`);
return both;
};
for (const ref of plan.arrays.values()) {
const arr = emitter.emitStableReceiver(ref, []);
const kindPtr = B.tmp(),
kind = B.tmp(),
isInt32 = B.tmp();
B.line(
`${kindPtr} = getelementptr inbounds %ScrArr, ptr ${arr.name}, i32 0, i32 ${SCR_ARR_FIELD.elem}`,
);
emitter.markMemoryPointer(kindPtr, "array:header");
B.line(`${kind} = load i8, ptr ${kindPtr}${emitter.fieldAliasAttachment(kindPtr)}`);
B.line(`${isInt32} = icmp eq i8 ${kind}, ${SCR_ELEM_I32}`);
allInt32 = and(allInt32, isInt32);
if (several) {
const isWide = B.tmp();
B.line(`${isWide} = xor i1 ${isInt32}, true`);
allWide = and(allWide, isWide);
}
}
const fast = B.newLabel("kind.fast"),
wide = B.newLabel("kind.wide"),
mixed = several ? B.newLabel("kind.mixed") : null;
if (mixed === null) B.condBr(allInt32, fast, wide);
else {
const other = B.newLabel("kind.other");
B.condBr(allInt32, fast, other);
B.startBlock(other);
B.condBr(allWide, wide, mixed);
}
const resume = new Map<IrStmt, { label: string; placed: boolean }>();
for (const s of plan.writes) resume.set(s, { label: B.newLabel("kind.resume"), placed: false });
// The header the third copy enters: the mixed copy's, or the wide copy's.
let header: string | null = null;
const capture = (label: string): void => {
header = label;
};
B.startBlock(wide);
if (resume.size > 0 && mixed === null) emitter.speculationHeader = capture;
emitter.int32Kinds = { int32: new Set(), wide: keys, resume: new Map() };
try {
ordinary(exit);
} finally {
emitter.speculationHeader = null;
emitter.int32Kinds = null;
}
if (mixed !== null) {
B.startBlock(mixed);
if (resume.size > 0) emitter.speculationHeader = capture;
try {
ordinary(exit);
} finally {
emitter.speculationHeader = null;
}
}
if (resume.size > 0) {
if (header === null)
throw new InternalCompilerError("llvm emitter bug: int32 kind loop copy without a header");
// A failed store finishes its iteration here, as in a speculative
// integer loop's third copy.
B.startBlock(B.newLabel("kind.tail"));
emitter.speculationResume = resume;
emitter.speculationHeader = () => {
B.br(header!);
B.startBlock(B.newLabel("kind.unused"));
};
try {
ordinary(exit);
} finally {
emitter.speculationResume = null;
emitter.speculationHeader = null;
}
}
B.startBlock(fast);
if (![...resume.values()].every((r) => r.placed)) {
// A write the third copy never reached: this copy tests the kinds.
ordinary(exit);
return;
}
emitter.int32Kinds = {
int32: keys,
wide: new Set(),
resume: new Map([...resume].map(([s, r]) => [s, r.label])),
};
try {
ordinary(exit);
} finally {
emitter.int32Kinds = null;
}
}
/** At the top of a loop copy's header: lets a speculative loop find its
* ordinary copy's header, and keeps its third copy out of the loop. */
function enterLoopHeader(emitter: LlEmitter, header: string): void {
+224 -22
View File
@@ -79,9 +79,18 @@ import {
} from "./emit-journal.js";
import { specializeNumericCalls } from "../../ir/numeric-call-specialization.js";
import { versionLoopBoundParameters } from "../../ir/parameter-versions.js";
import { allocatedLiteral, selectInt32Arrays } from "../../ir/int32-arrays.js";
import {
allocatedLiteral,
fillAllocationLength,
selectInt32Arrays,
} from "../../ir/int32-arrays.js";
import { everyExpr, everyExprChild, everyStmtChild, everyStmtList } from "../../ir/traverse.js";
import { INT32_RANGE, type IntegerRange, type IntegerRanges } from "../../ir/integer-ranges.js";
import {
INT32_RANGE,
withinInt32,
type IntegerRange,
type IntegerRanges,
} from "../../ir/integer-ranges.js";
import { unshareNodes } from "../../ir/unshare.js";
import { copyStableCaptures } from "../../ir/capture-values.js";
import { resolveSelfCaptures } from "../../ir/self-captures.js";
@@ -101,7 +110,7 @@ import {
} from "./int32-abi.js";
import { type Int32Local } from "./int32-locals.js";
import { findIntegerClones, NO_INTEGER_CLONES, type IntegerClones } from "./integer-clones.js";
import { exactInteger, narrowInteger } from "./integer-values.js";
import { exactInteger, narrowInteger, widenInteger } from "./integer-values.js";
import { findInitializerBindings } from "../../ir/initializer-bindings.js";
import { findConstantNumericTables, type ConstantNumericTable } from "../../ir/constant-tables.js";
import {
@@ -148,7 +157,12 @@ import {
} from "./inline-unions.js";
import { OptionalArrayReads, type LocalArrayRead } from "./local-array-reads.js";
import { CheckedNarrows } from "./checked-narrows.js";
import { emitDenseArraySet, emitDenseSlotUnlink, type ArrayAccess } from "./dense-array-access.js";
import {
emitDenseArraySet,
emitDenseSlotUnlink,
SCR_ELEM_I32,
type ArrayAccess,
} from "./dense-array-access.js";
import { matchMapRead, type MapReadLifetimes } from "./map-read-lifetimes.js";
import { type MapEntryFusion } from "./map-entry-fusion.js";
import { emitBorrowedFieldSequence, guardedValue } from "./borrowed-receivers.js";
@@ -169,7 +183,7 @@ import {
} from "./string-slices.js";
import { LazyCaptures } from "./lazy-captures.js";
import { StackCaptures } from "./stack-captures.js";
import { emitFieldAbsentTest, f64Lit } from "./common.js";
import { emitFieldAbsentTest, f64Lit, SCR_ARR_FIELD } from "./common.js";
import { ffiExtendsNarrowIntegers } from "../targets.js";
import {
emitLiteralExpr,
@@ -484,13 +498,14 @@ function versionParameters(mod: IrModule): IrModule {
return versionLoopBoundParameters(mod, mangleFunction);
}
/** The int32 arrays the int32 slot analysis keeps in `mod`, under the
* inputs LlEmitter gives it (selectInt32Arrays). */
/** The int32 arrays the int32 slot analysis keeps in `mod` (proven and
* checked), under the inputs LlEmitter gives it (selectInt32Arrays). */
function provenInt32Arrays(mod: IrModule): ReadonlySet<string> {
return analyzeInt32Slots(mod, {
const slots = analyzeInt32Slots(mod, {
dynamicStores: dynamicFieldStores(mod),
initializerBindings: findInitializerBindings(mod),
}).int32Arrays;
});
return new Set([...slots.int32Arrays, ...slots.checkedInt32Arrays]);
}
/** The emitter for `input` after the module passes every emission runs. */
@@ -738,25 +753,42 @@ export class LlEmitter {
/** The fill temporaries (`new Array(n).fill(v)`) of those allocations:
* the same arrays, so int32 too. */
int32ArrayTemps = new Set<string>();
/** The same for the current function's int32 arrays with checked stores,
* which are only allocated by fills (ir/int32-arrays.ts
* checkableInt32Allocations): their temporaries' literals, with the fill's
* length (a literal constructs SCR_ELEM_I32 storage when that length is at
* least CHECKED_INT32_MIN_LENGTH, SCR_ELEM_F64 otherwise), and their
* temporaries. */
checkedAllocations = new Map<IrExpr, IrExpr>();
checkedArrayTemps = new Set<string>();
/** The fill calls (`.fill(v)`) of those allocations: a value that is not
* an int32 widens the new array first. */
checkedFills = new Set<IrExpr>();
/** Collect the int32 allocations of `fn`, about to be emitted. */
findInt32Allocations(fn: IrFunction): void {
this.int32Allocations.clear();
this.int32ArrayTemps.clear();
if (this.int32Slots.int32Arrays.size === 0) return;
this.checkedAllocations.clear();
this.checkedArrayTemps.clear();
this.checkedFills.clear();
const { int32Arrays, checkedInt32Arrays } = this.int32Slots;
if (int32Arrays.size === 0 && checkedInt32Arrays.size === 0) return;
everyStmtList(fn.body, {
stmt: (s) => {
if (
(s.kind === "varDecl" || s.kind === "assign") &&
this.int32Slots.int32Arrays.has(`${fn.name}\0${s.localId}`)
) {
const value = s.kind === "varDecl" ? s.init : s.value;
const allocation = value && allocatedLiteral(value);
if (allocation) {
this.int32Allocations.add(allocation.literal);
if (allocation.temporary !== null) this.int32ArrayTemps.add(allocation.temporary);
}
}
if (s.kind !== "varDecl" && s.kind !== "assign") return true;
const key = `${fn.name}\0${s.localId}`;
const checked = checkedInt32Arrays.has(key);
if (!checked && !int32Arrays.has(key)) return true;
const value = s.kind === "varDecl" ? s.init : s.value;
const allocation = value && allocatedLiteral(value);
if (!allocation) return true;
const length = checked ? fillAllocationLength(value) : null;
if (length !== null) this.checkedAllocations.set(allocation.literal, length);
else if (!checked) this.int32Allocations.add(allocation.literal);
if (allocation.temporary !== null)
(checked ? this.checkedArrayTemps : this.int32ArrayTemps).add(allocation.temporary);
if (checked && value !== allocation.literal) this.checkedFills.add(value!);
return true;
},
expr: () => true,
@@ -766,16 +798,174 @@ export class LlEmitter {
/** Whether `arr` is one of the current function's int32 arrays
* (ir/int32-arrays.ts, kept by the int32 slot analysis). */
int32Array(arr: IrExpr): boolean {
if (arr.kind !== "varRef" || this.int32Slots.int32Arrays.size === 0) return false;
if (arr.kind !== "varRef") return false;
if (this.int32KindArray(arr)) return true;
if (this.int32Slots.int32Arrays.size === 0) return false;
return (
this.int32ArrayTemps.has(arr.localId) ||
this.int32Slots.int32Arrays.has(`${this.currentFunctionName}\0${arr.localId}`)
);
}
/** Whether `arr` is one of the current function's int32 arrays with
* checked stores: int32 slots until a store widens it to doubles. Every
* access tests its kind (emitByElementKind), except in the fast copy of
* an int32 kind loop, which reads and writes it as an int32 array. */
checkedInt32Array(arr: IrExpr): boolean {
if (arr.kind !== "varRef" || this.int32KindArray(arr)) return false;
if (this.int32Kinds?.wide.has(`${this.currentFunctionName}\0${arr.localId}`)) return false;
return this.checkedInt32Local(arr.localId);
}
/** Whether the local `localId` holds one of the current function's int32
* arrays with checked stores. */
checkedInt32Local(localId: string): boolean {
if (this.int32Slots.checkedInt32Arrays.size === 0) return false;
return (
this.checkedArrayTemps.has(localId) ||
this.int32Slots.checkedInt32Arrays.has(`${this.currentFunctionName}\0${localId}`)
);
}
/** Whether `arr` is an int32 array with checked stores that the fast copy
* of the int32 kind loop being emitted holds as an int32 array. */
int32KindArray(arr: IrExpr): boolean {
return (
this.int32Kinds !== null &&
arr.kind === "varRef" &&
this.int32Kinds.int32.has(`${this.currentFunctionName}\0${arr.localId}`)
);
}
/** Before a store of `value` (of `expr`) into `arr`, in the fast copy of
* an int32 kind loop: continue when it is an int32 that is not -0, and
* otherwise leave for the statement's resume point, where it re-executes
* with kind tests. Returns the value as an i32 when it is at hand as an
* integer, or null when the store converts the double itself. */
emitInt32KindStoreCheck(arr: IrExpr, value: LlValue, expr: IrExpr): string | null {
if (!this.int32KindArray(arr) || withinInt32(this.integerRanges.get(expr))) return null;
const integer = exactInteger(this, value, expr);
if (integer !== null && withinInt32(integer.range)) return narrowInteger(this, integer);
const resume = this.int32KindResume;
if (resume === null)
throw new InternalCompilerError("llvm emitter bug: an int32 kind store without a resume");
const B = this.B;
const ok = B.tmp(),
int = B.tmp();
if (integer !== null) {
// An exact integer: a range test on its 64-bit form.
const wide = widenInteger(this, integer),
low = B.tmp(),
high = B.tmp();
B.line(`${low} = icmp sge i64 ${wide}, -2147483648`);
B.line(`${high} = icmp sle i64 ${wide}, 2147483647`);
B.line(`${ok} = and i1 ${low}, ${high}`);
const store = B.newLabel("arr.store.i32");
B.condBr(ok, store, resume);
B.startBlock(store);
B.line(`${int} = trunc i64 ${wide} to i32`);
return int;
}
this.declare("declare i32 @llvm.fptosi.sat.i32.f64(double)");
const back = B.tmp(),
exact = B.tmp(),
bits = B.tmp(),
notNegativeZero = B.tmp();
B.line(`${int} = call i32 @llvm.fptosi.sat.i32.f64(double ${value.name})`);
B.line(`${back} = sitofp i32 ${int} to double`);
B.line(`${exact} = fcmp oeq double ${back}, ${value.name}`);
B.line(`${bits} = bitcast double ${value.name} to i64`);
B.line(`${notNegativeZero} = icmp ne i64 ${bits}, -9223372036854775808`);
B.line(`${ok} = and i1 ${exact}, ${notNegativeZero}`);
const store = B.newLabel("arr.store.i32");
B.condBr(ok, store, resume);
B.startBlock(store);
return int;
}
/** Emit an access to the checked int32 array `array` once per element
* kind: `arm("i32", widened)` while it holds int32 slots and
* `arm("f64", widened)` after it widened, joined after both. `widened` is
* the label of the double arm, where an int32 arm's store of a value that
* is not an int32 continues after widening the array. Returns the phi of
* the arms' results (of LLVM type `ty`), or null when `ty` is null. */
emitByElementKind(
array: string,
ty: string | null,
arm: (acc: "i32" | "f64", widened: string) => string | null,
): string | null {
const B = this.B;
const narrow = B.newLabel("arr.kind.i32"),
wide = B.newLabel("arr.kind.f64"),
join = B.newLabel("arr.kind.join");
const kindPtr = B.tmp(),
kind = B.tmp(),
isInt32 = B.tmp();
B.line(
`${kindPtr} = getelementptr inbounds %ScrArr, ptr ${array}, i32 0, i32 ${SCR_ARR_FIELD.elem}`,
);
this.markMemoryPointer(kindPtr, "array:header");
B.line(`${kind} = load i8, ptr ${kindPtr}${this.fieldAliasAttachment(kindPtr)}`);
B.line(`${isInt32} = icmp eq i8 ${kind}, ${SCR_ELEM_I32}`);
B.condBr(isInt32, narrow, wide);
B.startBlock(narrow);
const narrowValue = arm("i32", wide);
const narrowEnd = B.isTerminated() ? null : B.currentLabel();
if (narrowEnd !== null) B.br(join);
B.startBlock(wide);
const wideValue = arm("f64", wide);
const wideEnd = B.isTerminated() ? null : B.currentLabel();
if (wideEnd !== null) B.br(join);
B.startBlock(join);
// Arms that end in their own branches (a read's shared join) leave this
// join without predecessors.
if (narrowEnd === null && wideEnd === null) B.terminate("unreachable");
if (ty === null) return null;
const arms: string[] = [];
if (narrowEnd !== null) arms.push(`[ ${narrowValue}, %${narrowEnd} ]`);
if (wideEnd !== null) arms.push(`[ ${wideValue}, %${wideEnd} ]`);
const result = B.tmp();
B.line(`${result} = phi ${ty} ${arms.join(", ")}`);
return result;
}
/** Before storing the double `value` (of `expr`) into an int32 arm
* (emitByElementKind) of the checked int32 array `array`: continue in this
* arm when it is an int32 that is not -0, or widen the array and continue
* at `widened`. A value proven an int32 needs no test. */
emitCheckedInt32Store(array: string, value: string, expr: IrExpr, widened: string): void {
if (withinInt32(this.integerRanges.get(expr))) return;
const B = this.B;
this.declare("declare i32 @llvm.fptosi.sat.i32.f64(double)");
this.declare("declare void @scr_arr_widen_i32(ptr)");
const int = B.tmp(),
back = B.tmp(),
exact = B.tmp(),
bits = B.tmp(),
notNegativeZero = B.tmp(),
ok = B.tmp();
B.line(`${int} = call i32 @llvm.fptosi.sat.i32.f64(double ${value})`);
B.line(`${back} = sitofp i32 ${int} to double`);
B.line(`${exact} = fcmp oeq double ${back}, ${value}`);
B.line(`${bits} = bitcast double ${value} to i64`);
B.line(`${notNegativeZero} = icmp ne i64 ${bits}, -9223372036854775808`);
B.line(`${ok} = and i1 ${exact}, ${notNegativeZero}`);
const store = B.newLabel("arr.store.i32"),
widen = B.newLabel("arr.widen");
B.condBr(ok, store, widen);
B.startBlock(widen);
B.line(`call void @scr_arr_widen_i32(ptr ${array})`);
B.br(widened);
B.startBlock(store);
}
/** The access an element operation on `arr` (of element type `elem`)
* takes: int32 arrays read and write their 4-byte slots. */
arrayAccess(arr: IrExpr, elem: IrType): ArrayAccess {
if (this.checkedInt32Array(arr))
throw new InternalCompilerError(
"llvm emitter bug: a checked int32 array without its kind test",
);
return this.int32Array(arr) ? "i32" : elemAccess(elem);
}
@@ -1225,6 +1415,14 @@ export class LlEmitter {
* (`enterLoopHeader`): records the ordinary copy's header, or sends the
* third copy's header there. */
speculationHeader: ((header: string) => void) | null = null;
/** A copy of an int32 kind loop (int32-kind-loops.ts) being emitted: the
* int32 arrays with checked stores it reads and writes as int32 arrays
* (the fast copy) or as double arrays (an ordinary copy entered only
* after they widened), as `function\0local` keys, and where each of the
* fast copy's writes resumes in the third copy. */
int32Kinds: { int32: Set<string>; wide: Set<string>; resume: Map<IrStmt, string> } | null = null;
/** The resume label of the fast-copy write being emitted. */
int32KindResume: string | null = null;
/** The function being emitted, for speculative globals. */
speculativeFunction = "";
/** Every function that writes each number global (computed on demand). */
@@ -5317,10 +5515,14 @@ export class LlEmitter {
const loop =
s.kind === "while" || s.kind === "doWhile" || s.kind === "for" || s.kind === "forOf";
if (loop) this.loopDepth++;
// An int32 kind loop's fast-copy write leaves for its resume point.
const kindResume = this.int32KindResume;
this.int32KindResume = this.int32Kinds?.resume.get(s) ?? null;
try {
this.emitStmtBody(s);
} finally {
this.B.debugLocation = previous;
this.int32KindResume = kindResume;
if (loop) this.loopDepth--;
}
}
@@ -615,9 +615,12 @@ export function emitArrIntrinsic(
throw new InternalCompilerError("llvm emitter bug: arrIntrinsic on non-array");
const elem = e.receiver.type.elem;
// Int32 arrays (ir/int32-arrays.ts) reach only these methods; the others
// take the runtime family's flat access.
const access = host.arrayAccess(e.receiver, elem);
if (access === "i32" && !INT32_ARRAY_METHODS.has(e.method))
// take the runtime family's flat access. One with checked stores tests
// its kind at each inline access (emitByElementKind) and otherwise takes
// the double entry points, which serve both kinds.
const checked = host.checkedInt32Array(e.receiver);
const access = checked ? "f64" : host.arrayAccess(e.receiver, elem);
if ((access === "i32" || checked) && !INT32_ARRAY_METHODS.has(e.method))
throw new InternalCompilerError(`llvm emitter bug: ${e.method} on an int32 array`);
const acc = runtimeAccess(access);
const accTy = acc === "f64" ? "double" : acc === "bool" ? "i1" : "ptr";
@@ -640,6 +643,13 @@ export function emitArrIntrinsic(
e.receiver.kind === "varRef"
? host.constantNumericTables.get(e.receiver.localId)
: undefined;
if (!table && elem.kind === "f64" && checked) {
const proven = host.provenDenseIndex(e.receiver, e.args[0]!);
const value = host.emitByElementKind(r.name, "double", (kind) =>
emitDenseArrayNumber(host, r.name, index, e.args[0], proven, kind),
)!;
return { name: value, type: e.type };
}
if (!table && elem.kind === "f64")
return {
name: emitDenseArrayNumber(
@@ -685,6 +695,21 @@ export function emitArrIntrinsic(
// no-op zero-argument call.
const vs = e.args.map((a) => host.emitExpr(a));
host.emitPublishedArrayGuard(r.name, e.receiver.type, 1, f64Lit(0), f64Lit(vs.length));
if (checked && vs.length > 0) {
// Each value appends in its own kind test: one that is not an int32
// widens the array before it is pushed.
let length = "";
vs.forEach((v, i) => {
length = host.emitByElementKind(r.name, "double", (kind, widened) => {
if (kind === "i32") host.emitCheckedInt32Store(r.name, v.name, e.args[i]!, widened);
return arrPush(host, r.name, kind, v.name, elem);
})!;
});
return { name: length, type: e.type };
}
// In an int32 kind loop's fast copy, every value is checked before
// the first appends.
vs.forEach((v, i) => host.emitInt32KindStoreCheck(e.receiver, v, e.args[i]!));
return { name: emitArrayValues(host, r.name, access, elem, vs), type: e.type };
}
case "pushSpread": {
@@ -759,7 +784,13 @@ export function emitArrIntrinsic(
let slowLabel = "";
if (method === "pop") {
slowLabel = B.newLabel("remove.slow");
denseRaw = emitDenseArrayPop(host, r.name, access, elem, la, slowLabel);
const empty = la,
slow = slowLabel;
denseRaw = checked
? host.emitByElementKind(r.name, "i64", (kind) =>
emitDenseArrayPop(host, r.name, kind, elem, empty, slow),
)!
: emitDenseArrayPop(host, r.name, access, elem, la, slowLabel);
denseLabel = B.currentLabel();
B.br(lp);
B.startBlock(slowLabel);
@@ -948,6 +979,14 @@ export function emitArrIntrinsic(
case "fillUndefined": {
const args = e.args.map((arg) => host.emitExpr(arg));
host.emitPublishedArrayGuard(r.name, e.receiver.type, 7);
if (host.checkedFills.has(e)) {
// A new int32 array with checked stores, filled with a value that
// may not be an int32: widen it first then.
const filled = B.newLabel("arr.fill.kind");
host.emitCheckedInt32Store(r.name, args[0]!.name, e.args[0]!, filled);
B.br(filled);
B.startBlock(filled);
}
const helper =
e.method === "copyWithin"
? "scr_arr_copy_within"
@@ -8,6 +8,7 @@ import { LlvmUnsupportedError } from "./unsupported.js";
import type { LlvmEmitterContext, ExprOf, LlValue } from "./expr-context.js";
import { BYTES_ELEM_NUM, closureIdentityEqual, f64Lit } from "./common.js";
import { CHECKED_INT32_MIN_LENGTH } from "../../ir/int32-arrays.js";
import { emitStringInputs } from "./string-lifetimes.js";
import { emitStringParts, numberPart, stringParts } from "./string-construction.js";
import { emitBorrowedInputs } from "./borrowed-inputs.js";
@@ -904,7 +905,30 @@ export function emitContainerExpr(
const firstSpread = e.elems.findIndex((_, i) => spreadSet.has(i));
const capacity = small ? 0 : (e.capacity ?? (firstSpread < 0 ? e.elems.length : firstSpread));
// An int32 array's allocation (ir/int32-arrays.ts) constructs
// SCR_ELEM_I32 storage; its elements are proven int32s.
// SCR_ELEM_I32 storage; its elements are proven int32s. The empty
// temporary of a fill that allocates an int32 array with checked
// stores takes SCR_ELEM_I32 storage when the fill's length (which
// evaluates without effects) is at least CHECKED_INT32_MIN_LENGTH, and
// SCR_ELEM_F64 storage otherwise.
const fillLength = host.checkedAllocations.get(e);
if (fillLength !== undefined && e.elems.length === 0) {
const length = host.emitExpr(fillLength);
const long = B.tmp(),
kind = B.tmp();
B.line(`${long} = fcmp oge double ${length.name}, ${f64Lit(CHECKED_INT32_MIN_LENGTH)}`);
B.line(`${kind} = select i1 ${long}, i32 ${SCR_ELEM_I32}, i32 0`);
host.declare(
small
? `declare ptr @scr_arr_new_small(i32)`
: `declare ptr @scr_arr_new(i32, ${host.sizeType})`,
);
B.line(
small
? `${arr} = call ptr @scr_arr_new_small(i32 ${kind})`
: `${arr} = call ptr @scr_arr_new(i32 ${kind}, ${host.sizeType} ${capacity})`,
);
return host.own({ name: arr, type: e.type });
}
const int32 = host.int32Allocations.has(e);
if (int32) {
host.declare(
@@ -986,6 +1010,26 @@ export function emitContainerExpr(
// Ref-element reads return +1 (the inline dense path retains like
// the runtime getter); own registers the owned temp in the frame.
const elem = e.arr.type.elem;
if (host.checkedInt32Array(e.arr)) {
const proven = host.provenDenseIndex(e.arr, e.index);
const value = host.emitByElementKind(
arr.name,
"double",
(kind) =>
emitDenseArrayRead(
host,
arr.name,
idx,
e.index,
kind,
elem,
"scr_arr_get_f64",
true,
proven,
).value,
)!;
return { name: value, type: e.type };
}
const acc = host.arrayAccess(e.arr, elem);
const read = emitDenseArrayRead(
host,
@@ -1013,14 +1057,18 @@ export function emitContainerExpr(
const idx = host.emitExpr(e.index);
if (e.arr.type.kind !== "array")
throw new InternalCompilerError("llvm emitter bug: arrayHas on non-array");
const t = emitDenseArrayState(
host,
arr.name,
idx,
e.index,
"has",
host.int32Array(e.arr) ? slotWidth("i32") : denseSlotWidth(host, e.arr.type.elem),
);
const t = host.checkedInt32Array(e.arr)
? host.emitByElementKind(arr.name, "i1", (kind) =>
emitDenseArrayState(host, arr.name, idx, e.index, "has", slotWidth(kind)),
)!
: emitDenseArrayState(
host,
arr.name,
idx,
e.index,
"has",
host.int32Array(e.arr) ? slotWidth("i32") : denseSlotWidth(host, e.arr.type.elem),
);
return { name: t, type: e.type };
}
case "arrayState": {
@@ -1028,14 +1076,18 @@ export function emitContainerExpr(
const idx = host.emitExpr(e.index);
if (e.arr.type.kind !== "array")
throw new InternalCompilerError("llvm emitter bug: arrayState on non-array");
const t = emitDenseArrayState(
host,
arr.name,
idx,
e.index,
"state",
host.int32Array(e.arr) ? slotWidth("i32") : denseSlotWidth(host, e.arr.type.elem),
);
const t = host.checkedInt32Array(e.arr)
? host.emitByElementKind(arr.name, "double", (kind) =>
emitDenseArrayState(host, arr.name, idx, e.index, "state", slotWidth(kind)),
)!
: emitDenseArrayState(
host,
arr.name,
idx,
e.index,
"state",
host.int32Array(e.arr) ? slotWidth("i32") : denseSlotWidth(host, e.arr.type.elem),
);
return { name: t, type: e.type };
}
case "arrIntrinsic":
@@ -0,0 +1,135 @@
import type { IrExpr, IrStmt } from "../../ir/ir.js";
import { everyExprChild, everyStmtChild } from "../../ir/traverse.js";
import { MAX_LOOP_NODES, SIMPLE_STATEMENTS } from "./speculative-integers.js";
/** Int32 kind loops.
*
* An int32 array with checked stores (ir/int32-slots.ts) holds int32 slots
* (or doubles, when its fill was short) until a store of a value that is
* not an int32 widens it to doubles in place, so an access to it tests its
* element kind first (LlEmitter.emitByElementKind). An innermost loop that
* accesses such arrays tests their kinds once on entry instead
* (emitLoopCopies). The fast copy runs when every one of them holds int32
* slots and reads and writes them as int32 arrays; the wide copy runs when
* every one holds doubles and accesses them as double arrays; a loop over
* several arrays whose kinds differ runs the mixed copy, which tests the
* kinds at each access. A store of the fast copy whose value is not an
* int32 branches to the same statement in a third copy, which re-executes
* it with kind tests (widening the array) and finishes the iteration, then
* enters the mixed copy, or the wide copy when the loop has one array.
* Re-execution is exact because such a store's statement evaluates nothing
* but locals, literals, number arithmetic and element reads before it
* writes.
*
* Nothing else can widen these arrays: they never reach code other than
* the compiled element operations of their own function
* (ir/int32-arrays.ts). */
export interface Int32KindLoopPlan {
/** The arrays the fast copy reads and writes as int32 arrays, by local,
* with one of their references in the loop. */
arrays: Map<string, IrExpr>;
/** Element writes and pushes whose value may not be an int32: the
* statements the fast copy may leave at. */
writes: Set<IrStmt>;
}
type KindLoop = Extract<IrStmt, { kind: "for" | "while" | "doWhile" }>;
/** A number or boolean expression whose evaluation has no effect: locals,
* literals, arithmetic, comparisons and element reads of local arrays. */
function isPure(e: IrExpr): boolean {
if (e.type.kind !== "f64" && e.type.kind !== "bool") return false;
switch (e.kind) {
case "numLit":
case "boolLit":
case "varRef":
return true;
case "bin":
return (
(e.left.type.kind === "f64" || e.left.type.kind === "bool") &&
isPure(e.left) &&
isPure(e.right)
);
case "unary":
return isPure(e.operand);
case "ternary":
return isPure(e.cond) && isPure(e.then) && isPure(e.else_);
case "arrayGet":
case "arrayHas":
return e.arr.kind === "varRef" && isPure(e.index);
default:
return false;
}
}
/** The plan for `loop`, or null. `checked` names the int32 array locals
* with checked stores of the function; `provenInt32` the values the static
* range analysis proves int32s (stores that never leave the fast copy). */
export function planInt32KindLoop(
loop: KindLoop,
checked: (localId: string) => boolean,
provenInt32: (e: IrExpr) => boolean,
): Int32KindLoopPlan | null {
const arrays = new Map<string, IrExpr>();
const writes = new Set<IrStmt>();
const assigned = new Set<string>();
let nodes = 0;
let rejected = false;
const reject = (): false => {
rejected = true;
return false;
};
const array = (e: IrExpr): string | null =>
e.kind === "varRef" && checked(e.localId) ? e.localId : null;
const push = (e: IrExpr): (IrExpr & { kind: "arrIntrinsic" }) | null =>
e.kind === "arrIntrinsic" && e.method === "push" && array(e.receiver) !== null ? e : null;
const expr = (e: IrExpr): boolean => {
if (rejected) return false;
nodes++;
if (e.kind === "varRef" && checked(e.localId) && !arrays.has(e.localId))
arrays.set(e.localId, e);
// A push the statement walk did not take as a write: its values must
// be proven int32s, since it cannot leave the fast copy.
const pushed = push(e);
if (pushed && !pushed.args.every(provenInt32)) return reject();
return everyExprChild(e, expr, nested);
};
// Statements inside expressions run mid-expression, where the fast copy
// cannot leave: they may not write these arrays.
const nested = (s: IrStmt): boolean => {
nodes++;
if (s.kind === "assign" || s.kind === "varDecl") assigned.add(s.localId);
if (s.kind === "arraySet" && array(s.arr) !== null && !provenInt32(s.value)) return reject();
return everyStmtChild(s, expr, nested);
};
const stmt = (s: IrStmt): boolean => {
if (rejected) return false;
nodes++;
if (!SIMPLE_STATEMENTS.has(s.kind)) return reject();
if (s.kind === "assign" || s.kind === "varDecl") assigned.add(s.localId);
if (s.kind === "arraySet" && array(s.arr) !== null && !provenInt32(s.value)) {
if (!isPure(s.index) || !isPure(s.value)) return reject();
writes.add(s);
return everyStmtChild(s, expr, stmt);
}
const pushed = s.kind === "exprStmt" ? push(s.expr) : null;
if (pushed && !pushed.args.every(provenInt32)) {
if (!pushed.args.every(isPure)) return reject();
writes.add(s);
expr(pushed.receiver);
pushed.args.forEach(expr);
return !rejected;
}
return everyStmtChild(s, expr, stmt);
};
if (loop.cond) expr(loop.cond);
if (loop.kind === "for" && loop.update) stmt(loop.update);
for (const s of loop.body) stmt(s);
if (rejected || nodes > MAX_LOOP_NODES || arrays.size === 0) return null;
// A loop that rebinds one of the arrays could reach another array of
// either kind.
for (const id of arrays.keys()) if (assigned.has(id)) return null;
return { arrays, writes };
}
@@ -767,7 +767,37 @@ export function emitLocalArrayRead(
}
B.line(`store i32 ${read.presentTag}, ptr ${tag}`);
};
if (inline)
if (inline && scalar && host.checkedInt32Array(read.array))
// An int32 array with checked stores: its int32 arm stores the element's
// double bits, as the double arm does.
host.emitByElementKind(array.name, null, (kind) => {
const present =
kind === "i32"
? (value: string): void => {
const number = B.tmp(),
bits = B.tmp();
B.line(`${number} = sitofp i32 ${value} to double`);
B.line(`${bits} = bitcast double ${number} to i64`);
storeValue(bits);
}
: storeValue;
const loadType = kind === "i32" ? "i32" : "i64";
const induction = loopIndex !== null;
emitDenseReferenceArrayRead(
host,
array,
index,
integerIndex,
induction,
present,
no,
slow,
join,
loadType,
);
return null;
});
else if (inline)
emitDenseReferenceArrayRead(
host,
array,
@@ -35,12 +35,12 @@ export interface SpeculativeLoopPlan {
type SpeculativeLoop = Extract<IrStmt, { kind: "for" | "while" | "doWhile" }>;
/** The fast copy duplicates the loop; keep the duplicated code small. */
const MAX_LOOP_NODES = 400;
export const MAX_LOOP_NODES = 400;
/** Statements whose emission is plain control flow or a single write.
* Nested loops, switches and try statements keep the loop on the ordinary
* path (their emission keeps state that the two copies would not share). */
const SIMPLE_STATEMENTS = new Set<string>([
export const SIMPLE_STATEMENTS: ReadonlySet<string> = new Set<string>([
"varDecl",
"assign",
"exprStmt",
+63 -3
View File
@@ -3,6 +3,7 @@ import { reparseExpr } from "./serialize.js";
import {
everyExprChild,
everyStmtChild,
everyStmtList,
transformExpr,
transformStmt,
transformStmtList,
@@ -23,9 +24,12 @@ import {
* Candidates are only proposed here. The int32 slot analysis
* (int32-slots.ts) keeps the ones whose every stored element is a proven
* int32 (never -0), assuming the candidates' own elements are int32s, at
* a fixpoint, and the LLVM backend stores those as SCR_ELEM_I32. Their
* element-reader helper calls are inlined first (rewriteCandidates), so
* no shared helper ever receives an int32 array. */
* a fixpoint, and the LLVM backend stores those as SCR_ELEM_I32. It keeps
* the ones whose stores are exact integers but not all proven int32s as
* int32 arrays with checked stores: SCR_ELEM_I32 until a store that is not
* an int32 widens them to doubles in place. Their element-reader helper
* calls are inlined first (rewriteCandidates), so no shared helper ever
* receives an int32 array. */
/** Array methods a candidate may be the receiver of: the runtime paths
* behind them read and write int32 slots through the double-valued slot
@@ -87,6 +91,62 @@ function fillAllocation(e: IrExpr): (IrExpr & { kind: "arrIntrinsic" }) | null {
return e;
}
/** Fill-form arrays shorter than this hold doubles even when they are int32
* arrays with checked stores: their doubles fit a typical 32 KiB L1 data
* cache, so 4-byte slots save no memory traffic. */
export const CHECKED_INT32_MIN_LENGTH = 4096;
/** The length `n` of `new Array(n).fill(v)`, or null for any other
* allocation. */
export function fillAllocationLength(e: IrExpr): IrExpr | null {
const fill = fillAllocation(e);
if (!fill || fill.receiver.kind !== "seqExpr") return null;
const set = fill.receiver.stmts[1]!;
return set.kind === "arraySetLength" ? set.length : null;
}
/** A number expression whose evaluation has no effect: locals, literals and
* arithmetic. */
function isPureLength(e: IrExpr): boolean {
if (e.type.kind !== "f64") return false;
switch (e.kind) {
case "numLit":
case "varRef":
return true;
case "bin":
return e.left.type.kind === "f64" && isPureLength(e.left) && isPureLength(e.right);
case "unary":
return isPureLength(e.operand);
default:
return false;
}
}
/** Whether the candidate `id` of `fn` may hold int32 slots with checked
* stores: every array it is assigned is the fill form, with a length that
* may reach CHECKED_INT32_MIN_LENGTH and evaluates without effects (the
* backend evaluates it once more to pick the new array's kind). */
export function checkableInt32Allocations(fn: IrFunction, id: string): boolean {
let ok = true;
everyStmtList(fn.body, {
stmt: (s) => {
if ((s.kind === "varDecl" || s.kind === "assign") && s.localId === id) {
const value = s.kind === "varDecl" ? s.init : s.value;
const length = value ? fillAllocationLength(value) : null;
if (
length === null ||
!isPureLength(length) ||
(length.kind === "numLit" && length.value < CHECKED_INT32_MIN_LENGTH)
)
ok = false;
}
return ok;
},
expr: () => ok,
});
return ok;
}
/** A fresh number array: a literal without spreads, or the fill form. */
export function int32ArrayAllocation(e: IrExpr): boolean {
return (e.kind === "arrayLit" && (e.spreads?.length ?? 0) === 0) || fillAllocation(e) !== null;
+21 -4
View File
@@ -10,7 +10,7 @@ import {
} from "./integer-ranges.js";
import { exprFunctionReference, moduleFunctionReferences } from "./function-references.js";
import { findInitializerBindings, withInitializerBindings } from "./initializer-bindings.js";
import { int32ArrayStores } from "./int32-arrays.js";
import { checkableInt32Allocations, int32ArrayStores } from "./int32-arrays.js";
import { everyExprChild, everyStmtChild, everyStmtList, everyTypeChild } from "./traverse.js";
/** The storage slot an array-valued expression reads, as an
@@ -251,6 +251,15 @@ export class Int32Slots {
* module's int32 array candidates whose every stored element the
* fixpoint proved an int32. */
readonly int32Arrays = new Set<string>();
/** Local number arrays that may hold int32 slots with checked stores: the
* candidates whose element stores the fixpoint proved exact integers but
* not int32s, allocated only by fills that may be long. A fill of at least
* CHECKED_INT32_MIN_LENGTH elements creates one with int32 slots and a
* shorter one with doubles. Their elements give no int32 facts; compiled
* accesses test the array's kind (once per innermost loop where they can,
* backend/llvm/int32-kind-loops.ts), and a store that is not an int32
* widens it first (scr_arr_widen_i32). */
readonly checkedInt32Arrays = new Set<string>();
/** Whether every present element of the number array `arr` evaluates to
* (read inside `fnName`) is an int32. */
hasInt32Elements(fnName: string, arr: IrExpr): boolean {
@@ -1108,8 +1117,13 @@ export function analyzeInt32Slots(mod: IrModule, inputs: Int32SlotInputs = {}):
const int32 = (e: IrExpr): boolean => withinInt32(ranges.get(e));
// Every element stored into an int32 array candidate must be an int32
// under the facts that assume the candidates' own elements are; a
// failing store drops its array, and the function is analyzed again
// without it.
// failing store drops its array's element facts, and the function is
// analyzed again without them. When every failing store is still an
// exact integer under those facts (a counter, `a[i]! + 1`) and the array
// is only ever allocated by a fill that may be long
// (checkableInt32Allocations), it keeps int32 slots with checked stores
// (checkedInt32Arrays), which widen it to doubles in place at the first
// store that is not an int32.
const arrays = arrayCandidates.get(fn.name);
if (arrays) {
let dropped = false;
@@ -1117,7 +1131,10 @@ export function analyzeInt32Slots(mod: IrModule, inputs: Int32SlotInputs = {}):
const key = `l:${fn.name}\0${id}`;
if (!slots.hasInt32ElementSlot(key) || values.every(int32)) continue;
slots.dropInt32Elements(key);
reason(`array:${fn.name}\0${id}`, "an element store is not int32");
const checkable = (value: IrExpr): boolean => int32(value) || ranges.get(value) != null;
if (values.every(checkable) && checkableInt32Allocations(fn, id))
slots.checkedInt32Arrays.add(`${fn.name}\0${id}`);
else reason(`array:${fn.name}\0${id}`, "an element store is not int32");
dropped = true;
}
if (dropped && !queued.has(fn.name)) {
+2 -1
View File
@@ -1076,7 +1076,8 @@ export interface IrModule {
/** Local number arrays (`function\0local`) the backend may store as
* int32 slots, proposed by ir/int32-arrays.ts after their element-reader
* calls were inlined; the int32 slot analysis keeps the ones whose every
* stored element it proves an int32. Absent when there is none. */
* stored element it proves an int32, and those whose stores are exact
* integers with checked stores. Absent when there is none. */
int32ArrayCandidates?: string[];
/** The WebAssembly modules translated into this program (wasm/
* translate.ts), present exactly when the program uses the WebAssembly
@@ -63,6 +63,16 @@ function negativeZero(): number {
return 1 / a[1]!;
}
function counter(n: number): number {
const a = new Array(n).fill(0) as number[];
for (let i = 0; i < 1000; i++) a[i % n] = a[i % n]! + 1;
return a[0]!;
}
function shortCounter(): number {
const a = new Array(16).fill(0) as number[];
for (let i = 0; i < 1000; i++) a[i % 16] = a[i % 16]! + 1;
return a[0]!;
}
function literalCounter(n: number): number {
const a = [0];
for (let i = 0; i < n; i++) a[0] = a[0]! + 1;
return a[0]!;
@@ -72,7 +82,16 @@ function joined(n: number): string {
return a.join(",");
}
const { n } = JSON.parse('{"n": 8}') as { n: number };
console.log(compose(n | 0, 3), counts(n | 0), fraction(n), negativeZero(), counter(n), joined(n));
console.log(
compose(n | 0, 3),
counts(n | 0),
fraction(n),
negativeZero(),
counter(n),
shortCounter(),
literalCounter(n),
joined(n),
);
`;
/** Every function emitted for the source function \`name\` (its int32 ABI
@@ -100,15 +119,64 @@ test("arrays storing only int32 elements get 4-byte slots", async () => {
expect(counts).not.toContain("popArray");
});
test("arrays with a store that may not be an int32, or that escape, keep doubles", async () => {
test("arrays with a store that has no exact integer range, or that escape, keep doubles", async () => {
const llvm = await emitLlvm(source);
for (const name of ["fraction", "negativeZero", "counter", "joined"]) {
for (const name of ["fraction", "negativeZero", "joined"]) {
const fn = functions(llvm, name);
expect(fn, name).not.toBe("");
expect(fn, name).not.toMatch(/@scr_arr_new(_small)?\(i32 7/);
expect(fn, name).not.toContain("arr.kind.");
}
});
test("a counter array keeps int32 slots with checked stores", async () => {
const llvm = await emitLlvm(source);
const counter = functions(llvm, "counter");
// `a[k] = a[k]! + 1` is an exact integer but no int32 proof: a fill of at
// least 4096 elements creates SCR_ELEM_I32 storage (SCR_ELEM_F64 for a
// shorter one), accesses test the kind, and a store widens the array
// before a value that is not an int32.
expect(counter).toMatch(
/fcmp oge double %\w+, [^\n]+\n\s+%\w+ = select i1 %\w+, i32 7, i32 0\n\s+%\w+ = call ptr @scr_arr_new(_small)?\(i32 %\w+/,
);
expect(counter).toMatch(/load i8, ptr %\w+[^\n]*\n\s+%\w+ = icmp eq i8 %\w+, 7/);
expect(counter).toContain("@llvm.fptosi.sat.i32.f64");
expect(counter).toContain("call void @scr_arr_widen_i32(");
// Arrays with proven stores take no kind test, and a fill of a constant
// short length or an array literal keeps doubles.
for (const name of ["compose", "counts", "shortCounter", "literalCounter"]) {
const fn = functions(llvm, name);
expect(fn, name).not.toContain("arr.kind.");
expect(fn, name).not.toContain("kind.fast");
expect(fn, name).not.toContain("scr_arr_widen_i32");
}
for (const name of ["shortCounter", "literalCounter"])
expect(functions(llvm, name), name).not.toMatch(/@scr_arr_new(_small)?\(i32 7/);
});
test("a loop over a counter array tests its kind once on entry", async () => {
const llvm = await emitLlvm(source);
const counter = functions(llvm, "counter");
// The loop's fast copy runs while the array holds int32 slots, with no
// kind test of its own: the sum is an integer, and a sum past the int32
// range leaves for the third copy, which re-executes the store with kind
// tests and widens the array.
expect(counter).toMatch(
/icmp eq i8 %\w+, 7\n\s+br i1 %\w+, label %kind\.fast\d+, label %kind\.wide/,
);
const rest = counter.slice(counter.search(/^kind\.fast\d+:/m));
// The fast copy is the last one, before the loop's exit.
const fast = rest.slice(0, rest.search(/^loop\.e\d+:/m));
expect(fast).not.toBe("");
expect(fast).not.toMatch(/icmp eq i8 %\w+, 7/);
expect(fast).toMatch(/add nsw i64 %\w+, 1/);
expect(fast).toMatch(
/icmp sle i64 %\w+, 2147483647\n[^\n]*\n\s+br i1 %\w+, label %arr\.store\.i32\d+, label %kind\.resume\d+/,
);
expect(fast).toMatch(/store i32 %\w+, ptr %\w+, !tbaa/);
expect(counter).toMatch(/^kind\.resume\d+:/m);
});
test("only the candidates the proof keeps are rewritten", async () => {
const mod = await lowerIr(source);
const proposed = markInt32ArrayCandidates(mod).int32ArrayCandidates ?? [];
@@ -0,0 +1,6 @@
{
"order": [
"<repo>/tests/corpus/7327-int32-array-widening.ts"
],
"diags": []
}
+30
View File
@@ -579,6 +579,36 @@ static void scr_arr_move_inline(ScrArr *a, size_t cap) {
a->spare &= (uint16_t)~SCR_ARR_INLINE_SLOTS;
}
/* Widen an int32 array (SCR_ELEM_I32) to doubles in place, one way: its
* dense values move to 8-byte slots, with their states when it has them, and
* its kind becomes SCR_ELEM_F64. Compiled code calls it before storing a
* value that is not an int32 into an int32 array with checked stores; `data`
* may move. Sparse entries already hold doubles. */
void scr_arr_widen_i32(ScrArr *a) {
if (a->elem != SCR_ELEM_I32) return;
size_t cap = a->cap;
if (cap > 0) {
/* Int32 arrays never shift, so they have no head room; inline slots
* have no ext prefix. */
bool inline_slots = scr_arr_inline(a);
bool states = scr_arr_has_states(a);
size_t unit = sizeof(uint64_t) + (states ? 1 : 0);
size_t prefix = inline_slots ? 0 : scr_arr_prefix(a);
if (cap > (SIZE_MAX - prefix) / unit) scr_arr_oom();
char *start = scr_mem_alloc(prefix + cap * unit);
if (!start) scr_arr_oom();
if (prefix) memcpy(start, scr_arr_alloc_start(a), prefix);
uint64_t *data = (uint64_t *)(start + prefix);
const int32_t *old = (const int32_t *)a->data;
for (size_t i = 0; i < cap; i++) data[i] = scr_arr_f64_slot((double)old[i]);
if (states) memcpy((uint8_t *)data + cap * sizeof(uint64_t), scr_arr_states_end(a) - cap, cap);
if (!inline_slots && a->rc != SIZE_MAX) scr_mem_free(scr_arr_alloc_start(a));
a->data = data;
a->spare &= (uint16_t)~SCR_ARR_INLINE_SLOTS;
}
a->elem = SCR_ELEM_F64;
}
/* How a dense allocation is sized. */
typedef enum {
SCR_ARR_GROW_APPEND, /* geometric headroom for further appends */
+3
View File
@@ -433,6 +433,9 @@ ScrArr *scr_arr_new_ops(const ScrArrOps *ops, size_t initial_cap);
* array plus five 8-byte slots). Otherwise exactly scr_arr_new(elem, 0) and
* scr_arr_new_ops(ops, 0). Returns +1. */
ScrArr *scr_arr_new_small(ScrElemKind elem);
/* An int32 array's storage as doubles, in place (SCR_ELEM_I32 to
* SCR_ELEM_F64); no-op for any other kind. */
void scr_arr_widen_i32(ScrArr *a);
ScrArr *scr_arr_new_ops_small(const ScrArrOps *ops);
/* scr_arr_new(elem, cap) for a known final length: the `cap` slots share
* the array's own block (one allocation). With `states`, the array starts
+38
View File
@@ -1670,6 +1670,44 @@ static void test_i32_slots(void) {
check(i32_layout_ok(k) && scr_arr_get_f64(k, 6) == 6 && scr_arr_get_f64(k, 0) == 0,
"a known-count int32 array grows out of its block");
scr_arr_release(k);
/* Widening: in place, to 8-byte double slots, keeping states. */
ScrArr *w = scr_arr_new(SCR_ELEM_I32, 3);
for (int i = 0; i < 3; i++) scr_arr_push_f64(w, i - 1);
check(inline_slots(w), "a three-element int32 construction keeps inline slots");
scr_arr_widen_i32(w);
check(w->elem == SCR_ELEM_F64 && !inline_slots(w) && scr_arr_len(w) == 3 && w->filled == 3 &&
scr_arr_get_f64(w, 0) == -1 && scr_arr_get_f64(w, 2) == 1,
"widening moves inline int32 slots out as doubles");
scr_arr_set_f64(w, 1, 2.5);
check(scr_arr_get_f64(w, 1) == 2.5, "a widened array stores fractions");
scr_arr_widen_i32(w);
check(w->elem == SCR_ELEM_F64 && scr_arr_get_f64(w, 1) == 2.5, "widening twice is a no-op");
scr_arr_release(w);
ScrArr *h = scr_arr_new_small(SCR_ELEM_I32);
for (int i = 0; i < 100; i++) scr_arr_push_f64(h, i * 3);
scr_arr_set_len(h, 120);
scr_arr_set_undefined(h, 5);
scr_arr_set_f64(h, 4000000, -7);
size_t cap = h->cap;
scr_arr_widen_i32(h);
bool widened = h->elem == SCR_ELEM_F64 && h->cap == cap && scr_arr_len(h) == 4000001 &&
scr_arr_states_end(h) == (const uint8_t *)h->data + 9 * (size_t)h->cap;
for (int i = 0; i < 100; i++)
if (i != 5) widened &= scr_arr_get_f64(h, i) == i * 3;
check(widened && scr_arr_state(h, 5) == SCR_ARR_UNDEFINED && scr_arr_state(h, 110) == SCR_ARR_HOLE &&
scr_arr_get_f64(h, 4000000) == -7 && arr_sparse_len(h) == 1,
"widening keeps values, present undefined, holes and sparse entries");
scr_arr_push_f64(h, 0.5);
check(scr_arr_get_f64(h, 4000001) == 0.5, "a widened array keeps growing as doubles");
scr_arr_release(h);
ScrArr *e = scr_arr_new_small(SCR_ELEM_I32);
scr_arr_widen_i32(e);
scr_arr_push_f64(e, -0.0);
check(e->elem == SCR_ELEM_F64 && signbit(scr_arr_get_f64(e, 0)), "an empty int32 array widens too");
scr_arr_release(e);
}
static void test_inline_slots(void) {
+186
View File
@@ -0,0 +1,186 @@
// Local number arrays with checked int32 stores: `new Array(n).fill(v)`
// arrays whose stores are exact integers but not proven int32s. A fill of
// at least 4096 elements creates int32 slots, and the first store past the
// int32 range widens the array to doubles in place; a shorter fill creates
// doubles. Each case runs at both sizes. Arrays with a store that has no
// exact integer range (a product of two int32s, a fraction, a value that may
// be -0), and arrays allocated as literals, keep doubles throughout.
function counters(n: number, rounds: number): string {
const c: number[] = new Array(n).fill(0);
for (let r = 0; r < rounds; r++)
for (let i = 0; i < c.length; i++) c[i] = c[i]! + ((i & 3) + 1) * 1000000;
return `counters ${n} ${c[0]} ${c[1]} ${c[2]} ${c[3]} ${c[n - 1]} ${c.length}`;
}
// The store at i = 45 passes 2^31 and widens the array partway through a
// loop; the remaining passes store doubles.
function denseLoop(n: number, k: number): string {
const a: number[] = new Array(n).fill(0);
for (let i = 0; i < a.length; i++) a[i] = i + k + 2147483600;
let s = 0;
for (let i = 0; i < a.length; i++) s = (s + a[i]!) % 1000003;
return `dense ${n} ${a[44]} ${a[45]} ${a[n - 1]} ${s}`;
}
// Checked stores whose values stay int32: no widening.
function bfs(n: number): string {
const dist: number[] = new Array(n).fill(-1);
const queue: number[] = [0];
dist[0] = 0;
for (let h = 0; h < queue.length; h++) {
const u = queue[h]!;
const edges = [(u + 1) & (n - 1), (u * 5 + 1) & (n - 1)];
for (let j = 0; j < edges.length; j++) {
const v = edges[j]!;
if (dist[v]! < 0) {
dist[v] = dist[u]! + 1;
queue.push(v);
}
}
}
let far = 0,
sum = 0;
for (let i = 0; i < n; i++) {
if (dist[i]! > far) far = dist[i]!;
sum += dist[i]!;
}
return `bfs ${n} ${far} ${sum} ${queue.length}`;
}
// A fill value that is not an int32 widens the new array before the fill.
function filledPastInt32(n: number, k: number): string {
const a: number[] = new Array(n).fill(k * 1000000000);
a[1] = a[1]! + 1;
const b: number[] = new Array(n).fill(k);
b[2] = b[2]! * 2;
return `filled ${n} ${a[0]} ${a[1]} ${a.length} ${b[0]} ${b[2]}`;
}
// A chain of sums crosses 2^31 partway through; the second loop runs after
// the array widened.
function chain(n: number, step: number): string {
const a: number[] = new Array(n).fill(0);
for (let i = 1; i < n; i++) a[i] = a[i - 1]! + (i < 16 ? step : 1);
let s = 0;
for (let i = 0; i < n; i++) s = (s + (a[i]! % 1000003)) % 1000003;
for (let i = 1; i < n; i++) a[i] = a[i]! - a[i - 1]!;
return `chain ${n} ${step} ${a[1]} ${a[5]} ${a[6]} ${a[n - 1]} ${s}`;
}
// Two arrays in one loop: one widens partway through, the other keeps
// int32 slots; then both are read in another loop.
function twoArrays(n: number, m: number, k: number): string {
const small: number[] = new Array(n).fill(1);
const big: number[] = new Array(m).fill(k);
const count = Math.min(n, m);
for (let i = 1; i < count; i++) {
small[i] = small[i - 1]! + 1;
big[i] = big[i - 1]! + (i < 8 ? 700000000 : 1);
}
let s = 0,
t = 0;
for (let i = 0; i < count; i++) {
s += small[i]!;
t = (t + (big[i]! % 997)) % 997;
}
return `two ${n} ${m} ${small[count - 1]} ${big[2]} ${big[3]} ${big[count - 1]} ${s} ${t}`;
}
// Pushes that pass 2^31 widen the array before they append.
function pushes(n: number, k: number): string {
const a: number[] = new Array(n).fill(k);
for (let i = 0; i < 8; i++) a.push(a[a.length - 1]! + 900000000);
let s = 0;
for (let i = 0; i < a.length; i++) s = (s + (a[i]! % 10007)) % 10007;
return `pushes ${n} ${a.length} ${a[n]} ${a[n + 2]} ${a[n + 7]} ${s}`;
}
function controlFlow(n: number, k: number): string {
const a: number[] = new Array(n).fill(k);
let stores = 0;
for (let i = 1; i < n; i++) {
if (i % 2 === 0) continue;
a[i] = a[i - 1]! + 1000000000 * (i % 4);
stores++;
if (a[i]! > 2500000000) break;
}
let i = 0;
while (i < n && a[i]! < 2000000000) i++;
let j = n;
do j--;
while (j > 0 && a[j] === k);
return `flow ${n} ${stores} ${a[1]} ${a[2]} ${a[3]} ${i} ${j}`;
}
function readOnly(n: number, wide: boolean): string {
const a: number[] = new Array(n).fill(2);
if (wide) a[0] = a[0]! + 3000000000;
let s = 0,
present = 0;
for (let i = 0; i < n + 2; i++) {
s += a[i] ?? 0;
if (i in a) present++;
}
return `read ${n} ${wide} ${s} ${present}`;
}
function holes(n: number): string {
// Holes, present undefined and sparse entries survive widening.
const a: number[] = new Array(n).fill(3);
a[n + 3] = a[1]! + 1;
a[n + 2] = a[n + 3]! - 1;
a[2 * n + 100000] = a[0]! + 1;
a[n + 1] = a[n + 2]! + 3000000000;
return `holes ${n} ${a.length} ${a[n]} ${n in a} ${a[n + 1]} ${a[n + 2]} ${a[2 * n + 100000]}`;
}
// These arrays never take int32 slots: a fractional store, a product of
// two int32s, -0, an escape, and arrays allocated as literals.
function neverInt32(n: number, x: number, y: number): string[] {
const f: number[] = new Array(n).fill(1);
f[0] = f[0]! / x;
const m: number[] = new Array(n).fill(3);
m[1] = m[1]! * y;
const z: number[] = new Array(n).fill(4);
z[1] = z[1]! % 2;
z[2] = -z[0]! % 2;
const j: number[] = new Array(n).fill(3);
j[0] = j[0]! + 1;
const lit = [1, 2, 3];
lit[0] = lit[0]! + 3000000000;
const pushed: number[] = [];
for (let i = 0; i < 4; i++) pushed.push(i * 1000000000);
pushed[0] = pushed[3]! + pushed[2]!;
return [
`never ${f[0]} ${m[1]} ${z[1]} ${Object.is(z[2], -0)} ${j.slice(0, 2).join("|")}`,
`literals ${lit[0]} ${lit[2]} ${pushed[0]} ${pushed.length}`,
];
}
const config = JSON.parse('{"small": 64, "large": 8192, "k": 3, "y": 2000000000}') as {
small: number;
large: number;
k: number;
y: number;
};
const lines: string[] = [];
for (const n of [config.small | 0, config.large | 0]) {
lines.push(counters(n, 1000));
lines.push(denseLoop(n, config.k | 0));
lines.push(bfs(n));
lines.push(filledPastInt32(n, config.k | 0));
lines.push(chain(n, 400000000));
lines.push(chain(n, 100000000));
lines.push(twoArrays(n, n, config.k | 0));
lines.push(pushes(n, config.k | 0));
lines.push(controlFlow(n, config.k | 0));
lines.push(readOnly(n, false));
lines.push(readOnly(n, true));
lines.push(holes(n));
}
// One long array and one short: their kinds differ on entry to the loop.
lines.push(twoArrays(config.large | 0, config.small | 0, config.k | 0));
lines.push(twoArrays(config.small | 0, config.large | 0, config.k | 0));
lines.push(...neverInt32(config.small | 0, config.k, config.y | 0));
for (const line of lines) console.log(line);
@@ -0,0 +1 @@
{ "plain": 4500, "san": 6100 }