mirror of
https://github.com/vercel-labs/scriptc.git
synced 2026-10-27 06:43:19 +08:00
Keep int32 slots for long number arrays with checked integer stores
A local number[] candidate whose stores are exact integers but not all proven int32s (a distance array updated with dist[v] = dist[u]! + 1), and that is only ever allocated by new Array(n).fill(v), now holds SCR_ELEM_I32 slots when n is at least 4096. A shorter fill creates doubles, which fit a typical L1 data cache. The first store of a value that is not an int32 widens the array to doubles in place (scr_arr_widen_i32). An innermost loop that accesses such arrays tests their kinds once on entry (backend/llvm/int32-kind-loops.ts). Its fast copy reads and writes them as int32 arrays; a store that would widen one leaves for a copy that re-executes it with kind tests and finishes the loop on doubles. When the arrays already hold doubles, a copy accesses them as double arrays. Other accesses test the kind each time. Only these arrays take kind tests: every other array, and every program without one, emits the same code as before.
This commit is contained in:
@@ -542,7 +542,9 @@ export function emitDenseArrayState(
|
||||
* value's reference. A dense index stores the slot, marks it present, and
|
||||
* extends the length when it lands at or past it; a replaced present
|
||||
* reference is released after the new edge is published (a release can
|
||||
* collect cycles). Every other index takes the unchanged runtime call. */
|
||||
* collect cycles). Every other index takes the unchanged runtime call.
|
||||
* `int32`, for an int32 array, is the value as an i32 when the caller has
|
||||
* it at hand. */
|
||||
export function emitDenseArraySet(
|
||||
host: LlvmEmitterContext,
|
||||
array: string,
|
||||
@@ -552,6 +554,7 @@ export function emitDenseArraySet(
|
||||
element: IrType,
|
||||
value: string,
|
||||
proven = false,
|
||||
int32: string | null = null,
|
||||
): void {
|
||||
const B = host.B;
|
||||
const ty = valueType(acc);
|
||||
@@ -566,8 +569,8 @@ export function emitDenseArraySet(
|
||||
}
|
||||
// An int32 array's stores are proven int32s (ir/int32-arrays.ts); both
|
||||
// dense paths below store the converted value.
|
||||
let int32Value: string | null = null;
|
||||
if (acc === "i32") {
|
||||
let int32Value: string | null = int32;
|
||||
if (acc === "i32" && int32Value === null) {
|
||||
int32Value = B.tmp();
|
||||
B.line(`${int32Value} = fptosi double ${value} to i32`);
|
||||
}
|
||||
|
||||
@@ -139,6 +139,8 @@ export function emitFunction(emitter: LlEmitter, fn: IrFunction): string {
|
||||
emitter.speculativeFunction = fn.name;
|
||||
emitter.speculation = null;
|
||||
emitter.speculationResume = null;
|
||||
emitter.int32Kinds = null;
|
||||
emitter.int32KindResume = null;
|
||||
emitter.finalFieldBorrows =
|
||||
!fn.async &&
|
||||
!fn.generator &&
|
||||
|
||||
@@ -34,9 +34,16 @@ import { dynHoleRead } from "./loop-array-borrows.js";
|
||||
import { endsWithJump, matchStringSelfConcat } from "../../ir/analysis.js";
|
||||
import { charAtUnitRead, emitDiscardedSplice } from "./expr-containers.js";
|
||||
import { InternalCompilerError } from "../../errors.js";
|
||||
import { emitDenseArrayGet, emitDenseArrayPop, emitDenseArraySet } from "./dense-array-access.js";
|
||||
import {
|
||||
emitDenseArrayGet,
|
||||
emitDenseArrayPop,
|
||||
emitDenseArraySet,
|
||||
SCR_ELEM_I32,
|
||||
} from "./dense-array-access.js";
|
||||
import { emitBorrowedInput } from "./borrowed-inputs.js";
|
||||
import { f64Lit, llvmCommentText, loadArrCount, sizeToDouble } from "./common.js";
|
||||
import { f64Lit, llvmCommentText, loadArrCount, SCR_ARR_FIELD, sizeToDouble } from "./common.js";
|
||||
import { planInt32KindLoop, type Int32KindLoopPlan } from "./int32-kind-loops.js";
|
||||
import { withinInt32 } from "../../ir/integer-ranges.js";
|
||||
import { resultSlotStore } from "../result-slots.js";
|
||||
import {
|
||||
matchIntegerArrayForLoop,
|
||||
@@ -411,7 +418,21 @@ export function emitStmtBody(emitter: LlEmitter, s: IrStmt): void {
|
||||
const v = emitter.emitExpr(s.value);
|
||||
if (s.arr.type.kind !== "array")
|
||||
throw new InternalCompilerError("llvm emitter bug: arraySet on non-array");
|
||||
if (emitter.checkedInt32Array(s.arr)) {
|
||||
// An int32 array with checked stores: the int32 arm stores an int32
|
||||
// value, and widens the array for any other first.
|
||||
emitter.emitPublishedArrayGuard(arr.name, s.arr.type, 0, idx.name);
|
||||
const elem = s.arr.type.elem;
|
||||
const proven = emitter.provenDenseIndex(s.arr, s.index);
|
||||
emitter.emitByElementKind(arr.name, null, (acc, widened) => {
|
||||
if (acc === "i32") emitter.emitCheckedInt32Store(arr.name, v.name, s.value, widened);
|
||||
emitDenseArraySet(emitter, arr.name, idx, s.index, acc, elem, v.name, proven);
|
||||
return null;
|
||||
});
|
||||
break;
|
||||
}
|
||||
const acc = emitter.arrayAccess(s.arr, s.arr.type.elem);
|
||||
const int32 = emitter.emitInt32KindStoreCheck(s.arr, v, s.value);
|
||||
emitter.emitPublishedArrayGuard(arr.name, s.arr.type, 0, idx.name);
|
||||
if (acc === "ref") emitter.toHeap(v);
|
||||
emitDenseArraySet(
|
||||
@@ -423,6 +444,7 @@ export function emitStmtBody(emitter: LlEmitter, s: IrStmt): void {
|
||||
s.arr.type.elem,
|
||||
v.name,
|
||||
emitter.provenDenseIndex(s.arr, s.index),
|
||||
int32,
|
||||
);
|
||||
break;
|
||||
}
|
||||
@@ -478,12 +500,19 @@ export function emitStmtBody(emitter: LlEmitter, s: IrStmt): void {
|
||||
B.condBr(dropsLast, pop, call);
|
||||
B.startBlock(pop);
|
||||
const elem = s.arr.type.elem;
|
||||
const acc = emitter.arrayAccess(s.arr, elem);
|
||||
const raw = emitDenseArrayPop(emitter, arr.name, acc, elem, call, call);
|
||||
if (acc === "ref" && isRefCounted(elem)) {
|
||||
const old = B.tmp();
|
||||
B.line(`${old} = inttoptr i64 ${raw} to ptr`);
|
||||
emitter.releaseHeap(old, elem);
|
||||
if (emitter.checkedInt32Array(s.arr)) {
|
||||
emitter.emitByElementKind(arr.name, null, (acc) => {
|
||||
emitDenseArrayPop(emitter, arr.name, acc, elem, call, call);
|
||||
return null;
|
||||
});
|
||||
} else {
|
||||
const acc = emitter.arrayAccess(s.arr, elem);
|
||||
const raw = emitDenseArrayPop(emitter, arr.name, acc, elem, call, call);
|
||||
if (acc === "ref" && isRefCounted(elem)) {
|
||||
const old = B.tmp();
|
||||
B.line(`${old} = inttoptr i64 ${raw} to ptr`);
|
||||
emitter.releaseHeap(old, elem);
|
||||
}
|
||||
}
|
||||
B.br(done);
|
||||
B.startBlock(call);
|
||||
@@ -2098,9 +2127,26 @@ export function emitLoopCopies(
|
||||
)
|
||||
: null;
|
||||
if (!plan) {
|
||||
const proven = allowed ? planProvenLoop(emitter, loop) : null;
|
||||
if (proven) emitProvenLoop(emitter, proven, exit, emitCopy);
|
||||
else emitCopy(exit);
|
||||
const ordinary = (exit: string): void => {
|
||||
const proven = allowed ? planProvenLoop(emitter, loop) : null;
|
||||
if (proven) emitProvenLoop(emitter, proven, exit, emitCopy);
|
||||
else emitCopy(exit);
|
||||
};
|
||||
const kinds =
|
||||
allowed &&
|
||||
emitter.speculativeLoopsEnabled &&
|
||||
emitter.int32Slots.checkedInt32Arrays.size > 0 &&
|
||||
emitter.int32Kinds === null &&
|
||||
emitter.speculation === null &&
|
||||
emitter.speculationResume === null
|
||||
? planInt32KindLoop(
|
||||
loop,
|
||||
(id) => emitter.checkedInt32Local(id),
|
||||
(e) => withinInt32(emitter.integerRanges.get(e)),
|
||||
)
|
||||
: null;
|
||||
if (kinds) emitInt32KindLoop(emitter, kinds, exit, kinds.writes.size > 0 ? emitCopy : ordinary);
|
||||
else ordinary(exit);
|
||||
return;
|
||||
}
|
||||
const B = emitter.B;
|
||||
@@ -2243,6 +2289,124 @@ export function emitLoopCopies(
|
||||
}
|
||||
}
|
||||
|
||||
/** An int32 kind loop (int32-kind-loops.ts). The entry test reads the kind
|
||||
* of each of the plan's arrays: when every one holds int32 slots the fast
|
||||
* copy runs, when every one holds doubles the wide copy, which reads and
|
||||
* writes them as double arrays, and otherwise (a plan with several arrays)
|
||||
* the mixed copy, which tests their kinds at each access. A failed store of
|
||||
* the fast copy re-executes in the third copy, which finishes its
|
||||
* iteration with kind tests and then enters the mixed copy, or the wide
|
||||
* copy when the plan has one array (it just widened). Each copy is emitted
|
||||
* by `ordinary`: the loop's plain copy when a store may fail, since a
|
||||
* proven integer loop keeps locals in shadows of its own copy that another
|
||||
* copy cannot resume from. */
|
||||
function emitInt32KindLoop(
|
||||
emitter: LlEmitter,
|
||||
plan: Int32KindLoopPlan,
|
||||
exit: string,
|
||||
ordinary: (exit: string) => void,
|
||||
): void {
|
||||
const B = emitter.B;
|
||||
const fn = emitter.currentFunctionName;
|
||||
const keys = new Set([...plan.arrays.keys()].map((id) => `${fn}\0${id}`));
|
||||
const several = keys.size > 1;
|
||||
let allInt32 = "true",
|
||||
allWide = "true";
|
||||
const and = (all: string, next: string): string => {
|
||||
if (all === "true") return next;
|
||||
const both = B.tmp();
|
||||
B.line(`${both} = and i1 ${all}, ${next}`);
|
||||
return both;
|
||||
};
|
||||
for (const ref of plan.arrays.values()) {
|
||||
const arr = emitter.emitStableReceiver(ref, []);
|
||||
const kindPtr = B.tmp(),
|
||||
kind = B.tmp(),
|
||||
isInt32 = B.tmp();
|
||||
B.line(
|
||||
`${kindPtr} = getelementptr inbounds %ScrArr, ptr ${arr.name}, i32 0, i32 ${SCR_ARR_FIELD.elem}`,
|
||||
);
|
||||
emitter.markMemoryPointer(kindPtr, "array:header");
|
||||
B.line(`${kind} = load i8, ptr ${kindPtr}${emitter.fieldAliasAttachment(kindPtr)}`);
|
||||
B.line(`${isInt32} = icmp eq i8 ${kind}, ${SCR_ELEM_I32}`);
|
||||
allInt32 = and(allInt32, isInt32);
|
||||
if (several) {
|
||||
const isWide = B.tmp();
|
||||
B.line(`${isWide} = xor i1 ${isInt32}, true`);
|
||||
allWide = and(allWide, isWide);
|
||||
}
|
||||
}
|
||||
const fast = B.newLabel("kind.fast"),
|
||||
wide = B.newLabel("kind.wide"),
|
||||
mixed = several ? B.newLabel("kind.mixed") : null;
|
||||
if (mixed === null) B.condBr(allInt32, fast, wide);
|
||||
else {
|
||||
const other = B.newLabel("kind.other");
|
||||
B.condBr(allInt32, fast, other);
|
||||
B.startBlock(other);
|
||||
B.condBr(allWide, wide, mixed);
|
||||
}
|
||||
const resume = new Map<IrStmt, { label: string; placed: boolean }>();
|
||||
for (const s of plan.writes) resume.set(s, { label: B.newLabel("kind.resume"), placed: false });
|
||||
// The header the third copy enters: the mixed copy's, or the wide copy's.
|
||||
let header: string | null = null;
|
||||
const capture = (label: string): void => {
|
||||
header = label;
|
||||
};
|
||||
B.startBlock(wide);
|
||||
if (resume.size > 0 && mixed === null) emitter.speculationHeader = capture;
|
||||
emitter.int32Kinds = { int32: new Set(), wide: keys, resume: new Map() };
|
||||
try {
|
||||
ordinary(exit);
|
||||
} finally {
|
||||
emitter.speculationHeader = null;
|
||||
emitter.int32Kinds = null;
|
||||
}
|
||||
if (mixed !== null) {
|
||||
B.startBlock(mixed);
|
||||
if (resume.size > 0) emitter.speculationHeader = capture;
|
||||
try {
|
||||
ordinary(exit);
|
||||
} finally {
|
||||
emitter.speculationHeader = null;
|
||||
}
|
||||
}
|
||||
if (resume.size > 0) {
|
||||
if (header === null)
|
||||
throw new InternalCompilerError("llvm emitter bug: int32 kind loop copy without a header");
|
||||
// A failed store finishes its iteration here, as in a speculative
|
||||
// integer loop's third copy.
|
||||
B.startBlock(B.newLabel("kind.tail"));
|
||||
emitter.speculationResume = resume;
|
||||
emitter.speculationHeader = () => {
|
||||
B.br(header!);
|
||||
B.startBlock(B.newLabel("kind.unused"));
|
||||
};
|
||||
try {
|
||||
ordinary(exit);
|
||||
} finally {
|
||||
emitter.speculationResume = null;
|
||||
emitter.speculationHeader = null;
|
||||
}
|
||||
}
|
||||
B.startBlock(fast);
|
||||
if (![...resume.values()].every((r) => r.placed)) {
|
||||
// A write the third copy never reached: this copy tests the kinds.
|
||||
ordinary(exit);
|
||||
return;
|
||||
}
|
||||
emitter.int32Kinds = {
|
||||
int32: keys,
|
||||
wide: new Set(),
|
||||
resume: new Map([...resume].map(([s, r]) => [s, r.label])),
|
||||
};
|
||||
try {
|
||||
ordinary(exit);
|
||||
} finally {
|
||||
emitter.int32Kinds = null;
|
||||
}
|
||||
}
|
||||
|
||||
/** At the top of a loop copy's header: lets a speculative loop find its
|
||||
* ordinary copy's header, and keeps its third copy out of the loop. */
|
||||
function enterLoopHeader(emitter: LlEmitter, header: string): void {
|
||||
|
||||
@@ -79,9 +79,18 @@ import {
|
||||
} from "./emit-journal.js";
|
||||
import { specializeNumericCalls } from "../../ir/numeric-call-specialization.js";
|
||||
import { versionLoopBoundParameters } from "../../ir/parameter-versions.js";
|
||||
import { allocatedLiteral, selectInt32Arrays } from "../../ir/int32-arrays.js";
|
||||
import {
|
||||
allocatedLiteral,
|
||||
fillAllocationLength,
|
||||
selectInt32Arrays,
|
||||
} from "../../ir/int32-arrays.js";
|
||||
import { everyExpr, everyExprChild, everyStmtChild, everyStmtList } from "../../ir/traverse.js";
|
||||
import { INT32_RANGE, type IntegerRange, type IntegerRanges } from "../../ir/integer-ranges.js";
|
||||
import {
|
||||
INT32_RANGE,
|
||||
withinInt32,
|
||||
type IntegerRange,
|
||||
type IntegerRanges,
|
||||
} from "../../ir/integer-ranges.js";
|
||||
import { unshareNodes } from "../../ir/unshare.js";
|
||||
import { copyStableCaptures } from "../../ir/capture-values.js";
|
||||
import { resolveSelfCaptures } from "../../ir/self-captures.js";
|
||||
@@ -101,7 +110,7 @@ import {
|
||||
} from "./int32-abi.js";
|
||||
import { type Int32Local } from "./int32-locals.js";
|
||||
import { findIntegerClones, NO_INTEGER_CLONES, type IntegerClones } from "./integer-clones.js";
|
||||
import { exactInteger, narrowInteger } from "./integer-values.js";
|
||||
import { exactInteger, narrowInteger, widenInteger } from "./integer-values.js";
|
||||
import { findInitializerBindings } from "../../ir/initializer-bindings.js";
|
||||
import { findConstantNumericTables, type ConstantNumericTable } from "../../ir/constant-tables.js";
|
||||
import {
|
||||
@@ -148,7 +157,12 @@ import {
|
||||
} from "./inline-unions.js";
|
||||
import { OptionalArrayReads, type LocalArrayRead } from "./local-array-reads.js";
|
||||
import { CheckedNarrows } from "./checked-narrows.js";
|
||||
import { emitDenseArraySet, emitDenseSlotUnlink, type ArrayAccess } from "./dense-array-access.js";
|
||||
import {
|
||||
emitDenseArraySet,
|
||||
emitDenseSlotUnlink,
|
||||
SCR_ELEM_I32,
|
||||
type ArrayAccess,
|
||||
} from "./dense-array-access.js";
|
||||
import { matchMapRead, type MapReadLifetimes } from "./map-read-lifetimes.js";
|
||||
import { type MapEntryFusion } from "./map-entry-fusion.js";
|
||||
import { emitBorrowedFieldSequence, guardedValue } from "./borrowed-receivers.js";
|
||||
@@ -169,7 +183,7 @@ import {
|
||||
} from "./string-slices.js";
|
||||
import { LazyCaptures } from "./lazy-captures.js";
|
||||
import { StackCaptures } from "./stack-captures.js";
|
||||
import { emitFieldAbsentTest, f64Lit } from "./common.js";
|
||||
import { emitFieldAbsentTest, f64Lit, SCR_ARR_FIELD } from "./common.js";
|
||||
import { ffiExtendsNarrowIntegers } from "../targets.js";
|
||||
import {
|
||||
emitLiteralExpr,
|
||||
@@ -484,13 +498,14 @@ function versionParameters(mod: IrModule): IrModule {
|
||||
return versionLoopBoundParameters(mod, mangleFunction);
|
||||
}
|
||||
|
||||
/** The int32 arrays the int32 slot analysis keeps in `mod`, under the
|
||||
* inputs LlEmitter gives it (selectInt32Arrays). */
|
||||
/** The int32 arrays the int32 slot analysis keeps in `mod` (proven and
|
||||
* checked), under the inputs LlEmitter gives it (selectInt32Arrays). */
|
||||
function provenInt32Arrays(mod: IrModule): ReadonlySet<string> {
|
||||
return analyzeInt32Slots(mod, {
|
||||
const slots = analyzeInt32Slots(mod, {
|
||||
dynamicStores: dynamicFieldStores(mod),
|
||||
initializerBindings: findInitializerBindings(mod),
|
||||
}).int32Arrays;
|
||||
});
|
||||
return new Set([...slots.int32Arrays, ...slots.checkedInt32Arrays]);
|
||||
}
|
||||
|
||||
/** The emitter for `input` after the module passes every emission runs. */
|
||||
@@ -738,25 +753,42 @@ export class LlEmitter {
|
||||
/** The fill temporaries (`new Array(n).fill(v)`) of those allocations:
|
||||
* the same arrays, so int32 too. */
|
||||
int32ArrayTemps = new Set<string>();
|
||||
/** The same for the current function's int32 arrays with checked stores,
|
||||
* which are only allocated by fills (ir/int32-arrays.ts
|
||||
* checkableInt32Allocations): their temporaries' literals, with the fill's
|
||||
* length (a literal constructs SCR_ELEM_I32 storage when that length is at
|
||||
* least CHECKED_INT32_MIN_LENGTH, SCR_ELEM_F64 otherwise), and their
|
||||
* temporaries. */
|
||||
checkedAllocations = new Map<IrExpr, IrExpr>();
|
||||
checkedArrayTemps = new Set<string>();
|
||||
/** The fill calls (`.fill(v)`) of those allocations: a value that is not
|
||||
* an int32 widens the new array first. */
|
||||
checkedFills = new Set<IrExpr>();
|
||||
|
||||
/** Collect the int32 allocations of `fn`, about to be emitted. */
|
||||
findInt32Allocations(fn: IrFunction): void {
|
||||
this.int32Allocations.clear();
|
||||
this.int32ArrayTemps.clear();
|
||||
if (this.int32Slots.int32Arrays.size === 0) return;
|
||||
this.checkedAllocations.clear();
|
||||
this.checkedArrayTemps.clear();
|
||||
this.checkedFills.clear();
|
||||
const { int32Arrays, checkedInt32Arrays } = this.int32Slots;
|
||||
if (int32Arrays.size === 0 && checkedInt32Arrays.size === 0) return;
|
||||
everyStmtList(fn.body, {
|
||||
stmt: (s) => {
|
||||
if (
|
||||
(s.kind === "varDecl" || s.kind === "assign") &&
|
||||
this.int32Slots.int32Arrays.has(`${fn.name}\0${s.localId}`)
|
||||
) {
|
||||
const value = s.kind === "varDecl" ? s.init : s.value;
|
||||
const allocation = value && allocatedLiteral(value);
|
||||
if (allocation) {
|
||||
this.int32Allocations.add(allocation.literal);
|
||||
if (allocation.temporary !== null) this.int32ArrayTemps.add(allocation.temporary);
|
||||
}
|
||||
}
|
||||
if (s.kind !== "varDecl" && s.kind !== "assign") return true;
|
||||
const key = `${fn.name}\0${s.localId}`;
|
||||
const checked = checkedInt32Arrays.has(key);
|
||||
if (!checked && !int32Arrays.has(key)) return true;
|
||||
const value = s.kind === "varDecl" ? s.init : s.value;
|
||||
const allocation = value && allocatedLiteral(value);
|
||||
if (!allocation) return true;
|
||||
const length = checked ? fillAllocationLength(value) : null;
|
||||
if (length !== null) this.checkedAllocations.set(allocation.literal, length);
|
||||
else if (!checked) this.int32Allocations.add(allocation.literal);
|
||||
if (allocation.temporary !== null)
|
||||
(checked ? this.checkedArrayTemps : this.int32ArrayTemps).add(allocation.temporary);
|
||||
if (checked && value !== allocation.literal) this.checkedFills.add(value!);
|
||||
return true;
|
||||
},
|
||||
expr: () => true,
|
||||
@@ -766,16 +798,174 @@ export class LlEmitter {
|
||||
/** Whether `arr` is one of the current function's int32 arrays
|
||||
* (ir/int32-arrays.ts, kept by the int32 slot analysis). */
|
||||
int32Array(arr: IrExpr): boolean {
|
||||
if (arr.kind !== "varRef" || this.int32Slots.int32Arrays.size === 0) return false;
|
||||
if (arr.kind !== "varRef") return false;
|
||||
if (this.int32KindArray(arr)) return true;
|
||||
if (this.int32Slots.int32Arrays.size === 0) return false;
|
||||
return (
|
||||
this.int32ArrayTemps.has(arr.localId) ||
|
||||
this.int32Slots.int32Arrays.has(`${this.currentFunctionName}\0${arr.localId}`)
|
||||
);
|
||||
}
|
||||
|
||||
/** Whether `arr` is one of the current function's int32 arrays with
|
||||
* checked stores: int32 slots until a store widens it to doubles. Every
|
||||
* access tests its kind (emitByElementKind), except in the fast copy of
|
||||
* an int32 kind loop, which reads and writes it as an int32 array. */
|
||||
checkedInt32Array(arr: IrExpr): boolean {
|
||||
if (arr.kind !== "varRef" || this.int32KindArray(arr)) return false;
|
||||
if (this.int32Kinds?.wide.has(`${this.currentFunctionName}\0${arr.localId}`)) return false;
|
||||
return this.checkedInt32Local(arr.localId);
|
||||
}
|
||||
|
||||
/** Whether the local `localId` holds one of the current function's int32
|
||||
* arrays with checked stores. */
|
||||
checkedInt32Local(localId: string): boolean {
|
||||
if (this.int32Slots.checkedInt32Arrays.size === 0) return false;
|
||||
return (
|
||||
this.checkedArrayTemps.has(localId) ||
|
||||
this.int32Slots.checkedInt32Arrays.has(`${this.currentFunctionName}\0${localId}`)
|
||||
);
|
||||
}
|
||||
|
||||
/** Whether `arr` is an int32 array with checked stores that the fast copy
|
||||
* of the int32 kind loop being emitted holds as an int32 array. */
|
||||
int32KindArray(arr: IrExpr): boolean {
|
||||
return (
|
||||
this.int32Kinds !== null &&
|
||||
arr.kind === "varRef" &&
|
||||
this.int32Kinds.int32.has(`${this.currentFunctionName}\0${arr.localId}`)
|
||||
);
|
||||
}
|
||||
|
||||
/** Before a store of `value` (of `expr`) into `arr`, in the fast copy of
|
||||
* an int32 kind loop: continue when it is an int32 that is not -0, and
|
||||
* otherwise leave for the statement's resume point, where it re-executes
|
||||
* with kind tests. Returns the value as an i32 when it is at hand as an
|
||||
* integer, or null when the store converts the double itself. */
|
||||
emitInt32KindStoreCheck(arr: IrExpr, value: LlValue, expr: IrExpr): string | null {
|
||||
if (!this.int32KindArray(arr) || withinInt32(this.integerRanges.get(expr))) return null;
|
||||
const integer = exactInteger(this, value, expr);
|
||||
if (integer !== null && withinInt32(integer.range)) return narrowInteger(this, integer);
|
||||
const resume = this.int32KindResume;
|
||||
if (resume === null)
|
||||
throw new InternalCompilerError("llvm emitter bug: an int32 kind store without a resume");
|
||||
const B = this.B;
|
||||
const ok = B.tmp(),
|
||||
int = B.tmp();
|
||||
if (integer !== null) {
|
||||
// An exact integer: a range test on its 64-bit form.
|
||||
const wide = widenInteger(this, integer),
|
||||
low = B.tmp(),
|
||||
high = B.tmp();
|
||||
B.line(`${low} = icmp sge i64 ${wide}, -2147483648`);
|
||||
B.line(`${high} = icmp sle i64 ${wide}, 2147483647`);
|
||||
B.line(`${ok} = and i1 ${low}, ${high}`);
|
||||
const store = B.newLabel("arr.store.i32");
|
||||
B.condBr(ok, store, resume);
|
||||
B.startBlock(store);
|
||||
B.line(`${int} = trunc i64 ${wide} to i32`);
|
||||
return int;
|
||||
}
|
||||
this.declare("declare i32 @llvm.fptosi.sat.i32.f64(double)");
|
||||
const back = B.tmp(),
|
||||
exact = B.tmp(),
|
||||
bits = B.tmp(),
|
||||
notNegativeZero = B.tmp();
|
||||
B.line(`${int} = call i32 @llvm.fptosi.sat.i32.f64(double ${value.name})`);
|
||||
B.line(`${back} = sitofp i32 ${int} to double`);
|
||||
B.line(`${exact} = fcmp oeq double ${back}, ${value.name}`);
|
||||
B.line(`${bits} = bitcast double ${value.name} to i64`);
|
||||
B.line(`${notNegativeZero} = icmp ne i64 ${bits}, -9223372036854775808`);
|
||||
B.line(`${ok} = and i1 ${exact}, ${notNegativeZero}`);
|
||||
const store = B.newLabel("arr.store.i32");
|
||||
B.condBr(ok, store, resume);
|
||||
B.startBlock(store);
|
||||
return int;
|
||||
}
|
||||
|
||||
/** Emit an access to the checked int32 array `array` once per element
|
||||
* kind: `arm("i32", widened)` while it holds int32 slots and
|
||||
* `arm("f64", widened)` after it widened, joined after both. `widened` is
|
||||
* the label of the double arm, where an int32 arm's store of a value that
|
||||
* is not an int32 continues after widening the array. Returns the phi of
|
||||
* the arms' results (of LLVM type `ty`), or null when `ty` is null. */
|
||||
emitByElementKind(
|
||||
array: string,
|
||||
ty: string | null,
|
||||
arm: (acc: "i32" | "f64", widened: string) => string | null,
|
||||
): string | null {
|
||||
const B = this.B;
|
||||
const narrow = B.newLabel("arr.kind.i32"),
|
||||
wide = B.newLabel("arr.kind.f64"),
|
||||
join = B.newLabel("arr.kind.join");
|
||||
const kindPtr = B.tmp(),
|
||||
kind = B.tmp(),
|
||||
isInt32 = B.tmp();
|
||||
B.line(
|
||||
`${kindPtr} = getelementptr inbounds %ScrArr, ptr ${array}, i32 0, i32 ${SCR_ARR_FIELD.elem}`,
|
||||
);
|
||||
this.markMemoryPointer(kindPtr, "array:header");
|
||||
B.line(`${kind} = load i8, ptr ${kindPtr}${this.fieldAliasAttachment(kindPtr)}`);
|
||||
B.line(`${isInt32} = icmp eq i8 ${kind}, ${SCR_ELEM_I32}`);
|
||||
B.condBr(isInt32, narrow, wide);
|
||||
B.startBlock(narrow);
|
||||
const narrowValue = arm("i32", wide);
|
||||
const narrowEnd = B.isTerminated() ? null : B.currentLabel();
|
||||
if (narrowEnd !== null) B.br(join);
|
||||
B.startBlock(wide);
|
||||
const wideValue = arm("f64", wide);
|
||||
const wideEnd = B.isTerminated() ? null : B.currentLabel();
|
||||
if (wideEnd !== null) B.br(join);
|
||||
B.startBlock(join);
|
||||
// Arms that end in their own branches (a read's shared join) leave this
|
||||
// join without predecessors.
|
||||
if (narrowEnd === null && wideEnd === null) B.terminate("unreachable");
|
||||
if (ty === null) return null;
|
||||
const arms: string[] = [];
|
||||
if (narrowEnd !== null) arms.push(`[ ${narrowValue}, %${narrowEnd} ]`);
|
||||
if (wideEnd !== null) arms.push(`[ ${wideValue}, %${wideEnd} ]`);
|
||||
const result = B.tmp();
|
||||
B.line(`${result} = phi ${ty} ${arms.join(", ")}`);
|
||||
return result;
|
||||
}
|
||||
|
||||
/** Before storing the double `value` (of `expr`) into an int32 arm
|
||||
* (emitByElementKind) of the checked int32 array `array`: continue in this
|
||||
* arm when it is an int32 that is not -0, or widen the array and continue
|
||||
* at `widened`. A value proven an int32 needs no test. */
|
||||
emitCheckedInt32Store(array: string, value: string, expr: IrExpr, widened: string): void {
|
||||
if (withinInt32(this.integerRanges.get(expr))) return;
|
||||
const B = this.B;
|
||||
this.declare("declare i32 @llvm.fptosi.sat.i32.f64(double)");
|
||||
this.declare("declare void @scr_arr_widen_i32(ptr)");
|
||||
const int = B.tmp(),
|
||||
back = B.tmp(),
|
||||
exact = B.tmp(),
|
||||
bits = B.tmp(),
|
||||
notNegativeZero = B.tmp(),
|
||||
ok = B.tmp();
|
||||
B.line(`${int} = call i32 @llvm.fptosi.sat.i32.f64(double ${value})`);
|
||||
B.line(`${back} = sitofp i32 ${int} to double`);
|
||||
B.line(`${exact} = fcmp oeq double ${back}, ${value}`);
|
||||
B.line(`${bits} = bitcast double ${value} to i64`);
|
||||
B.line(`${notNegativeZero} = icmp ne i64 ${bits}, -9223372036854775808`);
|
||||
B.line(`${ok} = and i1 ${exact}, ${notNegativeZero}`);
|
||||
const store = B.newLabel("arr.store.i32"),
|
||||
widen = B.newLabel("arr.widen");
|
||||
B.condBr(ok, store, widen);
|
||||
B.startBlock(widen);
|
||||
B.line(`call void @scr_arr_widen_i32(ptr ${array})`);
|
||||
B.br(widened);
|
||||
B.startBlock(store);
|
||||
}
|
||||
|
||||
/** The access an element operation on `arr` (of element type `elem`)
|
||||
* takes: int32 arrays read and write their 4-byte slots. */
|
||||
arrayAccess(arr: IrExpr, elem: IrType): ArrayAccess {
|
||||
if (this.checkedInt32Array(arr))
|
||||
throw new InternalCompilerError(
|
||||
"llvm emitter bug: a checked int32 array without its kind test",
|
||||
);
|
||||
return this.int32Array(arr) ? "i32" : elemAccess(elem);
|
||||
}
|
||||
|
||||
@@ -1225,6 +1415,14 @@ export class LlEmitter {
|
||||
* (`enterLoopHeader`): records the ordinary copy's header, or sends the
|
||||
* third copy's header there. */
|
||||
speculationHeader: ((header: string) => void) | null = null;
|
||||
/** A copy of an int32 kind loop (int32-kind-loops.ts) being emitted: the
|
||||
* int32 arrays with checked stores it reads and writes as int32 arrays
|
||||
* (the fast copy) or as double arrays (an ordinary copy entered only
|
||||
* after they widened), as `function\0local` keys, and where each of the
|
||||
* fast copy's writes resumes in the third copy. */
|
||||
int32Kinds: { int32: Set<string>; wide: Set<string>; resume: Map<IrStmt, string> } | null = null;
|
||||
/** The resume label of the fast-copy write being emitted. */
|
||||
int32KindResume: string | null = null;
|
||||
/** The function being emitted, for speculative globals. */
|
||||
speculativeFunction = "";
|
||||
/** Every function that writes each number global (computed on demand). */
|
||||
@@ -5317,10 +5515,14 @@ export class LlEmitter {
|
||||
const loop =
|
||||
s.kind === "while" || s.kind === "doWhile" || s.kind === "for" || s.kind === "forOf";
|
||||
if (loop) this.loopDepth++;
|
||||
// An int32 kind loop's fast-copy write leaves for its resume point.
|
||||
const kindResume = this.int32KindResume;
|
||||
this.int32KindResume = this.int32Kinds?.resume.get(s) ?? null;
|
||||
try {
|
||||
this.emitStmtBody(s);
|
||||
} finally {
|
||||
this.B.debugLocation = previous;
|
||||
this.int32KindResume = kindResume;
|
||||
if (loop) this.loopDepth--;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -615,9 +615,12 @@ export function emitArrIntrinsic(
|
||||
throw new InternalCompilerError("llvm emitter bug: arrIntrinsic on non-array");
|
||||
const elem = e.receiver.type.elem;
|
||||
// Int32 arrays (ir/int32-arrays.ts) reach only these methods; the others
|
||||
// take the runtime family's flat access.
|
||||
const access = host.arrayAccess(e.receiver, elem);
|
||||
if (access === "i32" && !INT32_ARRAY_METHODS.has(e.method))
|
||||
// take the runtime family's flat access. One with checked stores tests
|
||||
// its kind at each inline access (emitByElementKind) and otherwise takes
|
||||
// the double entry points, which serve both kinds.
|
||||
const checked = host.checkedInt32Array(e.receiver);
|
||||
const access = checked ? "f64" : host.arrayAccess(e.receiver, elem);
|
||||
if ((access === "i32" || checked) && !INT32_ARRAY_METHODS.has(e.method))
|
||||
throw new InternalCompilerError(`llvm emitter bug: ${e.method} on an int32 array`);
|
||||
const acc = runtimeAccess(access);
|
||||
const accTy = acc === "f64" ? "double" : acc === "bool" ? "i1" : "ptr";
|
||||
@@ -640,6 +643,13 @@ export function emitArrIntrinsic(
|
||||
e.receiver.kind === "varRef"
|
||||
? host.constantNumericTables.get(e.receiver.localId)
|
||||
: undefined;
|
||||
if (!table && elem.kind === "f64" && checked) {
|
||||
const proven = host.provenDenseIndex(e.receiver, e.args[0]!);
|
||||
const value = host.emitByElementKind(r.name, "double", (kind) =>
|
||||
emitDenseArrayNumber(host, r.name, index, e.args[0], proven, kind),
|
||||
)!;
|
||||
return { name: value, type: e.type };
|
||||
}
|
||||
if (!table && elem.kind === "f64")
|
||||
return {
|
||||
name: emitDenseArrayNumber(
|
||||
@@ -685,6 +695,21 @@ export function emitArrIntrinsic(
|
||||
// no-op zero-argument call.
|
||||
const vs = e.args.map((a) => host.emitExpr(a));
|
||||
host.emitPublishedArrayGuard(r.name, e.receiver.type, 1, f64Lit(0), f64Lit(vs.length));
|
||||
if (checked && vs.length > 0) {
|
||||
// Each value appends in its own kind test: one that is not an int32
|
||||
// widens the array before it is pushed.
|
||||
let length = "";
|
||||
vs.forEach((v, i) => {
|
||||
length = host.emitByElementKind(r.name, "double", (kind, widened) => {
|
||||
if (kind === "i32") host.emitCheckedInt32Store(r.name, v.name, e.args[i]!, widened);
|
||||
return arrPush(host, r.name, kind, v.name, elem);
|
||||
})!;
|
||||
});
|
||||
return { name: length, type: e.type };
|
||||
}
|
||||
// In an int32 kind loop's fast copy, every value is checked before
|
||||
// the first appends.
|
||||
vs.forEach((v, i) => host.emitInt32KindStoreCheck(e.receiver, v, e.args[i]!));
|
||||
return { name: emitArrayValues(host, r.name, access, elem, vs), type: e.type };
|
||||
}
|
||||
case "pushSpread": {
|
||||
@@ -759,7 +784,13 @@ export function emitArrIntrinsic(
|
||||
let slowLabel = "";
|
||||
if (method === "pop") {
|
||||
slowLabel = B.newLabel("remove.slow");
|
||||
denseRaw = emitDenseArrayPop(host, r.name, access, elem, la, slowLabel);
|
||||
const empty = la,
|
||||
slow = slowLabel;
|
||||
denseRaw = checked
|
||||
? host.emitByElementKind(r.name, "i64", (kind) =>
|
||||
emitDenseArrayPop(host, r.name, kind, elem, empty, slow),
|
||||
)!
|
||||
: emitDenseArrayPop(host, r.name, access, elem, la, slowLabel);
|
||||
denseLabel = B.currentLabel();
|
||||
B.br(lp);
|
||||
B.startBlock(slowLabel);
|
||||
@@ -948,6 +979,14 @@ export function emitArrIntrinsic(
|
||||
case "fillUndefined": {
|
||||
const args = e.args.map((arg) => host.emitExpr(arg));
|
||||
host.emitPublishedArrayGuard(r.name, e.receiver.type, 7);
|
||||
if (host.checkedFills.has(e)) {
|
||||
// A new int32 array with checked stores, filled with a value that
|
||||
// may not be an int32: widen it first then.
|
||||
const filled = B.newLabel("arr.fill.kind");
|
||||
host.emitCheckedInt32Store(r.name, args[0]!.name, e.args[0]!, filled);
|
||||
B.br(filled);
|
||||
B.startBlock(filled);
|
||||
}
|
||||
const helper =
|
||||
e.method === "copyWithin"
|
||||
? "scr_arr_copy_within"
|
||||
|
||||
@@ -8,6 +8,7 @@ import { LlvmUnsupportedError } from "./unsupported.js";
|
||||
import type { LlvmEmitterContext, ExprOf, LlValue } from "./expr-context.js";
|
||||
|
||||
import { BYTES_ELEM_NUM, closureIdentityEqual, f64Lit } from "./common.js";
|
||||
import { CHECKED_INT32_MIN_LENGTH } from "../../ir/int32-arrays.js";
|
||||
import { emitStringInputs } from "./string-lifetimes.js";
|
||||
import { emitStringParts, numberPart, stringParts } from "./string-construction.js";
|
||||
import { emitBorrowedInputs } from "./borrowed-inputs.js";
|
||||
@@ -904,7 +905,30 @@ export function emitContainerExpr(
|
||||
const firstSpread = e.elems.findIndex((_, i) => spreadSet.has(i));
|
||||
const capacity = small ? 0 : (e.capacity ?? (firstSpread < 0 ? e.elems.length : firstSpread));
|
||||
// An int32 array's allocation (ir/int32-arrays.ts) constructs
|
||||
// SCR_ELEM_I32 storage; its elements are proven int32s.
|
||||
// SCR_ELEM_I32 storage; its elements are proven int32s. The empty
|
||||
// temporary of a fill that allocates an int32 array with checked
|
||||
// stores takes SCR_ELEM_I32 storage when the fill's length (which
|
||||
// evaluates without effects) is at least CHECKED_INT32_MIN_LENGTH, and
|
||||
// SCR_ELEM_F64 storage otherwise.
|
||||
const fillLength = host.checkedAllocations.get(e);
|
||||
if (fillLength !== undefined && e.elems.length === 0) {
|
||||
const length = host.emitExpr(fillLength);
|
||||
const long = B.tmp(),
|
||||
kind = B.tmp();
|
||||
B.line(`${long} = fcmp oge double ${length.name}, ${f64Lit(CHECKED_INT32_MIN_LENGTH)}`);
|
||||
B.line(`${kind} = select i1 ${long}, i32 ${SCR_ELEM_I32}, i32 0`);
|
||||
host.declare(
|
||||
small
|
||||
? `declare ptr @scr_arr_new_small(i32)`
|
||||
: `declare ptr @scr_arr_new(i32, ${host.sizeType})`,
|
||||
);
|
||||
B.line(
|
||||
small
|
||||
? `${arr} = call ptr @scr_arr_new_small(i32 ${kind})`
|
||||
: `${arr} = call ptr @scr_arr_new(i32 ${kind}, ${host.sizeType} ${capacity})`,
|
||||
);
|
||||
return host.own({ name: arr, type: e.type });
|
||||
}
|
||||
const int32 = host.int32Allocations.has(e);
|
||||
if (int32) {
|
||||
host.declare(
|
||||
@@ -986,6 +1010,26 @@ export function emitContainerExpr(
|
||||
// Ref-element reads return +1 (the inline dense path retains like
|
||||
// the runtime getter); own registers the owned temp in the frame.
|
||||
const elem = e.arr.type.elem;
|
||||
if (host.checkedInt32Array(e.arr)) {
|
||||
const proven = host.provenDenseIndex(e.arr, e.index);
|
||||
const value = host.emitByElementKind(
|
||||
arr.name,
|
||||
"double",
|
||||
(kind) =>
|
||||
emitDenseArrayRead(
|
||||
host,
|
||||
arr.name,
|
||||
idx,
|
||||
e.index,
|
||||
kind,
|
||||
elem,
|
||||
"scr_arr_get_f64",
|
||||
true,
|
||||
proven,
|
||||
).value,
|
||||
)!;
|
||||
return { name: value, type: e.type };
|
||||
}
|
||||
const acc = host.arrayAccess(e.arr, elem);
|
||||
const read = emitDenseArrayRead(
|
||||
host,
|
||||
@@ -1013,14 +1057,18 @@ export function emitContainerExpr(
|
||||
const idx = host.emitExpr(e.index);
|
||||
if (e.arr.type.kind !== "array")
|
||||
throw new InternalCompilerError("llvm emitter bug: arrayHas on non-array");
|
||||
const t = emitDenseArrayState(
|
||||
host,
|
||||
arr.name,
|
||||
idx,
|
||||
e.index,
|
||||
"has",
|
||||
host.int32Array(e.arr) ? slotWidth("i32") : denseSlotWidth(host, e.arr.type.elem),
|
||||
);
|
||||
const t = host.checkedInt32Array(e.arr)
|
||||
? host.emitByElementKind(arr.name, "i1", (kind) =>
|
||||
emitDenseArrayState(host, arr.name, idx, e.index, "has", slotWidth(kind)),
|
||||
)!
|
||||
: emitDenseArrayState(
|
||||
host,
|
||||
arr.name,
|
||||
idx,
|
||||
e.index,
|
||||
"has",
|
||||
host.int32Array(e.arr) ? slotWidth("i32") : denseSlotWidth(host, e.arr.type.elem),
|
||||
);
|
||||
return { name: t, type: e.type };
|
||||
}
|
||||
case "arrayState": {
|
||||
@@ -1028,14 +1076,18 @@ export function emitContainerExpr(
|
||||
const idx = host.emitExpr(e.index);
|
||||
if (e.arr.type.kind !== "array")
|
||||
throw new InternalCompilerError("llvm emitter bug: arrayState on non-array");
|
||||
const t = emitDenseArrayState(
|
||||
host,
|
||||
arr.name,
|
||||
idx,
|
||||
e.index,
|
||||
"state",
|
||||
host.int32Array(e.arr) ? slotWidth("i32") : denseSlotWidth(host, e.arr.type.elem),
|
||||
);
|
||||
const t = host.checkedInt32Array(e.arr)
|
||||
? host.emitByElementKind(arr.name, "double", (kind) =>
|
||||
emitDenseArrayState(host, arr.name, idx, e.index, "state", slotWidth(kind)),
|
||||
)!
|
||||
: emitDenseArrayState(
|
||||
host,
|
||||
arr.name,
|
||||
idx,
|
||||
e.index,
|
||||
"state",
|
||||
host.int32Array(e.arr) ? slotWidth("i32") : denseSlotWidth(host, e.arr.type.elem),
|
||||
);
|
||||
return { name: t, type: e.type };
|
||||
}
|
||||
case "arrIntrinsic":
|
||||
|
||||
@@ -0,0 +1,135 @@
|
||||
import type { IrExpr, IrStmt } from "../../ir/ir.js";
|
||||
import { everyExprChild, everyStmtChild } from "../../ir/traverse.js";
|
||||
import { MAX_LOOP_NODES, SIMPLE_STATEMENTS } from "./speculative-integers.js";
|
||||
|
||||
/** Int32 kind loops.
|
||||
*
|
||||
* An int32 array with checked stores (ir/int32-slots.ts) holds int32 slots
|
||||
* (or doubles, when its fill was short) until a store of a value that is
|
||||
* not an int32 widens it to doubles in place, so an access to it tests its
|
||||
* element kind first (LlEmitter.emitByElementKind). An innermost loop that
|
||||
* accesses such arrays tests their kinds once on entry instead
|
||||
* (emitLoopCopies). The fast copy runs when every one of them holds int32
|
||||
* slots and reads and writes them as int32 arrays; the wide copy runs when
|
||||
* every one holds doubles and accesses them as double arrays; a loop over
|
||||
* several arrays whose kinds differ runs the mixed copy, which tests the
|
||||
* kinds at each access. A store of the fast copy whose value is not an
|
||||
* int32 branches to the same statement in a third copy, which re-executes
|
||||
* it with kind tests (widening the array) and finishes the iteration, then
|
||||
* enters the mixed copy, or the wide copy when the loop has one array.
|
||||
* Re-execution is exact because such a store's statement evaluates nothing
|
||||
* but locals, literals, number arithmetic and element reads before it
|
||||
* writes.
|
||||
*
|
||||
* Nothing else can widen these arrays: they never reach code other than
|
||||
* the compiled element operations of their own function
|
||||
* (ir/int32-arrays.ts). */
|
||||
export interface Int32KindLoopPlan {
|
||||
/** The arrays the fast copy reads and writes as int32 arrays, by local,
|
||||
* with one of their references in the loop. */
|
||||
arrays: Map<string, IrExpr>;
|
||||
/** Element writes and pushes whose value may not be an int32: the
|
||||
* statements the fast copy may leave at. */
|
||||
writes: Set<IrStmt>;
|
||||
}
|
||||
|
||||
type KindLoop = Extract<IrStmt, { kind: "for" | "while" | "doWhile" }>;
|
||||
|
||||
/** A number or boolean expression whose evaluation has no effect: locals,
|
||||
* literals, arithmetic, comparisons and element reads of local arrays. */
|
||||
function isPure(e: IrExpr): boolean {
|
||||
if (e.type.kind !== "f64" && e.type.kind !== "bool") return false;
|
||||
switch (e.kind) {
|
||||
case "numLit":
|
||||
case "boolLit":
|
||||
case "varRef":
|
||||
return true;
|
||||
case "bin":
|
||||
return (
|
||||
(e.left.type.kind === "f64" || e.left.type.kind === "bool") &&
|
||||
isPure(e.left) &&
|
||||
isPure(e.right)
|
||||
);
|
||||
case "unary":
|
||||
return isPure(e.operand);
|
||||
case "ternary":
|
||||
return isPure(e.cond) && isPure(e.then) && isPure(e.else_);
|
||||
case "arrayGet":
|
||||
case "arrayHas":
|
||||
return e.arr.kind === "varRef" && isPure(e.index);
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/** The plan for `loop`, or null. `checked` names the int32 array locals
|
||||
* with checked stores of the function; `provenInt32` the values the static
|
||||
* range analysis proves int32s (stores that never leave the fast copy). */
|
||||
export function planInt32KindLoop(
|
||||
loop: KindLoop,
|
||||
checked: (localId: string) => boolean,
|
||||
provenInt32: (e: IrExpr) => boolean,
|
||||
): Int32KindLoopPlan | null {
|
||||
const arrays = new Map<string, IrExpr>();
|
||||
const writes = new Set<IrStmt>();
|
||||
const assigned = new Set<string>();
|
||||
let nodes = 0;
|
||||
let rejected = false;
|
||||
const reject = (): false => {
|
||||
rejected = true;
|
||||
return false;
|
||||
};
|
||||
const array = (e: IrExpr): string | null =>
|
||||
e.kind === "varRef" && checked(e.localId) ? e.localId : null;
|
||||
const push = (e: IrExpr): (IrExpr & { kind: "arrIntrinsic" }) | null =>
|
||||
e.kind === "arrIntrinsic" && e.method === "push" && array(e.receiver) !== null ? e : null;
|
||||
|
||||
const expr = (e: IrExpr): boolean => {
|
||||
if (rejected) return false;
|
||||
nodes++;
|
||||
if (e.kind === "varRef" && checked(e.localId) && !arrays.has(e.localId))
|
||||
arrays.set(e.localId, e);
|
||||
// A push the statement walk did not take as a write: its values must
|
||||
// be proven int32s, since it cannot leave the fast copy.
|
||||
const pushed = push(e);
|
||||
if (pushed && !pushed.args.every(provenInt32)) return reject();
|
||||
return everyExprChild(e, expr, nested);
|
||||
};
|
||||
// Statements inside expressions run mid-expression, where the fast copy
|
||||
// cannot leave: they may not write these arrays.
|
||||
const nested = (s: IrStmt): boolean => {
|
||||
nodes++;
|
||||
if (s.kind === "assign" || s.kind === "varDecl") assigned.add(s.localId);
|
||||
if (s.kind === "arraySet" && array(s.arr) !== null && !provenInt32(s.value)) return reject();
|
||||
return everyStmtChild(s, expr, nested);
|
||||
};
|
||||
const stmt = (s: IrStmt): boolean => {
|
||||
if (rejected) return false;
|
||||
nodes++;
|
||||
if (!SIMPLE_STATEMENTS.has(s.kind)) return reject();
|
||||
if (s.kind === "assign" || s.kind === "varDecl") assigned.add(s.localId);
|
||||
if (s.kind === "arraySet" && array(s.arr) !== null && !provenInt32(s.value)) {
|
||||
if (!isPure(s.index) || !isPure(s.value)) return reject();
|
||||
writes.add(s);
|
||||
return everyStmtChild(s, expr, stmt);
|
||||
}
|
||||
const pushed = s.kind === "exprStmt" ? push(s.expr) : null;
|
||||
if (pushed && !pushed.args.every(provenInt32)) {
|
||||
if (!pushed.args.every(isPure)) return reject();
|
||||
writes.add(s);
|
||||
expr(pushed.receiver);
|
||||
pushed.args.forEach(expr);
|
||||
return !rejected;
|
||||
}
|
||||
return everyStmtChild(s, expr, stmt);
|
||||
};
|
||||
|
||||
if (loop.cond) expr(loop.cond);
|
||||
if (loop.kind === "for" && loop.update) stmt(loop.update);
|
||||
for (const s of loop.body) stmt(s);
|
||||
if (rejected || nodes > MAX_LOOP_NODES || arrays.size === 0) return null;
|
||||
// A loop that rebinds one of the arrays could reach another array of
|
||||
// either kind.
|
||||
for (const id of arrays.keys()) if (assigned.has(id)) return null;
|
||||
return { arrays, writes };
|
||||
}
|
||||
@@ -767,7 +767,37 @@ export function emitLocalArrayRead(
|
||||
}
|
||||
B.line(`store i32 ${read.presentTag}, ptr ${tag}`);
|
||||
};
|
||||
if (inline)
|
||||
if (inline && scalar && host.checkedInt32Array(read.array))
|
||||
// An int32 array with checked stores: its int32 arm stores the element's
|
||||
// double bits, as the double arm does.
|
||||
host.emitByElementKind(array.name, null, (kind) => {
|
||||
const present =
|
||||
kind === "i32"
|
||||
? (value: string): void => {
|
||||
const number = B.tmp(),
|
||||
bits = B.tmp();
|
||||
B.line(`${number} = sitofp i32 ${value} to double`);
|
||||
B.line(`${bits} = bitcast double ${number} to i64`);
|
||||
storeValue(bits);
|
||||
}
|
||||
: storeValue;
|
||||
const loadType = kind === "i32" ? "i32" : "i64";
|
||||
const induction = loopIndex !== null;
|
||||
emitDenseReferenceArrayRead(
|
||||
host,
|
||||
array,
|
||||
index,
|
||||
integerIndex,
|
||||
induction,
|
||||
present,
|
||||
no,
|
||||
slow,
|
||||
join,
|
||||
loadType,
|
||||
);
|
||||
return null;
|
||||
});
|
||||
else if (inline)
|
||||
emitDenseReferenceArrayRead(
|
||||
host,
|
||||
array,
|
||||
|
||||
@@ -35,12 +35,12 @@ export interface SpeculativeLoopPlan {
|
||||
type SpeculativeLoop = Extract<IrStmt, { kind: "for" | "while" | "doWhile" }>;
|
||||
|
||||
/** The fast copy duplicates the loop; keep the duplicated code small. */
|
||||
const MAX_LOOP_NODES = 400;
|
||||
export const MAX_LOOP_NODES = 400;
|
||||
|
||||
/** Statements whose emission is plain control flow or a single write.
|
||||
* Nested loops, switches and try statements keep the loop on the ordinary
|
||||
* path (their emission keeps state that the two copies would not share). */
|
||||
const SIMPLE_STATEMENTS = new Set<string>([
|
||||
export const SIMPLE_STATEMENTS: ReadonlySet<string> = new Set<string>([
|
||||
"varDecl",
|
||||
"assign",
|
||||
"exprStmt",
|
||||
|
||||
@@ -3,6 +3,7 @@ import { reparseExpr } from "./serialize.js";
|
||||
import {
|
||||
everyExprChild,
|
||||
everyStmtChild,
|
||||
everyStmtList,
|
||||
transformExpr,
|
||||
transformStmt,
|
||||
transformStmtList,
|
||||
@@ -23,9 +24,12 @@ import {
|
||||
* Candidates are only proposed here. The int32 slot analysis
|
||||
* (int32-slots.ts) keeps the ones whose every stored element is a proven
|
||||
* int32 (never -0), assuming the candidates' own elements are int32s, at
|
||||
* a fixpoint, and the LLVM backend stores those as SCR_ELEM_I32. Their
|
||||
* element-reader helper calls are inlined first (rewriteCandidates), so
|
||||
* no shared helper ever receives an int32 array. */
|
||||
* a fixpoint, and the LLVM backend stores those as SCR_ELEM_I32. It keeps
|
||||
* the ones whose stores are exact integers but not all proven int32s as
|
||||
* int32 arrays with checked stores: SCR_ELEM_I32 until a store that is not
|
||||
* an int32 widens them to doubles in place. Their element-reader helper
|
||||
* calls are inlined first (rewriteCandidates), so no shared helper ever
|
||||
* receives an int32 array. */
|
||||
|
||||
/** Array methods a candidate may be the receiver of: the runtime paths
|
||||
* behind them read and write int32 slots through the double-valued slot
|
||||
@@ -87,6 +91,62 @@ function fillAllocation(e: IrExpr): (IrExpr & { kind: "arrIntrinsic" }) | null {
|
||||
return e;
|
||||
}
|
||||
|
||||
/** Fill-form arrays shorter than this hold doubles even when they are int32
|
||||
* arrays with checked stores: their doubles fit a typical 32 KiB L1 data
|
||||
* cache, so 4-byte slots save no memory traffic. */
|
||||
export const CHECKED_INT32_MIN_LENGTH = 4096;
|
||||
|
||||
/** The length `n` of `new Array(n).fill(v)`, or null for any other
|
||||
* allocation. */
|
||||
export function fillAllocationLength(e: IrExpr): IrExpr | null {
|
||||
const fill = fillAllocation(e);
|
||||
if (!fill || fill.receiver.kind !== "seqExpr") return null;
|
||||
const set = fill.receiver.stmts[1]!;
|
||||
return set.kind === "arraySetLength" ? set.length : null;
|
||||
}
|
||||
|
||||
/** A number expression whose evaluation has no effect: locals, literals and
|
||||
* arithmetic. */
|
||||
function isPureLength(e: IrExpr): boolean {
|
||||
if (e.type.kind !== "f64") return false;
|
||||
switch (e.kind) {
|
||||
case "numLit":
|
||||
case "varRef":
|
||||
return true;
|
||||
case "bin":
|
||||
return e.left.type.kind === "f64" && isPureLength(e.left) && isPureLength(e.right);
|
||||
case "unary":
|
||||
return isPureLength(e.operand);
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/** Whether the candidate `id` of `fn` may hold int32 slots with checked
|
||||
* stores: every array it is assigned is the fill form, with a length that
|
||||
* may reach CHECKED_INT32_MIN_LENGTH and evaluates without effects (the
|
||||
* backend evaluates it once more to pick the new array's kind). */
|
||||
export function checkableInt32Allocations(fn: IrFunction, id: string): boolean {
|
||||
let ok = true;
|
||||
everyStmtList(fn.body, {
|
||||
stmt: (s) => {
|
||||
if ((s.kind === "varDecl" || s.kind === "assign") && s.localId === id) {
|
||||
const value = s.kind === "varDecl" ? s.init : s.value;
|
||||
const length = value ? fillAllocationLength(value) : null;
|
||||
if (
|
||||
length === null ||
|
||||
!isPureLength(length) ||
|
||||
(length.kind === "numLit" && length.value < CHECKED_INT32_MIN_LENGTH)
|
||||
)
|
||||
ok = false;
|
||||
}
|
||||
return ok;
|
||||
},
|
||||
expr: () => ok,
|
||||
});
|
||||
return ok;
|
||||
}
|
||||
|
||||
/** A fresh number array: a literal without spreads, or the fill form. */
|
||||
export function int32ArrayAllocation(e: IrExpr): boolean {
|
||||
return (e.kind === "arrayLit" && (e.spreads?.length ?? 0) === 0) || fillAllocation(e) !== null;
|
||||
|
||||
@@ -10,7 +10,7 @@ import {
|
||||
} from "./integer-ranges.js";
|
||||
import { exprFunctionReference, moduleFunctionReferences } from "./function-references.js";
|
||||
import { findInitializerBindings, withInitializerBindings } from "./initializer-bindings.js";
|
||||
import { int32ArrayStores } from "./int32-arrays.js";
|
||||
import { checkableInt32Allocations, int32ArrayStores } from "./int32-arrays.js";
|
||||
import { everyExprChild, everyStmtChild, everyStmtList, everyTypeChild } from "./traverse.js";
|
||||
|
||||
/** The storage slot an array-valued expression reads, as an
|
||||
@@ -251,6 +251,15 @@ export class Int32Slots {
|
||||
* module's int32 array candidates whose every stored element the
|
||||
* fixpoint proved an int32. */
|
||||
readonly int32Arrays = new Set<string>();
|
||||
/** Local number arrays that may hold int32 slots with checked stores: the
|
||||
* candidates whose element stores the fixpoint proved exact integers but
|
||||
* not int32s, allocated only by fills that may be long. A fill of at least
|
||||
* CHECKED_INT32_MIN_LENGTH elements creates one with int32 slots and a
|
||||
* shorter one with doubles. Their elements give no int32 facts; compiled
|
||||
* accesses test the array's kind (once per innermost loop where they can,
|
||||
* backend/llvm/int32-kind-loops.ts), and a store that is not an int32
|
||||
* widens it first (scr_arr_widen_i32). */
|
||||
readonly checkedInt32Arrays = new Set<string>();
|
||||
/** Whether every present element of the number array `arr` evaluates to
|
||||
* (read inside `fnName`) is an int32. */
|
||||
hasInt32Elements(fnName: string, arr: IrExpr): boolean {
|
||||
@@ -1108,8 +1117,13 @@ export function analyzeInt32Slots(mod: IrModule, inputs: Int32SlotInputs = {}):
|
||||
const int32 = (e: IrExpr): boolean => withinInt32(ranges.get(e));
|
||||
// Every element stored into an int32 array candidate must be an int32
|
||||
// under the facts that assume the candidates' own elements are; a
|
||||
// failing store drops its array, and the function is analyzed again
|
||||
// without it.
|
||||
// failing store drops its array's element facts, and the function is
|
||||
// analyzed again without them. When every failing store is still an
|
||||
// exact integer under those facts (a counter, `a[i]! + 1`) and the array
|
||||
// is only ever allocated by a fill that may be long
|
||||
// (checkableInt32Allocations), it keeps int32 slots with checked stores
|
||||
// (checkedInt32Arrays), which widen it to doubles in place at the first
|
||||
// store that is not an int32.
|
||||
const arrays = arrayCandidates.get(fn.name);
|
||||
if (arrays) {
|
||||
let dropped = false;
|
||||
@@ -1117,7 +1131,10 @@ export function analyzeInt32Slots(mod: IrModule, inputs: Int32SlotInputs = {}):
|
||||
const key = `l:${fn.name}\0${id}`;
|
||||
if (!slots.hasInt32ElementSlot(key) || values.every(int32)) continue;
|
||||
slots.dropInt32Elements(key);
|
||||
reason(`array:${fn.name}\0${id}`, "an element store is not int32");
|
||||
const checkable = (value: IrExpr): boolean => int32(value) || ranges.get(value) != null;
|
||||
if (values.every(checkable) && checkableInt32Allocations(fn, id))
|
||||
slots.checkedInt32Arrays.add(`${fn.name}\0${id}`);
|
||||
else reason(`array:${fn.name}\0${id}`, "an element store is not int32");
|
||||
dropped = true;
|
||||
}
|
||||
if (dropped && !queued.has(fn.name)) {
|
||||
|
||||
@@ -1076,7 +1076,8 @@ export interface IrModule {
|
||||
/** Local number arrays (`function\0local`) the backend may store as
|
||||
* int32 slots, proposed by ir/int32-arrays.ts after their element-reader
|
||||
* calls were inlined; the int32 slot analysis keeps the ones whose every
|
||||
* stored element it proves an int32. Absent when there is none. */
|
||||
* stored element it proves an int32, and those whose stores are exact
|
||||
* integers with checked stores. Absent when there is none. */
|
||||
int32ArrayCandidates?: string[];
|
||||
/** The WebAssembly modules translated into this program (wasm/
|
||||
* translate.ts), present exactly when the program uses the WebAssembly
|
||||
|
||||
@@ -63,6 +63,16 @@ function negativeZero(): number {
|
||||
return 1 / a[1]!;
|
||||
}
|
||||
function counter(n: number): number {
|
||||
const a = new Array(n).fill(0) as number[];
|
||||
for (let i = 0; i < 1000; i++) a[i % n] = a[i % n]! + 1;
|
||||
return a[0]!;
|
||||
}
|
||||
function shortCounter(): number {
|
||||
const a = new Array(16).fill(0) as number[];
|
||||
for (let i = 0; i < 1000; i++) a[i % 16] = a[i % 16]! + 1;
|
||||
return a[0]!;
|
||||
}
|
||||
function literalCounter(n: number): number {
|
||||
const a = [0];
|
||||
for (let i = 0; i < n; i++) a[0] = a[0]! + 1;
|
||||
return a[0]!;
|
||||
@@ -72,7 +82,16 @@ function joined(n: number): string {
|
||||
return a.join(",");
|
||||
}
|
||||
const { n } = JSON.parse('{"n": 8}') as { n: number };
|
||||
console.log(compose(n | 0, 3), counts(n | 0), fraction(n), negativeZero(), counter(n), joined(n));
|
||||
console.log(
|
||||
compose(n | 0, 3),
|
||||
counts(n | 0),
|
||||
fraction(n),
|
||||
negativeZero(),
|
||||
counter(n),
|
||||
shortCounter(),
|
||||
literalCounter(n),
|
||||
joined(n),
|
||||
);
|
||||
`;
|
||||
|
||||
/** Every function emitted for the source function \`name\` (its int32 ABI
|
||||
@@ -100,15 +119,64 @@ test("arrays storing only int32 elements get 4-byte slots", async () => {
|
||||
expect(counts).not.toContain("popArray");
|
||||
});
|
||||
|
||||
test("arrays with a store that may not be an int32, or that escape, keep doubles", async () => {
|
||||
test("arrays with a store that has no exact integer range, or that escape, keep doubles", async () => {
|
||||
const llvm = await emitLlvm(source);
|
||||
for (const name of ["fraction", "negativeZero", "counter", "joined"]) {
|
||||
for (const name of ["fraction", "negativeZero", "joined"]) {
|
||||
const fn = functions(llvm, name);
|
||||
expect(fn, name).not.toBe("");
|
||||
expect(fn, name).not.toMatch(/@scr_arr_new(_small)?\(i32 7/);
|
||||
expect(fn, name).not.toContain("arr.kind.");
|
||||
}
|
||||
});
|
||||
|
||||
test("a counter array keeps int32 slots with checked stores", async () => {
|
||||
const llvm = await emitLlvm(source);
|
||||
const counter = functions(llvm, "counter");
|
||||
// `a[k] = a[k]! + 1` is an exact integer but no int32 proof: a fill of at
|
||||
// least 4096 elements creates SCR_ELEM_I32 storage (SCR_ELEM_F64 for a
|
||||
// shorter one), accesses test the kind, and a store widens the array
|
||||
// before a value that is not an int32.
|
||||
expect(counter).toMatch(
|
||||
/fcmp oge double %\w+, [^\n]+\n\s+%\w+ = select i1 %\w+, i32 7, i32 0\n\s+%\w+ = call ptr @scr_arr_new(_small)?\(i32 %\w+/,
|
||||
);
|
||||
expect(counter).toMatch(/load i8, ptr %\w+[^\n]*\n\s+%\w+ = icmp eq i8 %\w+, 7/);
|
||||
expect(counter).toContain("@llvm.fptosi.sat.i32.f64");
|
||||
expect(counter).toContain("call void @scr_arr_widen_i32(");
|
||||
// Arrays with proven stores take no kind test, and a fill of a constant
|
||||
// short length or an array literal keeps doubles.
|
||||
for (const name of ["compose", "counts", "shortCounter", "literalCounter"]) {
|
||||
const fn = functions(llvm, name);
|
||||
expect(fn, name).not.toContain("arr.kind.");
|
||||
expect(fn, name).not.toContain("kind.fast");
|
||||
expect(fn, name).not.toContain("scr_arr_widen_i32");
|
||||
}
|
||||
for (const name of ["shortCounter", "literalCounter"])
|
||||
expect(functions(llvm, name), name).not.toMatch(/@scr_arr_new(_small)?\(i32 7/);
|
||||
});
|
||||
|
||||
test("a loop over a counter array tests its kind once on entry", async () => {
|
||||
const llvm = await emitLlvm(source);
|
||||
const counter = functions(llvm, "counter");
|
||||
// The loop's fast copy runs while the array holds int32 slots, with no
|
||||
// kind test of its own: the sum is an integer, and a sum past the int32
|
||||
// range leaves for the third copy, which re-executes the store with kind
|
||||
// tests and widens the array.
|
||||
expect(counter).toMatch(
|
||||
/icmp eq i8 %\w+, 7\n\s+br i1 %\w+, label %kind\.fast\d+, label %kind\.wide/,
|
||||
);
|
||||
const rest = counter.slice(counter.search(/^kind\.fast\d+:/m));
|
||||
// The fast copy is the last one, before the loop's exit.
|
||||
const fast = rest.slice(0, rest.search(/^loop\.e\d+:/m));
|
||||
expect(fast).not.toBe("");
|
||||
expect(fast).not.toMatch(/icmp eq i8 %\w+, 7/);
|
||||
expect(fast).toMatch(/add nsw i64 %\w+, 1/);
|
||||
expect(fast).toMatch(
|
||||
/icmp sle i64 %\w+, 2147483647\n[^\n]*\n\s+br i1 %\w+, label %arr\.store\.i32\d+, label %kind\.resume\d+/,
|
||||
);
|
||||
expect(fast).toMatch(/store i32 %\w+, ptr %\w+, !tbaa/);
|
||||
expect(counter).toMatch(/^kind\.resume\d+:/m);
|
||||
});
|
||||
|
||||
test("only the candidates the proof keeps are rewritten", async () => {
|
||||
const mod = await lowerIr(source);
|
||||
const proposed = markInt32ArrayCandidates(mod).int32ArrayCandidates ?? [];
|
||||
|
||||
+6
@@ -0,0 +1,6 @@
|
||||
{
|
||||
"order": [
|
||||
"<repo>/tests/corpus/7327-int32-array-widening.ts"
|
||||
],
|
||||
"diags": []
|
||||
}
|
||||
@@ -579,6 +579,36 @@ static void scr_arr_move_inline(ScrArr *a, size_t cap) {
|
||||
a->spare &= (uint16_t)~SCR_ARR_INLINE_SLOTS;
|
||||
}
|
||||
|
||||
/* Widen an int32 array (SCR_ELEM_I32) to doubles in place, one way: its
|
||||
* dense values move to 8-byte slots, with their states when it has them, and
|
||||
* its kind becomes SCR_ELEM_F64. Compiled code calls it before storing a
|
||||
* value that is not an int32 into an int32 array with checked stores; `data`
|
||||
* may move. Sparse entries already hold doubles. */
|
||||
void scr_arr_widen_i32(ScrArr *a) {
|
||||
if (a->elem != SCR_ELEM_I32) return;
|
||||
size_t cap = a->cap;
|
||||
if (cap > 0) {
|
||||
/* Int32 arrays never shift, so they have no head room; inline slots
|
||||
* have no ext prefix. */
|
||||
bool inline_slots = scr_arr_inline(a);
|
||||
bool states = scr_arr_has_states(a);
|
||||
size_t unit = sizeof(uint64_t) + (states ? 1 : 0);
|
||||
size_t prefix = inline_slots ? 0 : scr_arr_prefix(a);
|
||||
if (cap > (SIZE_MAX - prefix) / unit) scr_arr_oom();
|
||||
char *start = scr_mem_alloc(prefix + cap * unit);
|
||||
if (!start) scr_arr_oom();
|
||||
if (prefix) memcpy(start, scr_arr_alloc_start(a), prefix);
|
||||
uint64_t *data = (uint64_t *)(start + prefix);
|
||||
const int32_t *old = (const int32_t *)a->data;
|
||||
for (size_t i = 0; i < cap; i++) data[i] = scr_arr_f64_slot((double)old[i]);
|
||||
if (states) memcpy((uint8_t *)data + cap * sizeof(uint64_t), scr_arr_states_end(a) - cap, cap);
|
||||
if (!inline_slots && a->rc != SIZE_MAX) scr_mem_free(scr_arr_alloc_start(a));
|
||||
a->data = data;
|
||||
a->spare &= (uint16_t)~SCR_ARR_INLINE_SLOTS;
|
||||
}
|
||||
a->elem = SCR_ELEM_F64;
|
||||
}
|
||||
|
||||
/* How a dense allocation is sized. */
|
||||
typedef enum {
|
||||
SCR_ARR_GROW_APPEND, /* geometric headroom for further appends */
|
||||
|
||||
@@ -433,6 +433,9 @@ ScrArr *scr_arr_new_ops(const ScrArrOps *ops, size_t initial_cap);
|
||||
* array plus five 8-byte slots). Otherwise exactly scr_arr_new(elem, 0) and
|
||||
* scr_arr_new_ops(ops, 0). Returns +1. */
|
||||
ScrArr *scr_arr_new_small(ScrElemKind elem);
|
||||
/* An int32 array's storage as doubles, in place (SCR_ELEM_I32 to
|
||||
* SCR_ELEM_F64); no-op for any other kind. */
|
||||
void scr_arr_widen_i32(ScrArr *a);
|
||||
ScrArr *scr_arr_new_ops_small(const ScrArrOps *ops);
|
||||
/* scr_arr_new(elem, cap) for a known final length: the `cap` slots share
|
||||
* the array's own block (one allocation). With `states`, the array starts
|
||||
|
||||
@@ -1670,6 +1670,44 @@ static void test_i32_slots(void) {
|
||||
check(i32_layout_ok(k) && scr_arr_get_f64(k, 6) == 6 && scr_arr_get_f64(k, 0) == 0,
|
||||
"a known-count int32 array grows out of its block");
|
||||
scr_arr_release(k);
|
||||
|
||||
/* Widening: in place, to 8-byte double slots, keeping states. */
|
||||
ScrArr *w = scr_arr_new(SCR_ELEM_I32, 3);
|
||||
for (int i = 0; i < 3; i++) scr_arr_push_f64(w, i - 1);
|
||||
check(inline_slots(w), "a three-element int32 construction keeps inline slots");
|
||||
scr_arr_widen_i32(w);
|
||||
check(w->elem == SCR_ELEM_F64 && !inline_slots(w) && scr_arr_len(w) == 3 && w->filled == 3 &&
|
||||
scr_arr_get_f64(w, 0) == -1 && scr_arr_get_f64(w, 2) == 1,
|
||||
"widening moves inline int32 slots out as doubles");
|
||||
scr_arr_set_f64(w, 1, 2.5);
|
||||
check(scr_arr_get_f64(w, 1) == 2.5, "a widened array stores fractions");
|
||||
scr_arr_widen_i32(w);
|
||||
check(w->elem == SCR_ELEM_F64 && scr_arr_get_f64(w, 1) == 2.5, "widening twice is a no-op");
|
||||
scr_arr_release(w);
|
||||
|
||||
ScrArr *h = scr_arr_new_small(SCR_ELEM_I32);
|
||||
for (int i = 0; i < 100; i++) scr_arr_push_f64(h, i * 3);
|
||||
scr_arr_set_len(h, 120);
|
||||
scr_arr_set_undefined(h, 5);
|
||||
scr_arr_set_f64(h, 4000000, -7);
|
||||
size_t cap = h->cap;
|
||||
scr_arr_widen_i32(h);
|
||||
bool widened = h->elem == SCR_ELEM_F64 && h->cap == cap && scr_arr_len(h) == 4000001 &&
|
||||
scr_arr_states_end(h) == (const uint8_t *)h->data + 9 * (size_t)h->cap;
|
||||
for (int i = 0; i < 100; i++)
|
||||
if (i != 5) widened &= scr_arr_get_f64(h, i) == i * 3;
|
||||
check(widened && scr_arr_state(h, 5) == SCR_ARR_UNDEFINED && scr_arr_state(h, 110) == SCR_ARR_HOLE &&
|
||||
scr_arr_get_f64(h, 4000000) == -7 && arr_sparse_len(h) == 1,
|
||||
"widening keeps values, present undefined, holes and sparse entries");
|
||||
scr_arr_push_f64(h, 0.5);
|
||||
check(scr_arr_get_f64(h, 4000001) == 0.5, "a widened array keeps growing as doubles");
|
||||
scr_arr_release(h);
|
||||
|
||||
ScrArr *e = scr_arr_new_small(SCR_ELEM_I32);
|
||||
scr_arr_widen_i32(e);
|
||||
scr_arr_push_f64(e, -0.0);
|
||||
check(e->elem == SCR_ELEM_F64 && signbit(scr_arr_get_f64(e, 0)), "an empty int32 array widens too");
|
||||
scr_arr_release(e);
|
||||
}
|
||||
|
||||
static void test_inline_slots(void) {
|
||||
|
||||
@@ -0,0 +1,186 @@
|
||||
// Local number arrays with checked int32 stores: `new Array(n).fill(v)`
|
||||
// arrays whose stores are exact integers but not proven int32s. A fill of
|
||||
// at least 4096 elements creates int32 slots, and the first store past the
|
||||
// int32 range widens the array to doubles in place; a shorter fill creates
|
||||
// doubles. Each case runs at both sizes. Arrays with a store that has no
|
||||
// exact integer range (a product of two int32s, a fraction, a value that may
|
||||
// be -0), and arrays allocated as literals, keep doubles throughout.
|
||||
|
||||
function counters(n: number, rounds: number): string {
|
||||
const c: number[] = new Array(n).fill(0);
|
||||
for (let r = 0; r < rounds; r++)
|
||||
for (let i = 0; i < c.length; i++) c[i] = c[i]! + ((i & 3) + 1) * 1000000;
|
||||
return `counters ${n} ${c[0]} ${c[1]} ${c[2]} ${c[3]} ${c[n - 1]} ${c.length}`;
|
||||
}
|
||||
|
||||
// The store at i = 45 passes 2^31 and widens the array partway through a
|
||||
// loop; the remaining passes store doubles.
|
||||
function denseLoop(n: number, k: number): string {
|
||||
const a: number[] = new Array(n).fill(0);
|
||||
for (let i = 0; i < a.length; i++) a[i] = i + k + 2147483600;
|
||||
let s = 0;
|
||||
for (let i = 0; i < a.length; i++) s = (s + a[i]!) % 1000003;
|
||||
return `dense ${n} ${a[44]} ${a[45]} ${a[n - 1]} ${s}`;
|
||||
}
|
||||
|
||||
// Checked stores whose values stay int32: no widening.
|
||||
function bfs(n: number): string {
|
||||
const dist: number[] = new Array(n).fill(-1);
|
||||
const queue: number[] = [0];
|
||||
dist[0] = 0;
|
||||
for (let h = 0; h < queue.length; h++) {
|
||||
const u = queue[h]!;
|
||||
const edges = [(u + 1) & (n - 1), (u * 5 + 1) & (n - 1)];
|
||||
for (let j = 0; j < edges.length; j++) {
|
||||
const v = edges[j]!;
|
||||
if (dist[v]! < 0) {
|
||||
dist[v] = dist[u]! + 1;
|
||||
queue.push(v);
|
||||
}
|
||||
}
|
||||
}
|
||||
let far = 0,
|
||||
sum = 0;
|
||||
for (let i = 0; i < n; i++) {
|
||||
if (dist[i]! > far) far = dist[i]!;
|
||||
sum += dist[i]!;
|
||||
}
|
||||
return `bfs ${n} ${far} ${sum} ${queue.length}`;
|
||||
}
|
||||
|
||||
// A fill value that is not an int32 widens the new array before the fill.
|
||||
function filledPastInt32(n: number, k: number): string {
|
||||
const a: number[] = new Array(n).fill(k * 1000000000);
|
||||
a[1] = a[1]! + 1;
|
||||
const b: number[] = new Array(n).fill(k);
|
||||
b[2] = b[2]! * 2;
|
||||
return `filled ${n} ${a[0]} ${a[1]} ${a.length} ${b[0]} ${b[2]}`;
|
||||
}
|
||||
|
||||
// A chain of sums crosses 2^31 partway through; the second loop runs after
|
||||
// the array widened.
|
||||
function chain(n: number, step: number): string {
|
||||
const a: number[] = new Array(n).fill(0);
|
||||
for (let i = 1; i < n; i++) a[i] = a[i - 1]! + (i < 16 ? step : 1);
|
||||
let s = 0;
|
||||
for (let i = 0; i < n; i++) s = (s + (a[i]! % 1000003)) % 1000003;
|
||||
for (let i = 1; i < n; i++) a[i] = a[i]! - a[i - 1]!;
|
||||
return `chain ${n} ${step} ${a[1]} ${a[5]} ${a[6]} ${a[n - 1]} ${s}`;
|
||||
}
|
||||
|
||||
// Two arrays in one loop: one widens partway through, the other keeps
|
||||
// int32 slots; then both are read in another loop.
|
||||
function twoArrays(n: number, m: number, k: number): string {
|
||||
const small: number[] = new Array(n).fill(1);
|
||||
const big: number[] = new Array(m).fill(k);
|
||||
const count = Math.min(n, m);
|
||||
for (let i = 1; i < count; i++) {
|
||||
small[i] = small[i - 1]! + 1;
|
||||
big[i] = big[i - 1]! + (i < 8 ? 700000000 : 1);
|
||||
}
|
||||
let s = 0,
|
||||
t = 0;
|
||||
for (let i = 0; i < count; i++) {
|
||||
s += small[i]!;
|
||||
t = (t + (big[i]! % 997)) % 997;
|
||||
}
|
||||
return `two ${n} ${m} ${small[count - 1]} ${big[2]} ${big[3]} ${big[count - 1]} ${s} ${t}`;
|
||||
}
|
||||
|
||||
// Pushes that pass 2^31 widen the array before they append.
|
||||
function pushes(n: number, k: number): string {
|
||||
const a: number[] = new Array(n).fill(k);
|
||||
for (let i = 0; i < 8; i++) a.push(a[a.length - 1]! + 900000000);
|
||||
let s = 0;
|
||||
for (let i = 0; i < a.length; i++) s = (s + (a[i]! % 10007)) % 10007;
|
||||
return `pushes ${n} ${a.length} ${a[n]} ${a[n + 2]} ${a[n + 7]} ${s}`;
|
||||
}
|
||||
|
||||
function controlFlow(n: number, k: number): string {
|
||||
const a: number[] = new Array(n).fill(k);
|
||||
let stores = 0;
|
||||
for (let i = 1; i < n; i++) {
|
||||
if (i % 2 === 0) continue;
|
||||
a[i] = a[i - 1]! + 1000000000 * (i % 4);
|
||||
stores++;
|
||||
if (a[i]! > 2500000000) break;
|
||||
}
|
||||
let i = 0;
|
||||
while (i < n && a[i]! < 2000000000) i++;
|
||||
let j = n;
|
||||
do j--;
|
||||
while (j > 0 && a[j] === k);
|
||||
return `flow ${n} ${stores} ${a[1]} ${a[2]} ${a[3]} ${i} ${j}`;
|
||||
}
|
||||
|
||||
function readOnly(n: number, wide: boolean): string {
|
||||
const a: number[] = new Array(n).fill(2);
|
||||
if (wide) a[0] = a[0]! + 3000000000;
|
||||
let s = 0,
|
||||
present = 0;
|
||||
for (let i = 0; i < n + 2; i++) {
|
||||
s += a[i] ?? 0;
|
||||
if (i in a) present++;
|
||||
}
|
||||
return `read ${n} ${wide} ${s} ${present}`;
|
||||
}
|
||||
|
||||
function holes(n: number): string {
|
||||
// Holes, present undefined and sparse entries survive widening.
|
||||
const a: number[] = new Array(n).fill(3);
|
||||
a[n + 3] = a[1]! + 1;
|
||||
a[n + 2] = a[n + 3]! - 1;
|
||||
a[2 * n + 100000] = a[0]! + 1;
|
||||
a[n + 1] = a[n + 2]! + 3000000000;
|
||||
return `holes ${n} ${a.length} ${a[n]} ${n in a} ${a[n + 1]} ${a[n + 2]} ${a[2 * n + 100000]}`;
|
||||
}
|
||||
|
||||
// These arrays never take int32 slots: a fractional store, a product of
|
||||
// two int32s, -0, an escape, and arrays allocated as literals.
|
||||
function neverInt32(n: number, x: number, y: number): string[] {
|
||||
const f: number[] = new Array(n).fill(1);
|
||||
f[0] = f[0]! / x;
|
||||
const m: number[] = new Array(n).fill(3);
|
||||
m[1] = m[1]! * y;
|
||||
const z: number[] = new Array(n).fill(4);
|
||||
z[1] = z[1]! % 2;
|
||||
z[2] = -z[0]! % 2;
|
||||
const j: number[] = new Array(n).fill(3);
|
||||
j[0] = j[0]! + 1;
|
||||
const lit = [1, 2, 3];
|
||||
lit[0] = lit[0]! + 3000000000;
|
||||
const pushed: number[] = [];
|
||||
for (let i = 0; i < 4; i++) pushed.push(i * 1000000000);
|
||||
pushed[0] = pushed[3]! + pushed[2]!;
|
||||
return [
|
||||
`never ${f[0]} ${m[1]} ${z[1]} ${Object.is(z[2], -0)} ${j.slice(0, 2).join("|")}`,
|
||||
`literals ${lit[0]} ${lit[2]} ${pushed[0]} ${pushed.length}`,
|
||||
];
|
||||
}
|
||||
|
||||
const config = JSON.parse('{"small": 64, "large": 8192, "k": 3, "y": 2000000000}') as {
|
||||
small: number;
|
||||
large: number;
|
||||
k: number;
|
||||
y: number;
|
||||
};
|
||||
const lines: string[] = [];
|
||||
for (const n of [config.small | 0, config.large | 0]) {
|
||||
lines.push(counters(n, 1000));
|
||||
lines.push(denseLoop(n, config.k | 0));
|
||||
lines.push(bfs(n));
|
||||
lines.push(filledPastInt32(n, config.k | 0));
|
||||
lines.push(chain(n, 400000000));
|
||||
lines.push(chain(n, 100000000));
|
||||
lines.push(twoArrays(n, n, config.k | 0));
|
||||
lines.push(pushes(n, config.k | 0));
|
||||
lines.push(controlFlow(n, config.k | 0));
|
||||
lines.push(readOnly(n, false));
|
||||
lines.push(readOnly(n, true));
|
||||
lines.push(holes(n));
|
||||
}
|
||||
// One long array and one short: their kinds differ on entry to the loop.
|
||||
lines.push(twoArrays(config.large | 0, config.small | 0, config.k | 0));
|
||||
lines.push(twoArrays(config.small | 0, config.large | 0, config.k | 0));
|
||||
lines.push(...neverInt32(config.small | 0, config.k, config.y | 0));
|
||||
for (const line of lines) console.log(line);
|
||||
@@ -0,0 +1 @@
|
||||
{ "plain": 4500, "san": 6100 }
|
||||
Reference in New Issue
Block a user