diff --git a/packages/cli/test/native-link-info.test.ts b/packages/cli/test/native-link-info.test.ts index 39964898d..57c60b1bc 100644 --- a/packages/cli/test/native-link-info.test.ts +++ b/packages/cli/test/native-link-info.test.ts @@ -72,7 +72,7 @@ describe.runIf(supported)("macOS arm64 native link info", () => { minimum_os: "14.0", }, program: { object, entry_symbol: "main" }, - runtime_abi: { version: 22, marker: "scr_runtime_abi_v22" }, + runtime_abi: { version: 23, marker: "scr_runtime_abi_v23" }, runtime_pack: { kind: "precompiled", package: "@scriptc/runtime-darwin-arm64", @@ -165,6 +165,6 @@ describe.runIf(supported)("macOS arm64 native link info", () => { () => null, (failure: { stderr?: string }) => failure, ); - expect(error?.stderr).toContain("scr_runtime_abi_v22"); + expect(error?.stderr).toContain("scr_runtime_abi_v23"); }); }); diff --git a/packages/compiler/src/backend/llvm/array-search.ts b/packages/compiler/src/backend/llvm/array-search.ts index 8c9590bee..74d79f7be 100644 --- a/packages/compiler/src/backend/llvm/array-search.ts +++ b/packages/compiler/src/backend/llvm/array-search.ts @@ -63,7 +63,9 @@ export function emitArraySearchHelpers( `define internal double @${arraySearchHelperName(kind)}(ptr %a, ptr %v) ${FN_ATTRS} {`, `entry:`, ` %lenp = getelementptr inbounds %ScrArr, ptr %a, i32 0, i32 1`, - ` %len = load ${sz}, ptr %lenp${sz === "i64" ? ", !range !{i64 0, i64 4294967296}" : ""}`, + ...(sz === "i64" + ? [` %len32 = load i32, ptr %lenp`, ` %len = zext i32 %len32 to i64`] + : [` %len = load i32, ptr %lenp`]), ` %fp = getelementptr inbounds %ScrArr, ptr %a, i32 0, i32 6`, ` %f32 = load i32, ptr %fp`, widen, diff --git a/packages/compiler/src/backend/llvm/common.ts b/packages/compiler/src/backend/llvm/common.ts index 8a8778919..e6fc56f6f 100644 --- a/packages/compiler/src/backend/llvm/common.ts +++ b/packages/compiler/src/backend/llvm/common.ts @@ -10,7 +10,9 @@ import type { import type { LlvmEmitterContext } from "./expr-context.js"; /** Field numbers of %ScrArr, which mirrors scr_runtime.h's ScrArr field for - * field (its static assertions pin `cyc`, `filled` and `data`). */ + * field (its static assertions pin `len`, `cap`, `cyc`, `filled` and + * `data`). `len` and `cap` are i32 on every target; the dense states end + * the allocation after the `cap` value slots at `data` (scrArrStates). */ export const SCR_ARR_FIELD = { rc: 0, len: 1, @@ -21,10 +23,92 @@ export const SCR_ARR_FIELD = { filled: 6, ops: 7, data: 8, - present: 9, - ext: 10, + ext: 9, } as const; +/** Loads the i32 `len` or `cap` of a %ScrArr at `ptr` into `dest` as a + * size-type count. */ +export function loadArrCount( + host: Pick, + dest: string, + ptr: string, + attach = "", +): void { + const B = host.B; + if (host.sizeType === "i32") { + B.line(`${dest} = load i32, ptr ${ptr}${attach}`); + return; + } + const raw = B.tmp(); + B.line(`${raw} = load i32, ptr ${ptr}${attach}`); + B.line(`${dest} = zext i32 ${raw} to ${host.sizeType}`); +} + +/** Stores the size-type count `value` (at most 2^32 - 1) into the i32 `len` + * or `cap` of a %ScrArr at `ptr`. */ +export function storeArrCount( + host: Pick, + value: string, + ptr: string, + attach = "", +): void { + const B = host.B; + let narrow = value; + if (host.sizeType !== "i32") { + narrow = B.tmp(); + B.line(`${narrow} = trunc ${host.sizeType} ${value} to i32`); + } + B.line(`store i32 ${narrow}, ptr ${ptr}${attach}`); +} + +/** The end of the dense state bytes of a %ScrArr into `dest` + * (scr_arr_states_end): `cap` value slots of `width` bytes at `data`, then + * one state byte per slot in reverse slot order. `cap` is a size-type count. */ +export function scrArrStates( + host: Pick, + dest: string, + data: string, + cap: string, + width: number, +): void { + host.B.line( + `${dest} = getelementptr inbounds [${width + 1} x i8], ptr ${data}, ${host.sizeType} ${cap}`, + ); +} + +/** The state byte of dense slot `offset` into `dest`, given the end of the + * states (scrArrStates): `end - 1 - offset`. */ +export function scrArrStatePtr( + host: Pick, + dest: string, + end: string, + offset: string, +): void { + const B = host.B; + const back = B.tmp(); + B.line(`${back} = xor ${host.sizeType} ${offset}, -1`); + B.line(`${dest} = getelementptr inbounds i8, ptr ${end}, ${host.sizeType} ${back}`); +} + +/** The state byte of dense slot `offset` of a %ScrArr into `dest`, from its + * `data` and size-type `cap`: scrArrStates and scrArrStatePtr in one + * address computation, for paths that read a single state. */ +export function scrArrStateAt( + host: Pick, + dest: string, + data: string, + cap: string, + width: number, + offset: string, +): void { + const B = host.B; + const back = B.tmp(); + B.line(`${back} = xor ${host.sizeType} ${offset}, -1`); + B.line( + `${dest} = getelementptr inbounds [${width + 1} x i8], ptr ${data}, ${host.sizeType} ${cap}, ${host.sizeType} ${back}`, + ); +} + /** Lines that decide whether the double `index` is an integer in * `[0, length)`, where `length` is an unsigned `sizeType` count, and give * that integer as an i64. `%{ok}` is the answer; `%{wide}` is meaningful diff --git a/packages/compiler/src/backend/llvm/dense-array-access.test.ts b/packages/compiler/src/backend/llvm/dense-array-access.test.ts index 6da427841..1b84cf0a0 100644 --- a/packages/compiler/src/backend/llvm/dense-array-access.test.ts +++ b/packages/compiler/src/backend/llvm/dense-array-access.test.ts @@ -170,6 +170,10 @@ test.each([32, 64] as const)( expect(fn.match(new RegExp(`icmp ult ${size} `, "g"))).toHaveLength(3); expect(fn).toMatch(/store double 0x4004000000000000, ptr /); expect(fn).toMatch(/zext i1 .* to i8/); + // Per push: data + (cap - len), then the state byte `width * cap - 1` + // past it; plus the boolean value slot. + expect(fn.match(/getelementptr inbounds \[8 x i8\], ptr .* -1$/gm)).toHaveLength(2); + expect(fn.match(/getelementptr inbounds \[1 x i8\], ptr .* -1$/gm)).toHaveLength(1); expect(fn.match(/getelementptr inbounds i8, ptr /g)).toHaveLength(4); expect(fn.match(/store i8 1, ptr /g)).toHaveLength(3); expect(fn.match(new RegExp(`add nuw ${size} .*, 1$`, "gm"))).toHaveLength(3); diff --git a/packages/compiler/src/backend/llvm/dense-array-access.ts b/packages/compiler/src/backend/llvm/dense-array-access.ts index d02d7f7c3..4488f90cb 100644 --- a/packages/compiler/src/backend/llvm/dense-array-access.ts +++ b/packages/compiler/src/backend/llvm/dense-array-access.ts @@ -17,7 +17,15 @@ * whose writes extend the bound when they continue it. */ import { isRefCounted, type IrExpr, type IrType } from "../../ir/ir.js"; -import { f64Lit, SCR_ARR_FIELD, sizeToDouble } from "./common.js"; +import { + f64Lit, + loadArrCount, + SCR_ARR_FIELD, + scrArrStatePtr, + scrArrStates, + sizeToDouble, + storeArrCount, +} from "./common.js"; import type { LlValue, LlvmEmitterContext } from "./expr-context.js"; import { exactInteger, widenInteger } from "./integer-values.js"; @@ -36,7 +44,8 @@ interface DenseGuard { len: string; /** The hole-free prefix bound loaded by the guard (i32). */ filled: string; - /** Dense value and state storage base pointers. */ + /** The dense value storage base pointer, and the end of the states + * (scrArrStates). */ data: string; states: string; } @@ -48,7 +57,9 @@ interface DenseGuard { * below the hole-free prefix bound branches to `filledLabel` first; both * that block and the returned (state-checked) dense block may use the * guard's values. A `proven` index is known to lie below the bound (a dense - * array loop's fast copy, dense-array-loops.ts) and always takes it. */ + * array loop's fast copy, dense-array-loops.ts) and always takes it. With + * `prefixOnly`, every other index branches to `slow` and no block is left + * open: only `offset` and `data` are meaningful. */ function emitDenseGuard( host: LlvmEmitterContext, array: string, @@ -57,7 +68,9 @@ function emitDenseGuard( slow: string, requireBelowLen: boolean, filledLabel: string, + width: number, proven = false, + prefixOnly = false, ): DenseGuard { const B = host.B; const size = host.sizeType; @@ -89,7 +102,6 @@ function emitDenseGuard( cap = B.tmp(), dataPtr = B.tmp(), data = B.tmp(), - statesPtr = B.tmp(), states = B.tmp(); B.line( `${dataPtr} = getelementptr inbounds %ScrArr, ptr ${array}, i32 0, i32 ${SCR_ARR_FIELD.data}`, @@ -97,26 +109,23 @@ function emitDenseGuard( host.markMemoryPointer(dataPtr, "array:header"); B.line(`${data} = load ptr, ptr ${dataPtr}${host.fieldAliasAttachment(dataPtr)}`); const loadChecked = (): void => { - B.line(`${lenPtr} = getelementptr inbounds %ScrArr, ptr ${array}, i32 0, i32 1`); + B.line( + `${lenPtr} = getelementptr inbounds %ScrArr, ptr ${array}, i32 0, i32 ${SCR_ARR_FIELD.len}`, + ); host.markMemoryPointer(lenPtr, "array:header"); + loadArrCount(host, len, lenPtr, host.fieldAliasAttachment(lenPtr)); B.line( - `${len} = load ${size}, ptr ${lenPtr}${host.fieldAliasAttachment(lenPtr)}${host.arrayLengthRange}`, + `${capPtr} = getelementptr inbounds %ScrArr, ptr ${array}, i32 0, i32 ${SCR_ARR_FIELD.cap}`, ); - B.line(`${capPtr} = getelementptr inbounds %ScrArr, ptr ${array}, i32 0, i32 2`); host.markMemoryPointer(capPtr, "array:header"); - B.line(`${cap} = load ${size}, ptr ${capPtr}${host.fieldAliasAttachment(capPtr)}`); - B.line( - `${statesPtr} = getelementptr inbounds %ScrArr, ptr ${array}, i32 0, i32 ${SCR_ARR_FIELD.present}`, - ); - host.markMemoryPointer(statesPtr, "array:header"); - B.line(`${states} = load ptr, ptr ${statesPtr}${host.fieldAliasAttachment(statesPtr)}`); + loadArrCount(host, cap, capPtr, host.fieldAliasAttachment(capPtr)); }; // Reads load the storage bases before branching: the header of a live // array is always readable, and unconditional loads let LLVM hoist them // out of loops whose only calls are read-only fallbacks. Writes load the // fields that only the state-checked path needs inside it, which keeps // them off a hot loop of prefix stores (that path stores to the header). - const eagerHeader = requireBelowLen; + const eagerHeader = requireBelowLen && !prefixOnly; if (eagerHeader) loadChecked(); const filledPtr = B.tmp(), filled = B.tmp(), @@ -148,6 +157,10 @@ function emitDenseGuard( B.line(`${t} = and i1 ${integral}, ${inFilled}`); inFilled = t; } + if (prefixOnly) { + B.condBr(inFilled, filledLabel, slow); + return { offset, len, filled, data, states }; + } const check = B.newLabel("arr.check"); B.condBr(inFilled, filledLabel, check); B.startBlock(check); @@ -171,6 +184,10 @@ function emitDenseGuard( const dense = B.newLabel("arr.dense"); B.condBr(ok, dense, slow); B.startBlock(dense); + // Only the state-checked block needs the states: the address is pure + // arithmetic on loaded fields, so computing it here keeps it off the + // prefix path. + scrArrStates(host, states, data, cap, width); return { offset, len, filled, data, states }; } @@ -186,11 +203,8 @@ function statePointer( host: LlvmEmitterContext, guard: Pick, ): string { - const B = host.B; - const statePtr = B.tmp(); - B.line( - `${statePtr} = getelementptr inbounds i8, ptr ${guard.states}, ${host.sizeType} ${guard.offset}`, - ); + const statePtr = host.B.tmp(); + scrArrStatePtr(host, statePtr, guard.states, guard.offset); host.markMemoryPointer(statePtr, "array:present"); return statePtr; } @@ -201,6 +215,11 @@ function slotType(acc: ArrayAccess): "i8" | "i64" { return acc === "bool" ? "i8" : "i64"; } +/** Bytes per dense slot (scr_arr_slot_width). */ +export function slotWidth(acc: ArrayAccess): number { + return acc === "bool" ? 1 : 8; +} + function valuePointer( host: LlvmEmitterContext, guard: Pick, @@ -245,7 +264,7 @@ export function emitDenseSlotUnlink( const unlink = B.newLabel("arr.unlink"), swap = B.newLabel("arr.unlink.swap"), done = B.newLabel("arr.unlink.done"); - const guard = emitDenseGuard(host, array, index, indexExpr, done, true, unlink); + const guard = emitDenseGuard(host, array, index, indexExpr, done, true, unlink, 8); // The state-checked dense block: leave holes and the tail to the store. B.br(done); B.startBlock(unlink); @@ -290,11 +309,29 @@ export function emitDenseArrayGet( const slow = B.newLabel("arr.get.slow"), join = B.newLabel("arr.get.join"), valueLabel = B.newLabel("arr.get.value"); - const guard = emitDenseGuard(host, array, index, indexExpr, slow, true, valueLabel, proven); - const state = loadState(host, guard); - const present = B.tmp(); - B.line(`${present} = icmp eq i8 ${state}, ${SCR_ARR_VALUE}`); - B.condBr(present, valueLabel, slow); + // The state of slot i is at data + cap * (width + 1) - 1 - i: an inline + // state check indexed by a loop's induction variable makes LLVM keep + // -1 - i as a second induction variable, so above the prefix the runtime + // getter reads the slot. + const induction = indexExpr?.kind === "varRef" && host.integerLoopBindings.has(indexExpr.localId); + const guard = emitDenseGuard( + host, + array, + index, + indexExpr, + slow, + true, + valueLabel, + slotWidth(acc), + proven, + induction, + ); + if (!induction) { + const state = loadState(host, guard); + const present = B.tmp(); + B.line(`${present} = icmp eq i8 ${state}, ${SCR_ARR_VALUE}`); + B.condBr(present, valueLabel, slow); + } B.startBlock(valueLabel); const ptr = valuePointer(host, guard, acc); let fast: string; @@ -360,6 +397,7 @@ export function emitDenseArrayState( index: LlValue, indexExpr: IrExpr | undefined, kind: "state" | "has", + width: number, ): string { const B = host.B; const fn = kind === "state" ? "scr_arr_state" : "scr_arr_has"; @@ -368,7 +406,7 @@ export function emitDenseArrayState( const slow = B.newLabel("arr.state.slow"), join = B.newLabel("arr.state.join"), filledLabel = B.newLabel("arr.state.filled"); - const guard = emitDenseGuard(host, array, index, indexExpr, slow, true, filledLabel); + const guard = emitDenseGuard(host, array, index, indexExpr, slow, true, filledLabel, width); const state = loadState(host, guard); const fast = B.tmp(); if (kind === "state") B.line(`${fast} = uitofp i8 ${state} to double`); @@ -421,7 +459,17 @@ export function emitDenseArraySet( const slow = B.newLabel("arr.set.slow"), join = B.newLabel("arr.set.join"), filledLabel = B.newLabel("arr.set.filled"); - const guard = emitDenseGuard(host, array, index, indexExpr, slow, false, filledLabel, proven); + const guard = emitDenseGuard( + host, + array, + index, + indexExpr, + slow, + false, + filledLabel, + slotWidth(acc), + proven, + ); const { offset, len } = guard; const storeValue = (valuePtr: string): void => { if (acc === "f64") { @@ -482,9 +530,11 @@ export function emitDenseArraySet( const nextLen = B.tmp(), lenPtr = B.tmp(); B.line(`${nextLen} = add ${host.sizeType} ${offset}, 1`); - B.line(`${lenPtr} = getelementptr inbounds %ScrArr, ptr ${array}, i32 0, i32 1`); + B.line( + `${lenPtr} = getelementptr inbounds %ScrArr, ptr ${array}, i32 0, i32 ${SCR_ARR_FIELD.len}`, + ); host.markMemoryPointer(lenPtr, "array:header"); - B.line(`store ${host.sizeType} ${nextLen}, ptr ${lenPtr}${host.fieldAliasAttachment(lenPtr)}`); + storeArrCount(host, nextLen, lenPtr, host.fieldAliasAttachment(lenPtr)); B.br(storedLabel); B.startBlock(storedLabel); if (acc === "ref" && oldState && old) { @@ -542,7 +592,7 @@ export function emitDenseArrayPop( const lenPtr = header(SCR_ARR_FIELD.len), len = B.tmp(), none = B.tmp(); - B.line(`${len} = load ${size}, ptr ${lenPtr}${host.fieldAliasAttachment(lenPtr)}`); + loadArrCount(host, len, lenPtr, host.fieldAliasAttachment(lenPtr)); B.line(`${none} = icmp eq ${size} ${len}, 0`); const check = B.newLabel("arr.pop.check"), dense = B.newLabel("arr.pop.dense"); @@ -557,12 +607,15 @@ export function emitDenseArrayPop( B.startBlock(dense); const last = B.tmp(); B.line(`${last} = sub nuw ${size} ${len}, 1`); + // A packed array's last index is below its capacity (filled <= cap). const dataPtr = header(SCR_ARR_FIELD.data), data = B.tmp(), - statesPtr = header(SCR_ARR_FIELD.present), + capPtr = header(SCR_ARR_FIELD.cap), + cap = B.tmp(), states = B.tmp(); B.line(`${data} = load ptr, ptr ${dataPtr}${host.fieldAliasAttachment(dataPtr)}`); - B.line(`${states} = load ptr, ptr ${statesPtr}${host.fieldAliasAttachment(statesPtr)}`); + loadArrCount(host, cap, capPtr, host.fieldAliasAttachment(capPtr)); + scrArrStates(host, states, data, cap, slotWidth(acc)); const valuePtr = valuePointer(host, { offset: last, data }, acc); const statePtr = statePointer(host, { offset: last, states }); const slotTy = slotType(acc); @@ -581,7 +634,7 @@ export function emitDenseArrayPop( B.line(`${last32} = trunc i64 ${last} to i32`); } B.line(`store i32 ${last32}, ptr ${filledPtr}${host.fieldAliasAttachment(filledPtr)}`); - B.line(`store ${size} ${last}, ptr ${lenPtr}${host.fieldAliasAttachment(lenPtr)}`); + B.line(`store i32 ${last32}, ptr ${lenPtr}${host.fieldAliasAttachment(lenPtr)}`); return raw; } @@ -608,13 +661,13 @@ export function emitDenseArrayPush( host.markMemoryPointer(ptr, "array:header"); return ptr; }; - const lenPtr = header(1), + const lenPtr = header(SCR_ARR_FIELD.len), len = B.tmp(), cap = B.tmp(), room = B.tmp(); - B.line(`${len} = load ${size}, ptr ${lenPtr}${host.fieldAliasAttachment(lenPtr)}`); - const capPtr = header(2); - B.line(`${cap} = load ${size}, ptr ${capPtr}${host.fieldAliasAttachment(capPtr)}`); + loadArrCount(host, len, lenPtr, host.fieldAliasAttachment(lenPtr)); + const capPtr = header(SCR_ARR_FIELD.cap); + loadArrCount(host, cap, capPtr, host.fieldAliasAttachment(capPtr)); B.line(`${room} = icmp ult ${size} ${len}, ${cap}`); const fast = B.newLabel("arr.push.dense"), slow = B.newLabel("arr.push.slow"), @@ -623,11 +676,18 @@ export function emitDenseArrayPush( B.startBlock(fast); const dataPtr = header(SCR_ARR_FIELD.data), data = B.tmp(), - statesPtr = header(SCR_ARR_FIELD.present), - states = B.tmp(); + spare = B.tmp(), + near = B.tmp(), + statePtr = B.tmp(); B.line(`${data} = load ptr, ptr ${dataPtr}${host.fieldAliasAttachment(dataPtr)}`); - B.line(`${states} = load ptr, ptr ${statesPtr}${host.fieldAliasAttachment(statesPtr)}`); - const statePtr = statePointer(host, { offset: len, states }); + // The state of slot `len` is data + width * cap + (cap - len) - 1. With + // `cap - len` formed first, the store addresses it from one register sum. + B.line(`${spare} = sub nuw ${size} ${cap}, ${len}`); + B.line(`${near} = getelementptr inbounds i8, ptr ${data}, ${size} ${spare}`); + B.line( + `${statePtr} = getelementptr inbounds [${slotWidth(acc)} x i8], ptr ${near}, ${size} ${cap}, ${size} -1`, + ); + host.markMemoryPointer(statePtr, "array:present"); const valuePtr = valuePointer(host, { offset: len, data }, acc); if (acc === "f64") { B.line(`store double ${value}, ptr ${valuePtr}${host.fieldAliasAttachment(valuePtr)}`); @@ -642,7 +702,7 @@ export function emitDenseArrayPush( const next = B.tmp(), fastValue = B.tmp(); B.line(`${next} = add nuw ${size} ${len}, 1`); - B.line(`store ${size} ${next}, ptr ${lenPtr}${host.fieldAliasAttachment(lenPtr)}`); + storeArrCount(host, next, lenPtr, host.fieldAliasAttachment(lenPtr)); // An append to a hole-free array keeps it hole-free. const filledPtr = header(SCR_ARR_FIELD.filled), filled = B.tmp(), diff --git a/packages/compiler/src/backend/llvm/dense-array-loops.ts b/packages/compiler/src/backend/llvm/dense-array-loops.ts index 271a5ddf8..708825344 100644 --- a/packages/compiler/src/backend/llvm/dense-array-loops.ts +++ b/packages/compiler/src/backend/llvm/dense-array-loops.ts @@ -22,7 +22,7 @@ import type { IntegerRange } from "../../ir/integer-ranges.js"; import { preservesArrayLength } from "../../ir/array-length-stability.js"; import { everyExprChild, everyStmtChild } from "../../ir/traverse.js"; import type { LlvmEmitterContext } from "./expr-context.js"; -import { SCR_ARR_FIELD } from "./common.js"; +import { loadArrCount, SCR_ARR_FIELD } from "./common.js"; export const INT32_MAX = 2147483647; @@ -187,12 +187,10 @@ export function emitDenseLoopGuard(host: LlvmEmitterContext, loop: DenseArrayLoo return ptr; }; const receiver = host.emitStableReceiver(loop.length, []); - const lenPtr = header(receiver.name, 1), + const lenPtr = header(receiver.name, SCR_ARR_FIELD.len), len = B.tmp(), small = B.tmp(); - B.line( - `${len} = load ${size}, ptr ${lenPtr}${host.fieldAliasAttachment(lenPtr)}${host.arrayLengthRange}`, - ); + loadArrCount(host, len, lenPtr, host.fieldAliasAttachment(lenPtr)); B.line(`${small} = icmp ule ${size} ${len}, ${INT32_MAX}`); let valid = small; for (const [id, ref] of loop.arrays) { diff --git a/packages/compiler/src/backend/llvm/dyn.ts b/packages/compiler/src/backend/llvm/dyn.ts index 86b4f5be8..14069f3d9 100644 --- a/packages/compiler/src/backend/llvm/dyn.ts +++ b/packages/compiler/src/backend/llvm/dyn.ts @@ -1585,10 +1585,10 @@ export class LlDyn { : []), ] : [ - { index: 1, type: host.sizeType, name: "length" }, - { index: 2, type: host.sizeType, name: "capacity" }, + // The dense states follow the data slots: they move with data and capacity. + { index: SCR_ARR_FIELD.len, type: "i32", name: "length" }, + { index: SCR_ARR_FIELD.cap, type: "i32", name: "capacity" }, { index: SCR_ARR_FIELD.data, type: "ptr" as const, name: "data" }, - { index: SCR_ARR_FIELD.present, type: "ptr" as const, name: "presence" }, { index: SCR_ARR_FIELD.filled, type: "i32" as const, name: "filled" }, ]; members.forEach((member) => { diff --git a/packages/compiler/src/backend/llvm/emit-module.ts b/packages/compiler/src/backend/llvm/emit-module.ts index 7d60205e5..a42643bca 100644 --- a/packages/compiler/src/backend/llvm/emit-module.ts +++ b/packages/compiler/src/backend/llvm/emit-module.ts @@ -432,14 +432,15 @@ export function emitModuleParts(emitter: LlEmitter): string[] { `%ScrClosure = type { ${emitter.sizeType}, ptr, ${emitter.sizeType}, ptr, i32, i32, ptr, ptr }`, `%ScrFfiTable = type { ptr, ${emitter.sizeType}, ${emitter.sizeType}, ptr, i8, ptr, ptr, ${emitter.sizeType}, ${emitter.sizeType}, ${emitter.sizeType}, ptr, ptr }`, `%ScrRegex = type { ${emitter.sizeType}, ptr, ptr, ptr, double, ptr, ptr, ptr }`, - // ScrArr mirrors scr_runtime.h field-for-field: { rc, len, cap, elem, - // cyc, spare, filled, ops, data, present, ext } (field numbers in - // SCR_ARR_FIELD), where filled bounds the hole-free prefix, ops points at the element type's constant RC descriptor + // ScrArr mirrors scr_runtime.h field-for-field: { rc, len (i32), cap + // (i32), elem, cyc, spare, filled, ops, data, ext } (field numbers in + // SCR_ARR_FIELD), where the dense states follow the cap value slots at + // data, filled bounds the hole-free prefix, ops points at the element type's constant RC descriptor // (%ScrArrOps, SCR_ELEM_REF only) and ext at the out-of-line sparse, // property and metadata storage (NULL until first used). Live dynamic // stream commits swap its dense, presence and out-of-line storage while // preserving the target object's identity. - `%ScrArr = type { ${emitter.sizeType}, ${emitter.sizeType}, ${emitter.sizeType}, i8, i8, i16, i32, ptr, ptr, ptr, ptr }`, + `%ScrArr = type { ${emitter.sizeType}, i32, i32, i8, i8, i16, i32, ptr, ptr, ptr }`, `%ScrArrOps = type { ptr, ptr, ptr }`, // The ScrMap prefix { rc, key_kind, val_kind, traced } — the inline // RC fast paths test the traced byte (shapes.ts). @@ -680,17 +681,17 @@ export function emitModuleParts(emitter: LlEmitter): string[] { if (emitter.regexInstances.size > 0) out.push(``); for (const code of regexMatchers.values()) out.push(...code.lines, ``); for (const [, inst] of emitter.templateStringsInstances) { - // One immortal ScrArr per tagged-template site: a [N x ptr] data - // global of interned cooked-string literals, and the ScrArr header - // over it (rc == SIZE_MAX, len == cap == filled, SCR_ELEM_STR = 2, - // no REF entry points). Every dense slot is present; reads retain - // immortal strings — a no-op. + // One immortal ScrArr per tagged-template site: a data global of + // interned cooked-string literals followed by their states (the + // runtime finds them before data + cap * 9), and the ScrArr header over + // it (rc == SIZE_MAX, len == cap == filled, SCR_ELEM_STR = 2, no REF + // entry points). Every dense slot is present; reads retain immortal + // strings — a no-op. const n = inst.slots.length; const present = n === 0 ? "zeroinitializer" : `[ ${inst.slots.map(() => "i8 1").join(", ")} ]`; out.push( - `@${inst.sym}_data = internal constant [${n} x ptr] [ ${inst.slots.map((s) => `ptr ${s}`).join(", ")} ]`, - `@${inst.sym}_present = internal constant [${n} x i8] ${present}`, - `@${inst.sym} = internal global %ScrArr { ${emitter.sizeType} -1, ${emitter.sizeType} ${n}, ${emitter.sizeType} ${n}, i8 2, i8 0, i16 0, i32 ${n}, ptr null, ptr @${inst.sym}_data, ptr @${inst.sym}_present, ptr null }`, + `@${inst.sym}_data = internal constant <{ [${n} x ptr], [${n} x i8] }> <{ [${n} x ptr] [ ${inst.slots.map((s) => `ptr ${s}`).join(", ")} ], [${n} x i8] ${present} }>, align 8`, + `@${inst.sym} = internal global %ScrArr { ${emitter.sizeType} -1, i32 ${n}, i32 ${n}, i8 2, i8 0, i16 0, i32 ${n}, ptr null, ptr @${inst.sym}_data, ptr null }`, ); } if (emitter.templateStringsInstances.size > 0) out.push(``); diff --git a/packages/compiler/src/backend/llvm/emit-statements.ts b/packages/compiler/src/backend/llvm/emit-statements.ts index e641f3ecd..05f172884 100644 --- a/packages/compiler/src/backend/llvm/emit-statements.ts +++ b/packages/compiler/src/backend/llvm/emit-statements.ts @@ -31,7 +31,7 @@ import { emitDiscardedSplice } from "./expr-containers.js"; import { InternalCompilerError } from "../../errors.js"; import { emitDenseArrayGet, emitDenseArrayPop, emitDenseArraySet } from "./dense-array-access.js"; import { emitBorrowedInput } from "./borrowed-inputs.js"; -import { f64Lit, llvmCommentText, sizeToDouble } from "./common.js"; +import { f64Lit, llvmCommentText, loadArrCount, sizeToDouble } from "./common.js"; import { RESULT_DST_ID, RESULT_ENTRY_PREFIX } from "./record-results.js"; import { matchIntegerArrayForLoop, @@ -434,9 +434,7 @@ export function emitStmtBody(emitter: LlEmitter, s: IrStmt): void { done = B.newLabel("arr.len.done"); B.line(`${lenPtr} = getelementptr inbounds %ScrArr, ptr ${arr.name}, i32 0, i32 1`); emitter.markMemoryPointer(lenPtr, "array:header"); - B.line( - `${len} = load ${emitter.sizeType}, ptr ${lenPtr}${emitter.fieldAliasAttachment(lenPtr)}${emitter.arrayLengthRange}`, - ); + loadArrCount(emitter, len, lenPtr, emitter.fieldAliasAttachment(lenPtr)); B.line( `${lenNumber} = ${sizeToDouble(emitter.sizeType)} ${emitter.sizeType} ${len} to double`, ); @@ -1091,12 +1089,25 @@ export function emitStmtBody(emitter: LlEmitter, s: IrStmt): void { B.line( `${lenPtr} = getelementptr inbounds %${integerArrayLoop ? "ScrArr" : "ScrBytes"}, ptr ${receiver.name}, i64 0, i32 1`, ); - if (integerArrayLoop) emitter.markMemoryPointer(lenPtr, "array:header"); - B.line( - `${len} = load ${emitter.sizeType}, ptr ${lenPtr}${emitter.fieldAliasAttachment(lenPtr)}`, - ); B.line(`${index} = load ${emitter.sizeType}, ptr ${integerSlot}`); - B.line(`${inBounds} = icmp ult ${emitter.sizeType} ${index}, ${len}`); + if (integerArrayLoop) { + // The counter starts at most 2^32 - 1 and steps only while it is + // below the length, so it fits the i32 length field: compare in + // 32 bits, which lets the compare read the field directly. + emitter.markMemoryPointer(lenPtr, "array:header"); + B.line(`${len} = load i32, ptr ${lenPtr}${emitter.fieldAliasAttachment(lenPtr)}`); + let narrow = index; + if (emitter.sizeType === "i64") { + narrow = B.tmp(); + B.line(`${narrow} = trunc i64 ${index} to i32`); + } + B.line(`${inBounds} = icmp ult i32 ${narrow}, ${len}`); + } else { + B.line( + `${len} = load ${emitter.sizeType}, ptr ${lenPtr}${emitter.fieldAliasAttachment(lenPtr)}`, + ); + B.line(`${inBounds} = icmp ult ${emitter.sizeType} ${index}, ${len}`); + } B.condBr(inBounds, lb, exit); } else if (countedLoop && integerSlot && countedLimit) { const index = B.tmp(); @@ -1202,9 +1213,7 @@ export function emitStmtBody(emitter: LlEmitter, s: IrStmt): void { // The inline form of scr_arr_len, like the `length` intrinsic. B.line(`${lenPtr} = getelementptr inbounds %ScrArr, ptr ${arr!.name}, i32 0, i32 1`); emitter.markMemoryPointer(lenPtr, "array:header"); - B.line( - `${rawLen} = load ${emitter.sizeType}, ptr ${lenPtr}${emitter.fieldAliasAttachment(lenPtr)}${emitter.arrayLengthRange}`, - ); + loadArrCount(emitter, rawLen, lenPtr, emitter.fieldAliasAttachment(lenPtr)); B.line( `${len} = ${sizeToDouble(emitter.sizeType)} ${emitter.sizeType} ${rawLen} to double`, ); diff --git a/packages/compiler/src/backend/llvm/emitter.ts b/packages/compiler/src/backend/llvm/emitter.ts index 1d745b981..d17ee1480 100644 --- a/packages/compiler/src/backend/llvm/emitter.ts +++ b/packages/compiler/src/backend/llvm/emitter.ts @@ -606,13 +606,6 @@ export class LlEmitter { return `, !tbaa !${tag}`; } - /** The `!range` attachment of an array length load: a length is at most - * 2^32 - 1 (the runtime's SCR_ARR_MAX_LENGTH), so conversions to double - * and back fold. 32-bit targets load the full range anyway. */ - get arrayLengthRange(): string { - return this.sizeType === "i64" ? ", !range !{i64 0, i64 4294967296}" : ""; - } - /** Whether an element access at `index` of `arr` lies in the hole-free * prefix: the counter of a dense array loop's fast copy indexing one of * the arrays its entry test covered. */ diff --git a/packages/compiler/src/backend/llvm/expr-containers.ts b/packages/compiler/src/backend/llvm/expr-containers.ts index a04f46f7b..073fcb827 100644 --- a/packages/compiler/src/backend/llvm/expr-containers.ts +++ b/packages/compiler/src/backend/llvm/expr-containers.ts @@ -17,7 +17,7 @@ import { vAdapters, } from "./shapes.js"; import type { LlvmEmitterContext, LlValue } from "./expr-context.js"; -import { F64_INF, f64Lit, sizeToDouble } from "./common.js"; +import { F64_INF, f64Lit, loadArrCount, sizeToDouble } from "./common.js"; import { MUTATING_COLLECTION_METHODS } from "../publication.js"; import { borrowsStringInputs, emitStringInputs } from "./string-lifetimes.js"; import { @@ -564,9 +564,7 @@ export function emitArrIntrinsic( t = B.tmp(); B.line(`${p} = getelementptr inbounds %ScrArr, ptr ${r.name}, i32 0, i32 1`); host.markMemoryPointer(p, "array:header"); - B.line( - `${len} = load ${host.sizeType}, ptr ${p}${host.fieldAliasAttachment(p)}${host.arrayLengthRange}`, - ); + loadArrCount(host, len, p, host.fieldAliasAttachment(p)); B.line(`${t} = ${sizeToDouble(host.sizeType)} ${host.sizeType} ${len} to double`); return { name: t, type: e.type }; } diff --git a/packages/compiler/src/backend/llvm/expr-primitives.ts b/packages/compiler/src/backend/llvm/expr-primitives.ts index 35e759110..470cfbfb4 100644 --- a/packages/compiler/src/backend/llvm/expr-primitives.ts +++ b/packages/compiler/src/backend/llvm/expr-primitives.ts @@ -32,7 +32,7 @@ import { emitPowerOfTwoRemainder, emitSignedIntegerRemainder, } from "./integer-remainder.js"; -import { emitDenseArrayGet, emitDenseArrayState } from "./dense-array-access.js"; +import { emitDenseArrayGet, emitDenseArrayState, slotWidth } from "./dense-array-access.js"; import { emitWrappingSum, markWrappingOperands } from "./wrapping-sums.js"; export function emitLiteralExpr( @@ -894,7 +894,14 @@ export function emitContainerExpr( const idx = host.emitExpr(e.index); if (e.arr.type.kind !== "array") throw new InternalCompilerError("llvm emitter bug: arrayHas on non-array"); - const t = emitDenseArrayState(host, arr.name, idx, e.index, "has"); + const t = emitDenseArrayState( + host, + arr.name, + idx, + e.index, + "has", + slotWidth(elemAccess(e.arr.type.elem)), + ); return { name: t, type: e.type }; } case "arrayState": { @@ -902,7 +909,14 @@ export function emitContainerExpr( const idx = host.emitExpr(e.index); if (e.arr.type.kind !== "array") throw new InternalCompilerError("llvm emitter bug: arrayState on non-array"); - const t = emitDenseArrayState(host, arr.name, idx, e.index, "state"); + const t = emitDenseArrayState( + host, + arr.name, + idx, + e.index, + "state", + slotWidth(elemAccess(e.arr.type.elem)), + ); return { name: t, type: e.type }; } case "arrIntrinsic": diff --git a/packages/compiler/src/backend/llvm/expr-stream-bridges.ts b/packages/compiler/src/backend/llvm/expr-stream-bridges.ts index 8a53b5fa8..9b206e6dc 100644 --- a/packages/compiler/src/backend/llvm/expr-stream-bridges.ts +++ b/packages/compiler/src/backend/llvm/expr-stream-bridges.ts @@ -185,10 +185,10 @@ export function streamTypedRefCommitAdapter( ` call void @scr_arr_outline(ptr %next)`, ]; const storageMembers = [ - { name: "len", index: 1, type: host.sizeType }, - { name: "cap", index: 2, type: host.sizeType }, + // The dense states follow the data slots, so they move with data and cap. + { name: "len", index: SCR_ARR_FIELD.len, type: "i32" }, + { name: "cap", index: SCR_ARR_FIELD.cap, type: "i32" }, { name: "data", index: SCR_ARR_FIELD.data, type: "ptr" }, - { name: "present", index: SCR_ARR_FIELD.present, type: "ptr" }, { name: "filled", index: SCR_ARR_FIELD.filled, type: "i32" }, { name: "ext", index: SCR_ARR_FIELD.ext, type: "ptr" }, ]; diff --git a/packages/compiler/src/backend/llvm/integer-clones.ts b/packages/compiler/src/backend/llvm/integer-clones.ts index f3efa1259..314eae5ea 100644 --- a/packages/compiler/src/backend/llvm/integer-clones.ts +++ b/packages/compiler/src/backend/llvm/integer-clones.ts @@ -1009,10 +1009,10 @@ class CloneEmitter { B.line( `${field} = getelementptr inbounds %ScrArr, ptr ${array}, i32 0, i32 ${SCR_ARR_FIELD.len}`, ); - B.line(`${len} = load i64, ptr ${field}`); - B.line(`${fits} = icmp ule i64 ${len}, 2147483647`); + B.line(`${len} = load i32, ptr ${field}`); + B.line(`${fits} = icmp ule i32 ${len}, 2147483647`); this.require(fits); - B.line(`${value} = trunc i64 ${len} to i32`); + B.line(`${value} = add i32 ${len}, 0`); return value; } diff --git a/packages/compiler/src/backend/llvm/local-array-reads.ts b/packages/compiler/src/backend/llvm/local-array-reads.ts index 6951896d2..b9fd0f54b 100644 --- a/packages/compiler/src/backend/llvm/local-array-reads.ts +++ b/packages/compiler/src/backend/llvm/local-array-reads.ts @@ -12,7 +12,7 @@ import { everyStmtList } from "../../ir/traverse.js"; import { analyzeCallLifetimes, type CallLifetimes } from "./call-lifetimes.js"; import type { LlValue, LlvmEmitterContext } from "./expr-context.js"; import { ReferenceEffects } from "./reference-effects.js"; -import { SCR_ARR_FIELD, exactIndexLines } from "./common.js"; +import { SCR_ARR_FIELD, exactIndexLines, loadArrCount, scrArrStateAt } from "./common.js"; import { exactInteger, widenInteger } from "./integer-values.js"; import { READ_FALLBACK } from "./dense-array-access.js"; @@ -457,12 +457,14 @@ export function emitProjectedArrayRead(host: LlvmEmitterContext, read: LocalArra /** Share the checked dense lookup between optional stack boxes and strict * borrowed reads. Capacity, length, and presence guards precede every load; - * sparse and noncanonical indices keep their runtime lookup semantics. */ + * sparse and noncanonical indices keep their runtime lookup semantics. + * `induction` says `integerIndex` is a loop's induction variable. */ function emitDenseReferenceArrayRead( host: LlvmEmitterContext, array: LlValue, index: LlValue, integerIndex: string | null, + induction: boolean, present: (value: string) => void, no: string, slow: string, @@ -476,9 +478,11 @@ function emitDenseReferenceArrayRead( // noncanonical numeric properties retain the runtime lookup semantics. const capPtr = B.tmp(), cap = B.tmp(); - B.line(`${capPtr} = getelementptr inbounds %ScrArr, ptr ${array.name}, i32 0, i32 2`); + B.line( + `${capPtr} = getelementptr inbounds %ScrArr, ptr ${array.name}, i32 0, i32 ${SCR_ARR_FIELD.cap}`, + ); host.markMemoryPointer(capPtr, "array:header"); - B.line(`${cap} = load ${host.sizeType}, ptr ${capPtr}${host.fieldAliasAttachment(capPtr)}`); + loadArrCount(host, cap, capPtr, host.fieldAliasAttachment(capPtr)); let offset: string, inRange: string; if (integerIndex) { offset = integerIndex; @@ -499,6 +503,15 @@ function emitDenseReferenceArrayRead( B.startBlock(range); B.br(dense); B.startBlock(dense); + // The state and value paths below both index from `data`: one load here + // serves both. + const dataPtr = B.tmp(), + data = B.tmp(); + B.line( + `${dataPtr} = getelementptr inbounds %ScrArr, ptr ${array.name}, i32 0, i32 ${SCR_ARR_FIELD.data}`, + ); + host.markMemoryPointer(dataPtr, "array:header"); + B.line(`${data} = load ptr, ptr ${dataPtr}${host.fieldAliasAttachment(dataPtr)}`); // Below the hole-free prefix bound every state is a present value. const filledPtr = B.tmp(), filled = B.tmp(), @@ -516,45 +529,38 @@ function emitDenseReferenceArrayRead( } B.line(`${belowFilled} = icmp ult ${host.sizeType} ${offset}, ${filledSized}`); const valueLabel = B.newLabel("local.array.value"); - B.condBr(belowFilled, valueLabel, lenLabel); - B.startBlock(lenLabel); - const lenPtr = B.tmp(), - len = B.tmp(), - belowLen = B.tmp(); - B.line(`${lenPtr} = getelementptr inbounds %ScrArr, ptr ${array.name}, i32 0, i32 1`); - host.markMemoryPointer(lenPtr, "array:header"); - B.line( - `${len} = load ${host.sizeType}, ptr ${lenPtr}${host.fieldAliasAttachment(lenPtr)}${host.arrayLengthRange}`, - ); - B.line(`${belowLen} = icmp ult ${host.sizeType} ${offset}, ${len}`); - const stateLabel = B.newLabel("local.array.state"); - B.condBr(belowLen, stateLabel, no); - B.startBlock(stateLabel); - const statesPtr = B.tmp(), - states = B.tmp(), - statePtr = B.tmp(), - denseState = B.tmp(), - densePresent = B.tmp(); - B.line( - `${statesPtr} = getelementptr inbounds %ScrArr, ptr ${array.name}, i32 0, i32 ${SCR_ARR_FIELD.present}`, - ); - host.markMemoryPointer(statesPtr, "array:header"); - B.line(`${states} = load ptr, ptr ${statesPtr}${host.fieldAliasAttachment(statesPtr)}`); - B.line(`${statePtr} = getelementptr inbounds i8, ptr ${states}, ${host.sizeType} ${offset}`); - host.markMemoryPointer(statePtr, "array:present"); - B.line(`${denseState} = load i8, ptr ${statePtr}${host.fieldAliasAttachment(statePtr)}`); - B.line(`${densePresent} = icmp eq i8 ${denseState}, 1`); - B.condBr(densePresent, valueLabel, no); + if (induction) { + // The state of slot i is at data + cap * (width + 1) - 1 - i, and an + // inline state check makes LLVM keep -1 - i as a second induction + // variable in the loop; above the prefix the runtime reads the slot. + B.condBr(belowFilled, valueLabel, slow); + } else { + B.condBr(belowFilled, valueLabel, lenLabel); + B.startBlock(lenLabel); + const lenPtr = B.tmp(), + len = B.tmp(), + belowLen = B.tmp(); + B.line( + `${lenPtr} = getelementptr inbounds %ScrArr, ptr ${array.name}, i32 0, i32 ${SCR_ARR_FIELD.len}`, + ); + host.markMemoryPointer(lenPtr, "array:header"); + loadArrCount(host, len, lenPtr, host.fieldAliasAttachment(lenPtr)); + B.line(`${belowLen} = icmp ult ${host.sizeType} ${offset}, ${len}`); + const stateLabel = B.newLabel("local.array.state"); + B.condBr(belowLen, stateLabel, no); + B.startBlock(stateLabel); + const statePtr = B.tmp(), + denseState = B.tmp(), + densePresent = B.tmp(); + scrArrStateAt(host, statePtr, data, cap, loadType === "i8" ? 1 : 8, offset); + host.markMemoryPointer(statePtr, "array:present"); + B.line(`${denseState} = load i8, ptr ${statePtr}${host.fieldAliasAttachment(statePtr)}`); + B.line(`${densePresent} = icmp eq i8 ${denseState}, 1`); + B.condBr(densePresent, valueLabel, no); + } B.startBlock(valueLabel); - const dataPtr = B.tmp(), - data = B.tmp(), - valuePtr = B.tmp(), + const valuePtr = B.tmp(), raw = B.tmp(); - B.line( - `${dataPtr} = getelementptr inbounds %ScrArr, ptr ${array.name}, i32 0, i32 ${SCR_ARR_FIELD.data}`, - ); - host.markMemoryPointer(dataPtr, "array:header"); - B.line(`${data} = load ptr, ptr ${dataPtr}${host.fieldAliasAttachment(dataPtr)}`); // Boolean arrays store one byte per element; every other kind an 8-byte slot. const slotType = loadType === "i8" ? "i8" : "i64"; B.line( @@ -590,6 +596,7 @@ export function emitBorrowedArrayRead( array, index, integerIndex, + loopIndex !== null, (value) => B.line(`store ptr ${value}, ptr ${slot}`), slow, slow, @@ -632,7 +639,18 @@ export function emitLocalArrayRead( B.line(`store ptr ${read.borrow ? value : host.retainFrame(value, owned)}, ptr ${localSlot}`); }; if (inline) - emitDenseReferenceArrayRead(host, array, index, integerIndex, store, no, slow, join, "ptr"); + emitDenseReferenceArrayRead( + host, + array, + index, + integerIndex, + integerIndex !== null, + store, + no, + slow, + join, + "ptr", + ); else B.br(slow); B.startBlock(slow); const present = B.newLabel("local.array.slow.value"); @@ -687,6 +705,7 @@ export function emitLocalArrayRead( array, index, integerIndex, + integerIndex !== null, storeValue, no, slow, diff --git a/packages/compiler/src/backend/runtime-abi.ts b/packages/compiler/src/backend/runtime-abi.ts index 21edd235f..67be86571 100644 --- a/packages/compiler/src/backend/runtime-abi.ts +++ b/packages/compiler/src/backend/runtime-abi.ts @@ -2,8 +2,8 @@ * of the link contract: a program object keeps this symbol undefined and the * matching runtime defines it, so incompatible manual links fail before the * program can start. */ -export const RUNTIME_ABI_VERSION = 22 as const; -export const RUNTIME_ABI_MARKER = "scr_runtime_abi_v22" as const; +export const RUNTIME_ABI_VERSION = 23 as const; +export const RUNTIME_ABI_MARKER = "scr_runtime_abi_v23" as const; /** Object consumption is useful today, but the complete scr_* surface may * still change before 1.0. The versioned marker prevents accidental mixing; diff --git a/packages/compiler/src/backend/runtime-pack-targets.test.ts b/packages/compiler/src/backend/runtime-pack-targets.test.ts index fd57cb49d..e5ac7fa4b 100644 --- a/packages/compiler/src/backend/runtime-pack-targets.test.ts +++ b/packages/compiler/src/backend/runtime-pack-targets.test.ts @@ -52,7 +52,7 @@ async function fixture(target: NativeTargetSpec, workers = false) { object_format: target.objectFormat, minimum_os: target.minimumOs, }, - runtime_abi: { version: 22, marker: "scr_runtime_abi_v22" }, + runtime_abi: { version: 23, marker: "scr_runtime_abi_v23" }, compiler: { command: "fixture", identity: "fixture", target: target.llvmTriple }, macros: { executable: [], excluded: ["SCR_LIB"], sanitizer: "external-toolchain-required" }, flavors: { diff --git a/packages/compiler/src/backend/runtime-pack.test.ts b/packages/compiler/src/backend/runtime-pack.test.ts index 61d1dba5e..6d2978f0c 100644 --- a/packages/compiler/src/backend/runtime-pack.test.ts +++ b/packages/compiler/src/backend/runtime-pack.test.ts @@ -118,7 +118,7 @@ async function fixture(target: NativeTargetSpec = MACOS_ARM64_TARGET) { object_format: target.objectFormat, minimum_os: target.minimumOs, }, - runtime_abi: { version: 22, marker: "scr_runtime_abi_v22" }, + runtime_abi: { version: 23, marker: "scr_runtime_abi_v23" }, compiler: { command: "clang", identity: "fixture clang", diff --git a/packages/compiler/src/ir/array-length-stability.test.ts b/packages/compiler/src/ir/array-length-stability.test.ts index 44e461ca2..a0d0e49a7 100644 --- a/packages/compiler/src/ir/array-length-stability.test.ts +++ b/packages/compiler/src/ir/array-length-stability.test.ts @@ -4,6 +4,8 @@ import { BOOL, F64, arrayOf, + mapOf, + setOf, type IrExpr, type IrFunction, type IrLocal, @@ -111,6 +113,32 @@ test("captured receivers have no local proof", () => { expect(preserves([sum], "i", new Set(), boxed)).toBe(false); }); +test("Map and Set reads and checked union extractions run no user code", () => { + const counts: IrExpr = { kind: "varRef", localId: "m", type: mapOf(F64, F64), loc }; + const seen: IrExpr = { kind: "varRef", localId: "v", type: setOf(F64), loc }; + const onMap = (name: string, args: IrExpr[]): IrExpr => + ({ kind: "mapIntrinsic", method: name, receiver: counts, args, type: F64, loc }) as IrExpr; + const onSet = (name: string, args: IrExpr[]): IrExpr => + ({ kind: "setIntrinsic", method: name, receiver: seen, args, type: BOOL, loc }) as IrExpr; + const call = (callee: string): IrExpr => ({ + kind: "call", + callee, + args: [ref("s")], + type: F64, + loc, + }); + const key = read("a", ref("i")); + expect( + preserves([stmt(onMap("has", [key])), stmt(onMap("get", [key])), stmt(onMap("size", []))]), + ).toBe(true); + expect(preserves([stmt(onSet("has", [key])), stmt(onSet("size", []))])).toBe(true); + expect(preserves([stmt(call("%union.narrow.0"))])).toBe(true); + // Writes release replaced values; other calls run user code. + expect(preserves([stmt(onMap("set", [key, num(1)]))])).toBe(false); + expect(preserves([stmt(onSet("add", [key]))])).toBe(false); + expect(preserves([stmt(call("g"))])).toBe(false); +}); + function fn(body: IrStmt[], params: string[] = []): IrFunction { return { loc, diff --git a/packages/compiler/src/ir/array-length-stability.ts b/packages/compiler/src/ir/array-length-stability.ts index 2ac196d30..3f9c3b9ca 100644 --- a/packages/compiler/src/ir/array-length-stability.ts +++ b/packages/compiler/src/ir/array-length-stability.ts @@ -17,6 +17,11 @@ const READING_ARRAY_METHODS = new Set([ /** String methods that read an immutable string and run no user code. */ const READING_STRING_METHODS = new Set(["length", "charCodeAt", "codePointAt"]); +/** Map and Set methods that read the receiver and run no user code: keys + * hash and compare by value or identity, and no array is touched. */ +const READING_MAP_METHODS = new Set(["get", "has", "size"]); +const READING_SET_METHODS = new Set(["has", "size"]); + /** Expressions that compute from their operands and run no user code. Their * children are checked separately. */ const PLAIN_EXPRESSIONS = new Set([ @@ -162,8 +167,9 @@ export interface ArrayLengthLoop { /** Whether running `nodes` (a loop's condition, body and update) can never * change the length of the array held by local `loop.receiver`. The proof is * syntactic and conservative: every expression and statement must be one - * that runs no user code (no calls, callbacks, closures, iterators, - * suspension or exception regions); the binding is never written and only + * that runs no user code (no calls other than checked union extractions, no + * callbacks, closures, iterators, suspension or exception regions; Map and + * Set reads are accepted); the binding is never written and only * ever read as the receiver of an element access or a reading array method, * so the array cannot escape to code that resizes it; and nothing in the * nodes resizes any array that may be the same one. Element writes are @@ -215,6 +221,14 @@ export function preservesArrayLength( e.args.every(expr) ); } + case "call": + // A checked union extraction (`x!`, coercions/unions.ts): the + // asserted arm's payload or a TypeError, and no user code. + return e.callee.startsWith("%union.narrow.") && e.args.every(expr); + case "mapIntrinsic": + return READING_MAP_METHODS.has(e.method) && expr(e.receiver) && e.args.every(expr); + case "setIntrinsic": + return READING_SET_METHODS.has(e.method) && expr(e.receiver) && e.args.every(expr); case "libCall": return ( isBorrowSafeMath(e.fn) && diff --git a/packages/compiler/test/array-induction-read-emission.test.ts b/packages/compiler/test/array-induction-read-emission.test.ts new file mode 100644 index 000000000..f264c31dc --- /dev/null +++ b/packages/compiler/test/array-induction-read-emission.test.ts @@ -0,0 +1,45 @@ +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { expect, test } from "vitest"; +import { compile } from "../src/index.js"; + +async function emitLlvm(source: string): Promise { + const dir = await mkdtemp(join(tmpdir(), "scriptc-array-induction-read-")); + try { + const entry = join(dir, "main.ts"); + const outPath = join(dir, "main.ll"); + await writeFile(entry, source); + const result = await compile(entry, { outDir: dir, outPath, outputKind: "llvm" }); + expect(result.ok, JSON.stringify(result.diagnostics)).toBe(true); + return await readFile(outPath, "utf8"); + } finally { + await rm(dir, { recursive: true, force: true }); + } +} + +const source = ` +function sum(values: number[]): number { + let total = 0; + for (let i = 0; i < values.length; i++) total += values[i]!; + return total; +} +function at(values: number[], k: number): number { + return values[k]!; +} +console.log(sum([1, 2, 3]), sum(process.argv.map((a) => a.length)), at([4, 5], 1)); +`; + +test("an element read indexed by a loop counter reads above the prefix through the runtime", async () => { + const llvm = await emitLlvm(source); + const sum = /^define internal double @sc_bf_sum\([^]*?^}/m.exec(llvm)?.[0] ?? ""; + // No state address (data + cap * 9 - 1 - i) inside the loop: slots at or + // above the hole-free prefix take scr_arr_get_f64. + expect(sum).toMatch(/call double @scr_arr_get_f64\(/); + expect(sum).not.toMatch(/getelementptr inbounds \[9 x i8\], ptr /); + // The loop bound compares the counter with the i32 length field in 32 bits. + expect(sum).toMatch(/(%\S+) = trunc i64 %\S+ to i32\n\s*%\S+ = icmp ult i32 \1, %\S+/); + // Other indices keep the inline state check. + const at = /^define internal double @sc_bf_at[.(][^]*?^}/m.exec(llvm)?.[0] ?? ""; + expect(at).toMatch(/getelementptr inbounds \[9 x i8\], ptr /); +}); diff --git a/packages/compiler/test/array-length-range-emission.test.ts b/packages/compiler/test/array-length-range-emission.test.ts index 4628298dd..e24ebce45 100644 --- a/packages/compiler/test/array-length-range-emission.test.ts +++ b/packages/compiler/test/array-length-range-emission.test.ts @@ -41,7 +41,10 @@ test.runIf(process.arch === "x64" || process.arch === "arm64")( async () => { const llvm = await emit(source, "llvm"); const body = /^define internal double @sc_bf_count\([^]*?^}/m.exec(llvm)?.[0] ?? ""; - expect(body).toMatch(/= load i64, ptr %\S+(, !tbaa !\d+)?, !range !\{i64 0, i64 4294967296\}/); + // The length is an i32 field: its zero extension carries the range. + expect(body).toMatch( + /(%\S+) = load i32, ptr %\S+(, !tbaa !\d+)?\n\s*%\S+ = zext i32 \1 to i64/, + ); }, ); @@ -49,7 +52,9 @@ test.runIf(process.arch === "x64")( "a length used as an index is not converted to double and back", async () => { const asm = await emit(source, "asm"); - const body = /^sc_bf_count:[^]*?\.cfi_endproc/m.exec(asm)?.[0] ?? ""; - expect(body).not.toMatch(/cvtsi2sd|cvttsd2si/); + // `parameters[length - 1]` loads from data + length * 8 - 8: the index + // stays the integer length, folded into the address (whether or not + // `count` is inlined into its callers). + expect(asm).toMatch(/movq\t-8\(%r\w+,%r\w+,8\), %r\w+/); }, ); diff --git a/packages/compiler/test/native-codegen-integration.test.ts b/packages/compiler/test/native-codegen-integration.test.ts index 1c5f9e3e7..c0f2cfe84 100644 --- a/packages/compiler/test/native-codegen-integration.test.ts +++ b/packages/compiler/test/native-codegen-integration.test.ts @@ -378,7 +378,7 @@ done: expect(await symbols(helperObject, ["-u"])).toEqual(await symbols(clangObject, ["-u"])); expect(await symbols(helperObject, ["-gU"])).toEqual(await symbols(clangObject, ["-gU"])); expect(await symbols(helperObject, ["-gU"])).toEqual(["0000000000000000 T _main"]); - expect(await symbols(helperObject, ["-u"])).toContain("_scr_runtime_abi_v22"); + expect(await symbols(helperObject, ["-u"])).toContain("_scr_runtime_abi_v23"); const clangExe = join(dir, "clang-program"); const linkDriver = join(dir, "clang-driver.c"); @@ -712,7 +712,7 @@ done: .split("\n"); expect(undefinedSymbols).toContain("_sf_scale"); expect(undefinedSymbols).toContain("_sf_callback_mix"); - expect(undefinedSymbols).toContain("_scr_runtime_abi_v22"); + expect(undefinedSymbols).toContain("_scr_runtime_abi_v23"); }); test("helper and clang object paths retain the same outbound FFI C ABI", async () => { diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/array-length-limits.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/array-length-limits.ts.json new file mode 100644 index 000000000..08bab6d2a --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/array-length-limits.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/array-length-limits.ts" + ], + "diags": [] +} diff --git a/packages/runtime-darwin-arm64/scripts/build.mjs b/packages/runtime-darwin-arm64/scripts/build.mjs index dbde1564f..86ac887e4 100644 --- a/packages/runtime-darwin-arm64/scripts/build.mjs +++ b/packages/runtime-darwin-arm64/scripts/build.mjs @@ -335,7 +335,7 @@ async function build() { package: packageManifest.name, version: packageManifest.version, target: RUNTIME_PACK_MATRIX.target, - runtime_abi: { version: 22, marker: "scr_runtime_abi_v22" }, + runtime_abi: { version: 23, marker: "scr_runtime_abi_v23" }, compiler: { command: compiler, identity: compilerVersion, diff --git a/packages/runtime-pack-common/scripts/build.mjs b/packages/runtime-pack-common/scripts/build.mjs index a0f117cec..aa38b4715 100644 --- a/packages/runtime-pack-common/scripts/build.mjs +++ b/packages/runtime-pack-common/scripts/build.mjs @@ -432,7 +432,7 @@ async function build() { package: packageManifest.name, version: packageManifest.version, target: matrix.target, - runtime_abi: { version: 22, marker: "scr_runtime_abi_v22" }, + runtime_abi: { version: 23, marker: "scr_runtime_abi_v23" }, compiler: { command: compiler, identity: compilerVersion, diff --git a/packages/runtime/src/scr_array.c b/packages/runtime/src/scr_array.c index 1f4efc321..c21589031 100644 --- a/packages/runtime/src/scr_array.c +++ b/packages/runtime/src/scr_array.c @@ -162,7 +162,7 @@ static uint64_t scr_arr_retain_state_slot(const ScrArr *a, uint64_t slot, typedef struct { uint64_t *data; - uint8_t *present; + uint8_t *states; /* the end of the dense states (scr_arr_states_end) */ size_t cap; ScrArrSparseSlot *sparse; size_t sparse_len; @@ -318,18 +318,18 @@ static void scr_arr_sparse_drop_prefix(ScrArr *a, size_t count) { static bool scr_arr_inline(const ScrArr *a) { return (a->spare & SCR_ARR_INLINE_SLOTS) != 0; } /* Head room: removing elements from the front of a dense array of 8-byte - * slots with its own allocation (shift, splice(0, n)) advances `data` and - * `present` past them and lowers `cap`, instead of moving the tail down, so - * draining a queue costs O(1) per element. The vacated slots stay in the - * allocation in front of `data` (values) and of `present` (states, all - * holes) until an unshift reuses them, the array empties, or growth moves - * the elements back to the start of the allocation (scr_arr_drop_head). - * `spare` records that an array may have head room; its size is implied, - * because the states still follow the whole value area: - * present == (uint8_t *)(data + cap) + head. - * Every reader indexes from `data`, `present` and `cap` as before (emitted - * code included); only the code that frees, reallocates or swaps the - * allocation needs its start. Static and inline storage never has head room. */ + * slots with its own allocation (shift, splice(0, n)) advances `data` past + * them and lowers `cap`, instead of moving the tail down, so draining a + * queue costs O(1) per element. The states run backward from the end of the + * allocation (scr_arr_states_end), so the remaining ones stay where they + * are. The vacated slots (in front of `data`) and states (past the states + * end, all holes) stay in the allocation until an unshift reuses them, the + * array empties, or growth moves the elements back to the start of the + * allocation (scr_arr_drop_head). `spare` records head room, and the vacated + * slot just before `data` holds its size (scr_arr_head). Every reader + * indexes from `data` and `cap` as before (emitted code included); only the + * code that frees, reallocates or swaps the allocation needs its start. + * Static and inline storage never has head room. */ #define SCR_ARR_HEAD_ROOM 2u /* An unshift had to move every element: the next one that would have to * move them again opens head room instead (scr_arr_open_head). */ @@ -338,8 +338,17 @@ static bool scr_arr_inline(const ScrArr *a) { return (a->spare & SCR_ARR_INLINE_ #define SCR_ARR_HEAD_MIN ((size_t)16) static size_t scr_arr_head(const ScrArr *a) { - if (!(a->spare & SCR_ARR_HEAD_ROOM)) return 0; - return (size_t)(a->present - (uint8_t *)(a->data + a->cap)); + return (a->spare & SCR_ARR_HEAD_ROOM) ? (size_t)a->data[-1] : 0; +} + +/* Record `head` vacated slots in front of `data` (none: no head room). */ +static void scr_arr_set_head(ScrArr *a, size_t head) { + if (head == 0) { + a->spare &= (uint16_t)~SCR_ARR_HEAD_ROOM; + return; + } + a->data[-1] = head; + a->spare |= SCR_ARR_HEAD_ROOM; } /* The start of the dense allocation. */ @@ -352,17 +361,18 @@ static void scr_arr_drop_head(ScrArr *a) { a->spare &= (uint16_t)~SCR_ARR_HEAD_ROOM; if (head == 0) return; size_t cap = a->cap, total = cap + head, live = a->len < cap ? a->len : cap; + uint8_t *end = scr_arr_states_end(a); uint64_t *data = a->data - head; - uint8_t *present = (uint8_t *)(data + total); + /* The states end moves to the end of the allocation, past the `head` + * vacated states: the live states move up with it, and every other state + * becomes a hole. */ + uint8_t *full = end + head; if (live) { memmove(data, a->data, live * sizeof(uint64_t)); - memmove(present, a->present, live); + memmove(full - live, end - live, live); } - /* States past `live` were holes already, except the `head` that now sit - * where the moved states were. */ - memset(present + live, SCR_ARR_HOLE, head); + memset(full - total, SCR_ARR_HOLE, total - live); a->data = data; - a->present = present; a->cap = total; } @@ -373,29 +383,36 @@ static void scr_arr_drop_head(ScrArr *a) { static bool scr_arr_drop_front(ScrArr *a, size_t count) { if (a->elem == SCR_ELEM_BOOL || scr_arr_inline(a)) return false; if (count == 0) return true; - memset(a->present, SCR_ARR_HOLE, count); + size_t head = scr_arr_head(a); + memset(scr_arr_state_run(a, 0, count), SCR_ARR_HOLE, count); memset(a->data, 0, count * sizeof(uint64_t)); + if (head) a->data[-1] = 0; a->data += count; - a->present += count; a->cap -= count; a->len -= count; a->filled = a->filled > count ? (uint32_t)(a->filled - count) : 0; - a->spare |= SCR_ARR_HEAD_ROOM; + head += count; if (a->len == 0) { /* Every state in the allocation is a hole: start over at its front. */ - size_t head = scr_arr_head(a); a->data -= head; - a->present -= head; a->cap += head; a->spare &= (uint16_t)~SCR_ARR_HEAD_ROOM; + } else { + scr_arr_set_head(a, head); } return true; } -/* Slots requested by the small constructions, rounded up to fill a block - * on the 16-byte grid: 5 eight-byte slots (or 8 booleans) after a plain - * header. */ +/* Slots requested by the small constructions. The small allocator rounds + * them up to fill a block on the 16-byte grid: 7 eight-byte slots after a + * plain header (6 after a collector header) in a 112-byte block, or 8 + * booleans in a 64-byte block. Programs that free and reuse many small + * arrays run slower with 96-byte blocks of 5 slots. */ +#if SCR_SMALL_ALLOC +#define SCR_ARR_SMALL_SLOTS ((size_t)6) +#else #define SCR_ARR_SMALL_SLOTS ((size_t)4) +#endif /* Inline capacity of a small construction with `prefix` bytes of block * before the slots. */ @@ -413,9 +430,8 @@ static size_t scr_arr_small_cap(size_t prefix, bool bools) { static void scr_arr_use_inline(ScrArr *a, size_t cap) { a->data = (uint64_t *)((char *)a + SCR_ARR_INLINE_OFFSET); - a->present = (uint8_t *)a->data + cap * scr_arr_slot_width(a); - memset(a->present, SCR_ARR_HOLE, cap); - a->cap = cap; + a->cap = (uint32_t)cap; + memset(scr_arr_state_run(a, 0, cap), SCR_ARR_HOLE, cap); a->spare |= SCR_ARR_INLINE_SLOTS; } @@ -425,12 +441,11 @@ static void scr_arr_move_inline(ScrArr *a, size_t cap, size_t unit) { size_t width = scr_arr_slot_width(a); uint64_t *data = scr_mem_alloc(cap * unit); if (!data) scr_arr_oom(); - uint8_t *present = (uint8_t *)data + cap * width; + uint8_t *end = (uint8_t *)data + cap * (width + 1); memcpy(data, a->data, a->cap * width); - memcpy(present, a->present, a->cap); - memset(present + a->cap, SCR_ARR_HOLE, cap - a->cap); + memcpy(end - a->cap, scr_arr_state_run(a, 0, a->cap), a->cap); + memset(end - cap, SCR_ARR_HOLE, cap - a->cap); a->data = data; - a->present = present; a->cap = cap; a->spare &= (uint16_t)~SCR_ARR_INLINE_SLOTS; } @@ -495,12 +510,11 @@ static inline void scr_arr_grow_dense_to(ScrArr *a, size_t need, ScrArrGrow mode uint64_t *data = a->data ? scr_mem_realloc(a->data, cap * SCR_ARR_SLOT_BYTES) : scr_mem_alloc(cap * SCR_ARR_SLOT_BYTES); if (!data) scr_arr_oom(); - /* The states follow the values: move them past the larger value area. */ - uint8_t *present = (uint8_t *)(data + cap); - if (old) memmove(present, (uint8_t *)(data + old), old); - memset(present + old, SCR_ARR_HOLE, cap - old); + /* The states end the allocation: move them to its new end. */ + uint8_t *end = (uint8_t *)(data + cap) + cap; + if (old) memmove(end - old, (uint8_t *)(data + old), old); + memset(end - cap, SCR_ARR_HOLE, cap - old); a->data = data; - a->present = present; a->cap = cap; } @@ -509,7 +523,7 @@ static void scr_arr_absorb_sparse(ScrArr *a) { size_t moved = scr_arr_sparse_lower_bound(SCR_AX(a)->sparse, SCR_AX(a)->sparse_len, a->cap); for (size_t i = 0; i < moved; i++) { scr_arr_dense_put(a, SCR_AX(a)->sparse[i].index, SCR_AX(a)->sparse[i].slot); - a->present[SCR_AX(a)->sparse[i].index] = SCR_AX(a)->sparse[i].state; + *scr_arr_state_ptr(a, SCR_AX(a)->sparse[i].index) = SCR_AX(a)->sparse[i].state; } scr_arr_sparse_drop_prefix(a, moved); } @@ -560,21 +574,19 @@ static void scr_arr_grow_dense_slow(ScrArr *a, size_t need, ScrArrGrow mode) { uint64_t *data = scr_mem_alloc(cap * unit); if (!data) scr_arr_oom(); a->data = data; - a->present = (uint8_t *)data + cap * width; - memset(a->present, SCR_ARR_HOLE, cap); - a->cap = cap; + a->cap = (uint32_t)cap; + memset(scr_arr_state_run(a, 0, cap), SCR_ARR_HOLE, cap); return; } - /* Values and states share one allocation. Move the old states before - * publishing the larger capacity: their previous location is now part - * of the value area. Existing data and state pointers keep their ABI. */ + /* Values and states share one allocation. Move the old states to its new + * end before publishing the larger capacity: their previous location is + * now part of the value area. */ uint64_t *data = scr_mem_realloc(a->data, cap * unit); if (!data) scr_arr_oom(); - uint8_t *present = (uint8_t *)data + cap * width; - if (a->cap) memmove(present, (uint8_t *)data + a->cap * width, a->cap); - memset(present + a->cap, SCR_ARR_HOLE, cap - a->cap); + uint8_t *end = (uint8_t *)data + cap * (width + 1); + if (a->cap) memmove(end - a->cap, (uint8_t *)data + a->cap * width, a->cap); + memset(end - cap, SCR_ARR_HOLE, cap - a->cap); a->data = data; - a->present = present; a->cap = cap; scr_arr_absorb_sparse(a); } @@ -584,7 +596,7 @@ static void scr_arr_grow_dense(ScrArr *a, size_t need) { } /* Room for `count` values in front of the first element of a dense array: - * moves `data` and `present` back into head room, raising `cap`; the caller + * moves `data` back into head room, raising `cap`; the caller * stores the new first `count` slots and states. A shifted array reuses the * room its shifts left. An array that unshift already had to move once moves * into a new allocation with head room for half its length again, so a run @@ -607,19 +619,20 @@ static bool scr_arr_open_head(ScrArr *a, size_t count) { size_t total = room + a->cap; uint64_t *block = scr_mem_alloc(total * SCR_ARR_SLOT_BYTES); if (!block) scr_arr_oom(); - uint8_t *states = (uint8_t *)(block + total); + /* The window's states end `room` bytes before the allocation does; the + * room's states past it are holes. */ + uint8_t *end = (uint8_t *)(block + total) + a->cap; memcpy(block + room, a->data, len * sizeof(uint64_t)); - memset(states, SCR_ARR_HOLE, room); - memcpy(states + room, a->present, len); - memset(states + room + len, SCR_ARR_HOLE, a->cap - len); + memset(end, SCR_ARR_HOLE, room); + memcpy(end - len, scr_arr_state_run(a, 0, len), len); + memset(end - a->cap, SCR_ARR_HOLE, a->cap - len); scr_mem_free(scr_arr_block(a)); a->data = block + room; - a->present = states + room; - a->spare |= SCR_ARR_HEAD_ROOM; + head = room; } a->data -= count; - a->present -= count; a->cap += count; + scr_arr_set_head(a, head - count); return true; } @@ -628,9 +641,10 @@ static bool scr_arr_open_head(ScrArr *a, size_t count) { * within that gap past capacity. */ static bool scr_arr_dense_tail_populated(const ScrArr *a) { size_t window = a->cap < SCR_ARR_DENSE_GAP ? a->cap : SCR_ARR_DENSE_GAP; - if (window == 0 || a->present[a->cap - 1] == SCR_ARR_HOLE) return false; + const uint8_t *states = scr_arr_states_end(a); + if (window == 0 || *scr_arr_state_in(states, a->cap - 1) == SCR_ARR_HOLE) return false; size_t populated = 0; - for (size_t i = a->cap - window; i < a->cap; i++) populated += a->present[i] != SCR_ARR_HOLE; + for (size_t i = a->cap - window; i < a->cap; i++) populated += *scr_arr_state_in(states, i) != SCR_ARR_HOLE; return populated * 16 >= window * 3; } @@ -672,9 +686,9 @@ static void scr_arr_grow_for_index(ScrArr *a, size_t index) { static uint8_t scr_arr_state_at_storage(const ScrArrStorage *s, size_t index, uint64_t *out) { if (index < s->cap) { - if (s->present[index] == SCR_ARR_HOLE) return SCR_ARR_HOLE; + if (*scr_arr_state_in(s->states, index) == SCR_ARR_HOLE) return SCR_ARR_HOLE; if (out) *out = scr_arr_storage_get(s, index); - return s->present[index]; + return *scr_arr_state_in(s->states, index); } size_t pos = scr_arr_sparse_lower_bound(s->sparse, s->sparse_len, index); if (pos == s->sparse_len || s->sparse[pos].index != index) return SCR_ARR_HOLE; @@ -688,13 +702,13 @@ static bool scr_arr_slot_at_storage(const ScrArrStorage *s, size_t index, } static bool scr_arr_slot_at(const ScrArr *a, size_t index, uint64_t *out) { - ScrArrStorage s = {a->data, a->present, a->cap, SCR_AX(a)->sparse, + ScrArrStorage s = {a->data, scr_arr_states_end(a), a->cap, SCR_AX(a)->sparse, SCR_AX(a)->sparse_len, SCR_AX(a)->sparse_cap, a->elem, NULL}; return scr_arr_slot_at_storage(&s, index, out); } static uint8_t scr_arr_state_at(const ScrArr *a, size_t index, uint64_t *out) { - ScrArrStorage s = {a->data, a->present, a->cap, SCR_AX(a)->sparse, + ScrArrStorage s = {a->data, scr_arr_states_end(a), a->cap, SCR_AX(a)->sparse, SCR_AX(a)->sparse_len, SCR_AX(a)->sparse_cap, a->elem, NULL}; return scr_arr_state_at_storage(&s, index, out); } @@ -707,7 +721,7 @@ static void scr_arr_store_state_owned(ScrArr *a, size_t index, uint64_t slot, scr_arr_grow_for_index(a, index); if (index < a->cap) { scr_arr_dense_put(a, index, slot); - a->present[index] = state; + *scr_arr_state_ptr(a, index) = state; return; } size_t pos = scr_arr_sparse_lower_bound(SCR_AX(a)->sparse, SCR_AX(a)->sparse_len, index); @@ -726,11 +740,11 @@ static void scr_arr_store_owned(ScrArr *a, size_t index, uint64_t slot) { static bool scr_arr_take_state(ScrArr *a, size_t index, uint64_t *out, uint8_t *state_out) { if (index < a->cap) { - uint8_t state = a->present[index]; + uint8_t state = *scr_arr_state_ptr(a, index); if (state == SCR_ARR_HOLE) return false; scr_arr_cut(a, index); *out = scr_arr_dense_get(a, index); - a->present[index] = SCR_ARR_HOLE; + *scr_arr_state_ptr(a, index) = SCR_ARR_HOLE; scr_arr_dense_put(a, index, 0); if (state_out) *state_out = state; return true; @@ -754,10 +768,10 @@ static void scr_arr_replace_state_owned(ScrArr *a, size_t index, uint64_t slot, uint64_t old = 0; if (state != SCR_ARR_VALUE) scr_arr_cut(a, index); if (index < a->cap) { - old_state = a->present[index]; + old_state = *scr_arr_state_ptr(a, index); if (old_state == SCR_ARR_VALUE) old = scr_arr_dense_get(a, index); scr_arr_dense_put(a, index, slot); - a->present[index] = state; + *scr_arr_state_ptr(a, index) = state; } else { size_t pos = scr_arr_sparse_lower_bound(SCR_AX(a)->sparse, SCR_AX(a)->sparse_len, index); if (pos == SCR_AX(a)->sparse_len || SCR_AX(a)->sparse[pos].index != index) { @@ -788,7 +802,7 @@ static void scr_arr_refill_to(ScrArr *a, size_t limit) { size_t end = a->len < a->cap ? a->len : a->cap; if (limit < end) end = limit; size_t f = a->filled; - if (f < end) f = scr_arr_value_run(a->present, f, end); + if (f < end) f = scr_arr_value_run(scr_arr_states_end(a), f, end); a->filled = (uint32_t)f; } @@ -867,12 +881,11 @@ static bool scr_arr_prop_delete(ScrArr *a, double key) { } static ScrArrStorage scr_arr_take_storage(ScrArr *a) { - ScrArrStorage old = {a->data, a->present, a->cap, SCR_AX(a)->sparse, + ScrArrStorage old = {a->data, scr_arr_states_end(a), a->cap, SCR_AX(a)->sparse, SCR_AX(a)->sparse_len, SCR_AX(a)->sparse_cap, a->elem, scr_arr_inline(a) ? NULL : scr_arr_block(a)}; a->spare &= (uint16_t)~(SCR_ARR_INLINE_SLOTS | SCR_ARR_HEAD_ROOM); a->data = NULL; - a->present = NULL; a->cap = 0; a->filled = 0; if (a->ext) { @@ -900,16 +913,81 @@ static void scr_arr_move_dense(ScrArr *a, size_t to, size_t from, size_t count) if (count == 0) return; scr_arr_cut(a, to); memmove(scr_arr_dense_at(a, to), scr_arr_dense_at(a, from), count * scr_arr_slot_width(a)); - memmove(a->present + to, a->present + from, count); + memmove(scr_arr_state_run(a, to, count), scr_arr_state_run(a, from, count), count); } static void scr_arr_clear_dense(ScrArr *a, size_t from, size_t count) { if (count == 0) return; scr_arr_cut(a, from); - memset(a->present + from, SCR_ARR_HOLE, count); + memset(scr_arr_state_run(a, from, count), SCR_ARR_HOLE, count); memset(scr_arr_dense_at(a, from), 0, count * scr_arr_slot_width(a)); } +/* Copy `n` state bytes between distinct arrays. Most runs are a few bytes, + * which overlapping word copies move with fewer instructions than a call. */ +static inline void scr_arr_copy_states(uint8_t *dst, const uint8_t *src, size_t n) { + if (n >= 8 && n <= 16) { + uint64_t head, tail; + memcpy(&head, src, sizeof head); + memcpy(&tail, src + n - 8, sizeof tail); + memcpy(dst, &head, sizeof head); + memcpy(dst + n - 8, &tail, sizeof tail); + } else if (n >= 4 && n < 8) { + uint32_t head, tail; + memcpy(&head, src, sizeof head); + memcpy(&tail, src + n - 4, sizeof tail); + memcpy(dst, &head, sizeof head); + memcpy(dst + n - 4, &tail, sizeof tail); + } else if (n < 4) { + for (size_t i = 0; i < n; i++) dst[i] = src[i]; + } else { + memcpy(dst, src, n); + } +} + +/* Set `n` state bytes to `state`, with overlapping word stores for short + * runs. */ +static inline void scr_arr_set_states(uint8_t *dst, uint8_t state, size_t n) { + if (n >= 8 && n <= 16) { + uint64_t word = 0x0101010101010101ull * state; + memcpy(dst, &word, sizeof word); + memcpy(dst + n - 8, &word, sizeof word); + } else if (n >= 4 && n < 8) { + uint32_t word = 0x01010101u * state; + memcpy(dst, &word, sizeof word); + memcpy(dst + n - 4, &word, sizeof word); + } else if (n < 4) { + for (size_t i = 0; i < n; i++) dst[i] = state; + } else { + memset(dst, state, n); + } +} + +/* Copy `n` 8-byte slots between distinct arrays: up to 8 slots move as + * overlapping 16- or 32-byte blocks rather than through a call. */ +static inline void scr_arr_copy_slots(uint64_t *dst, const uint64_t *src, size_t n) { + size_t bytes = n * sizeof(uint64_t); + char *d = (char *)dst; + const char *s = (const char *)src; + if (bytes >= 32 && bytes <= 64) { + char head[32], tail[32]; + memcpy(head, s, 32); + memcpy(tail, s + bytes - 32, 32); + memcpy(d, head, 32); + memcpy(d + bytes - 32, tail, 32); + } else if (bytes >= 16 && bytes < 32) { + char head[16], tail[16]; + memcpy(head, s, 16); + memcpy(tail, s + bytes - 16, 16); + memcpy(d, head, 16); + memcpy(d + bytes - 16, tail, 16); + } else if (bytes == 8) { + memcpy(d, s, 8); + } else if (bytes) { + memcpy(d, s, bytes); + } +} + /* The destination owns no values in this range. Only retain adapters run * while copying references: no release can collect an intermediate copy. * Self-appends use disjoint ranges after reserving capacity. Both arrays @@ -919,29 +997,36 @@ static void scr_arr_copy_dense(ScrArr *dst, size_t to, const ScrArr *src, bool materialize_holes) { if (count == 0) return; scr_arr_cut(dst, to); - if (!reverse && dst->elem != SCR_ELEM_BOOL) { - /* 8-byte slots: a constant slot size. */ - memcpy(dst->data + to, src->data + from, count * sizeof(uint64_t)); - memcpy(dst->present + to, src->present + from, count); + /* 8-byte slots: a constant slot size, and constant-width state ends. */ + bool wide = dst->elem != SCR_ELEM_BOOL; + uint8_t *dst_states = wide ? scr_arr_states_end_w(dst, 8) : scr_arr_states_end_w(dst, 1); + const uint8_t *src_states = wide ? scr_arr_states_end_w(src, 8) : scr_arr_states_end_w(src, 1); + if (!reverse && wide) { + scr_arr_copy_slots(dst->data + to, src->data + from, count); + /* A range below the source's hole-free prefix holds only values. */ + if (from + count <= src->filled) + scr_arr_set_states(dst_states - to - count, SCR_ARR_VALUE, count); + else + scr_arr_copy_states(dst_states - to - count, src_states - from - count, count); } else if (!reverse) { memcpy(scr_arr_dense_at(dst, to), scr_arr_dense_at(src, from), count * scr_arr_slot_width(dst)); - memcpy(dst->present + to, src->present + from, count); + scr_arr_copy_states(dst_states - to - count, src_states - from - count, count); } else { for (size_t i = 0; i < count; i++) { size_t source = from + count - i - 1; scr_arr_dense_put(dst, to + i, scr_arr_dense_get(src, source)); - dst->present[to + i] = src->present[source]; + *scr_arr_state_in(dst_states, to + i) = *scr_arr_state_in(src_states, source); } } bool refs = scr_elem_is_ref(src->elem); if (!refs && !materialize_holes) return; for (size_t i = to; i < to + count; i++) { - if (dst->present[i] == SCR_ARR_VALUE) { + if (*scr_arr_state_in(dst_states, i) == SCR_ARR_VALUE) { if (refs) dst->data[i] = scr_elem_retain_slot(src, dst->data[i]); - } else if (materialize_holes && dst->present[i] == SCR_ARR_HOLE) { + } else if (materialize_holes && *scr_arr_state_in(dst_states, i) == SCR_ARR_HOLE) { scr_arr_dense_put(dst, i, 0); - dst->present[i] = SCR_ARR_UNDEFINED; + *scr_arr_state_in(dst_states, i) = SCR_ARR_UNDEFINED; } } } @@ -949,13 +1034,17 @@ static void scr_arr_copy_dense(ScrArr *dst, size_t to, const ScrArr *src, /* Store `count` 8-byte value slots into dense storage starting at `to`. */ static void scr_arr_put_slots(ScrArr *a, size_t to, size_t count, const uint64_t *slots) { if (a->elem != SCR_ELEM_BOOL) { - memcpy(a->data + to, slots, count * sizeof(*slots)); + scr_arr_copy_slots(a->data + to, slots, count); return; } for (size_t i = 0; i < count; i++) scr_arr_dense_put(a, to + i, slots[i]); } -static ScrArr *scr_arr_make(ScrElemKind elem, size_t initial_cap, bool small) { +/* Inlined into each constructor, so `small` and the slot unit fold to + * constants there. */ +static inline __attribute__((always_inline)) ScrArr *scr_arr_make(ScrElemKind elem, + size_t initial_cap, + bool small) { size_t unit = elem == SCR_ELEM_BOOL ? SCR_ARR_BOOL_SLOT_BYTES : SCR_ARR_SLOT_BYTES; size_t inline_cap = small ? scr_arr_small_cap(SCR_ARR_INLINE_OFFSET, elem == SCR_ELEM_BOOL) : 0; ScrArr *a = scr_mem_alloc(inline_cap ? SCR_ARR_INLINE_OFFSET + inline_cap * unit : sizeof(ScrArr)); @@ -969,7 +1058,6 @@ static ScrArr *scr_arr_make(ScrElemKind elem, size_t initial_cap, bool small) { a->filled = 0; a->ops = NULL; a->data = NULL; - a->present = NULL; a->ext = NULL; if (inline_cap) scr_arr_use_inline(a, inline_cap); else if (initial_cap > 0) scr_arr_grow_dense_to(a, initial_cap, SCR_ARR_GROW_EXACT); @@ -1004,7 +1092,6 @@ ScrArr *scr_arr_new_inline(ScrElemKind elem, size_t cap) { a->filled = 0; a->ops = NULL; a->data = NULL; - a->present = NULL; a->ext = NULL; scr_arr_use_inline(a, cap); #ifdef SCR_RC_AUDIT @@ -1051,8 +1138,9 @@ void scr_arr_trace_v(void *a0, ScrTraceVisit visit, void *ctx) { if (SCR_AX(a)->metadata) visit(SCR_AX(a)->metadata, ctx); /* Growth and truncation leave spare capacity empty. Sparse slots and * named properties are separate edges, independent of dense length. */ + const uint8_t *states = scr_arr_states_end(a); for (size_t i = 0, end = a->len < a->cap ? a->len : a->cap; i < end; i++) { - if (a->present[i] == SCR_ARR_VALUE) visit(scr_slot_to_ptr(a->data[i]), ctx); + if (*scr_arr_state_in(states, i) == SCR_ARR_VALUE) visit(scr_slot_to_ptr(a->data[i]), ctx); } for (size_t i = 0; i < SCR_AX(a)->sparse_len; i++) { if (SCR_AX(a)->sparse[i].state == SCR_ARR_VALUE) { @@ -1109,7 +1197,6 @@ static ScrArr *scr_arr_make_ops(const ScrArrOps *ops, size_t initial_cap, bool s a->filled = 0; a->ops = ops; a->data = NULL; - a->present = NULL; a->ext = NULL; if (inline_cap) scr_arr_use_inline(a, inline_cap); else if (initial_cap > 0) scr_arr_grow_dense_to(a, initial_cap, SCR_ARR_GROW_EXACT); @@ -1138,18 +1225,25 @@ static void scr_arr_destroy(void *object) { * first (and the storage cannot move under the loop); immortal strings * (interned characters, literals) need no call. */ const uint64_t *data = a->data; - const uint8_t *present = a->present; - for (size_t i = 0; i < end; i++) { - if (present[i] != SCR_ARR_VALUE) continue; + const uint8_t *states = scr_arr_states_end_w(a, 8); + /* Slots below the hole-free prefix bound hold values. */ + size_t filled = a->filled < end ? a->filled : end; + for (size_t i = 0; i < filled; i++) { + ScrStr *s = (ScrStr *)scr_slot_to_ptr(data[i]); + if (s && s->rc != SIZE_MAX) scr_str_release(s); + } + for (size_t i = filled; i < end; i++) { + if (*scr_arr_state_in(states, i) != SCR_ARR_VALUE) continue; ScrStr *s = (ScrStr *)scr_slot_to_ptr(data[i]); if (s && s->rc != SIZE_MAX) scr_str_release(s); } } else { + uint8_t *states = scr_arr_states_end(a); for (size_t i = 0; i < end; i++) { - if (a->present[i] != SCR_ARR_HOLE) { + if (*scr_arr_state_in(states, i) != SCR_ARR_HOLE) { uint64_t old = a->data[i]; - uint8_t state = a->present[i]; - a->present[i] = 0; + uint8_t state = *scr_arr_state_in(states, i); + *scr_arr_state_in(states, i) = 0; a->data[i] = 0; if (state == SCR_ARR_VALUE) scr_elem_release(a, old); } @@ -1221,8 +1315,9 @@ double scr_arr_next_present(const ScrArr *a, double start) { if (!(start < SCR_ARR_LEN_F64(a->len))) return SCR_ARR_LEN_F64(a->len); size_t i = start > 0 ? (size_t)ceil(start) : 0; size_t end = a->cap < a->len ? a->cap : a->len; + const uint8_t *states = scr_arr_states_end(a); for (; i < end; i++) { - if (a->present[i] != SCR_ARR_HOLE) return (double)i; + if (*scr_arr_state_in(states, i) != SCR_ARR_HOLE) return (double)i; } size_t pos = scr_arr_sparse_lower_bound(SCR_AX(a)->sparse, SCR_AX(a)->sparse_len, i); if (pos < SCR_AX(a)->sparse_len && SCR_AX(a)->sparse[pos].index < a->len) { @@ -1252,10 +1347,10 @@ void scr_arr_set_len(ScrArr *a, double length) { size_t dense_stop = next < dense_end ? next : dense_end; if (scr_elem_is_ref(a->elem)) { for (size_t i = dense_stop; i < dense_end; i++) { - if (a->present[i] != SCR_ARR_HOLE) { + if (*scr_arr_state_ptr(a, i) != SCR_ARR_HOLE) { uint64_t old = a->data[i]; - uint8_t state = a->present[i]; - a->present[i] = 0; + uint8_t state = *scr_arr_state_ptr(a, i); + *scr_arr_state_ptr(a, i) = 0; a->data[i] = 0; if (state == SCR_ARR_VALUE) scr_elem_release(a, old); } @@ -1272,7 +1367,7 @@ void scr_arr_set_len(ScrArr *a, double length) { } } else { if (dense_stop < dense_end) { - memset(a->present + dense_stop, 0, dense_end - dense_stop); + memset(scr_arr_state_run(a, dense_stop, dense_end - dense_stop), 0, dense_end - dense_stop); memset(scr_arr_dense_at(a, dense_stop), 0, (dense_end - dense_stop) * scr_arr_slot_width(a)); } @@ -1313,8 +1408,9 @@ static void scr_arr_copy_range_states(ScrArr *dst, size_t dst_start, const ScrAr bool materialize_holes) { if (src != dst && src_start <= src->cap && count <= src->cap - src_start && dst_start <= dst->cap && count <= dst->cap - dst_start) { + const uint8_t *dst_states = scr_arr_states_end(dst); size_t empty = 0; - while (empty < count && dst->present[dst_start + empty] == SCR_ARR_HOLE) empty++; + while (empty < count && *scr_arr_state_in(dst_states, dst_start + empty) == SCR_ARR_HOLE) empty++; if (empty == count) { scr_arr_copy_dense(dst, dst_start, src, src_start, count, reverse, materialize_holes); @@ -1401,7 +1497,7 @@ static ScrArr *scr_arr_fill_slot(ScrArr *a, uint64_t slot, uint8_t state, if (state != SCR_ARR_VALUE) scr_arr_cut(a, from); if (a->elem == SCR_ELEM_BOOL) memset(scr_arr_dense_at(a, from), (int)slot, until - from); else for (size_t i = from; i < until; i++) a->data[i] = slot; - memset(a->present + from, state, until - from); + memset(scr_arr_state_run(a, from, until - from), state, until - from); } else { for (size_t i = from; i < until; i++) { uint64_t value = refs && state == SCR_ARR_VALUE ? scr_elem_retain_slot(a, slot) : slot; @@ -1555,7 +1651,7 @@ void *scr_arr_peek_ref(const ScrArr *a, double i) { size_t idx = (size_t)i; if ((double)idx == i) { if (idx < a->filled) return scr_slot_to_ptr(a->data[idx]); - if (idx >= a->len || a->present[idx] != SCR_ARR_VALUE) return NULL; + if (idx >= a->len || *scr_arr_state_ptr(a, idx) != SCR_ARR_VALUE) return NULL; return scr_slot_to_ptr(a->data[idx]); } } @@ -1585,8 +1681,8 @@ bool scr_arr_index_eq(const ScrArr *a, double i, const ScrArr *b, double j) { if (i >= 0 && i < (double)a->cap && i == trunc(i) && j >= 0 && j < (double)b->cap && j == trunc(j)) { size_t ai = (size_t)i, bi = (size_t)j; - bool ap = ai < a->len && a->present[ai] == SCR_ARR_VALUE; - bool bp = bi < b->len && b->present[bi] == SCR_ARR_VALUE; + bool ap = ai < a->len && *scr_arr_state_ptr(a, ai) == SCR_ARR_VALUE; + bool bp = bi < b->len && *scr_arr_state_ptr(b, bi) == SCR_ARR_VALUE; if (!ap || !bp) return ap == bp; av = scr_arr_dense_get(a, ai); bv = scr_arr_dense_get(b, bi); @@ -1700,12 +1796,17 @@ double scr_arr_push_many(ScrArr *a, size_t count, const uint64_t *slots) { scr_arr_grow_dense(a, next); if (count) { scr_arr_put_slots(a, from, count, slots); - memset(a->present + from, SCR_ARR_VALUE, count); + scr_arr_set_states(scr_arr_state_run(a, from, count), SCR_ARR_VALUE, count); } - } else { - for (size_t i = 0; i < count; i++) scr_arr_store_owned(a, from + i, slots[i]); + a->len = (uint32_t)next; + /* The appended states are values: a bound at the old length moves to + * the new one without reading them back. */ + if (a->filled == from) a->filled = (uint32_t)next; + else scr_arr_refill(a); + return (double)next; } - a->len = next; + for (size_t i = 0; i < count; i++) scr_arr_store_owned(a, from + i, slots[i]); + a->len = (uint32_t)next; scr_arr_refill(a); return (double)next; } @@ -1786,7 +1887,7 @@ double scr_arr_concat_copy(ScrArr *a, const ScrArr *src) { } a->len = old_len + add; for (size_t i = 0; i < src->cap; i++) { - if (src->present[i] != SCR_ARR_HOLE) { + if (*scr_arr_state_ptr(src, i) != SCR_ARR_HOLE) { scr_arr_copy_index(a, old_len + i, src, i); } } @@ -1813,7 +1914,7 @@ static double scr_arr_unshift_slot(ScrArr *a, uint64_t slot) { /* The prefix stays put in front of the new first value. */ size_t f = a->filled; a->data[0] = slot; - a->present[0] = SCR_ARR_VALUE; + *scr_arr_state_ptr(a, 0) = SCR_ARR_VALUE; a->len++; a->filled = (uint32_t)(f + 1); return SCR_ARR_LEN_F64(a->len); @@ -1824,7 +1925,7 @@ static double scr_arr_unshift_slot(ScrArr *a, uint64_t slot) { size_t f = a->filled; scr_arr_move_dense(a, 1, 0, old_len); scr_arr_dense_put(a, 0, slot); - a->present[0] = SCR_ARR_VALUE; + *scr_arr_state_ptr(a, 0) = SCR_ARR_VALUE; a->len++; a->filled = (uint32_t)(f + 1); return SCR_ARR_LEN_F64(a->len); @@ -1833,8 +1934,8 @@ static double scr_arr_unshift_slot(ScrArr *a, uint64_t slot) { a->len = 0; scr_arr_store_owned(a, 0, slot); for (size_t i = 0; i < old.cap; i++) { - if (old.present[i]) - scr_arr_store_state_owned(a, i + 1, scr_arr_storage_get(&old, i), old.present[i]); + if (*scr_arr_state_in(old.states, i)) + scr_arr_store_state_owned(a, i + 1, scr_arr_storage_get(&old, i), *scr_arr_state_in(old.states, i)); } for (size_t i = 0; i < old.sparse_len; i++) { scr_arr_store_state_owned(a, old.sparse[i].index + 1, old.sparse[i].slot, @@ -1865,7 +1966,7 @@ double scr_arr_unshift_many(ScrArr *a, size_t count, const uint64_t *slots) { if (scr_arr_is_dense(a) && scr_arr_open_head(a, count)) { size_t f = a->filled; scr_arr_put_slots(a, 0, count, slots); - memset(a->present, SCR_ARR_VALUE, count); + memset(scr_arr_state_run(a, 0, count), SCR_ARR_VALUE, count); a->len = old_len + count; a->filled = (uint32_t)(f + count); return SCR_ARR_LEN_F64(a->len); @@ -1875,7 +1976,7 @@ double scr_arr_unshift_many(ScrArr *a, size_t count, const uint64_t *slots) { size_t f = a->filled; scr_arr_move_dense(a, count, 0, old_len); scr_arr_put_slots(a, 0, count, slots); - memset(a->present, SCR_ARR_VALUE, count); + memset(scr_arr_state_run(a, 0, count), SCR_ARR_VALUE, count); a->len = old_len + count; a->filled = (uint32_t)(f + count); return SCR_ARR_LEN_F64(a->len); @@ -1883,8 +1984,8 @@ double scr_arr_unshift_many(ScrArr *a, size_t count, const uint64_t *slots) { ScrArrStorage old = scr_arr_take_storage(a); for (size_t i = 0; i < count; i++) scr_arr_store_owned(a, i, slots[i]); for (size_t i = 0; i < old.cap; i++) { - if (old.present[i] != SCR_ARR_HOLE) - scr_arr_store_state_owned(a, i + count, scr_arr_storage_get(&old, i), old.present[i]); + if (*scr_arr_state_in(old.states, i) != SCR_ARR_HOLE) + scr_arr_store_state_owned(a, i + count, scr_arr_storage_get(&old, i), *scr_arr_state_in(old.states, i)); } for (size_t i = 0; i < old.sparse_len; i++) { scr_arr_store_state_owned(a, old.sparse[i].index + count, @@ -1933,8 +2034,8 @@ double scr_arr_unshift_spread(ScrArr *a, const ScrArr *src) { } } for (size_t i = 0; i < old.cap; i++) { - if (old.present[i] != SCR_ARR_HOLE) { - scr_arr_store_state_owned(a, i + add, scr_arr_storage_get(&old, i), old.present[i]); + if (*scr_arr_state_in(old.states, i) != SCR_ARR_HOLE) { + scr_arr_store_state_owned(a, i + add, scr_arr_storage_get(&old, i), *scr_arr_state_in(old.states, i)); } } for (size_t i = 0; i < old.sparse_len; i++) { @@ -1952,14 +2053,15 @@ ScrArr *scr_arr_reverse(ScrArr *a) { if (scr_arr_is_dense(a)) { /* A hole-free array stays hole-free; any other prefix is recomputed. */ if (a->filled != len) scr_arr_cut(a, 0); + uint8_t *states = scr_arr_states_end(a); for (size_t i = 0; i < len / 2; i++) { size_t j = len - i - 1; uint64_t slot = scr_arr_dense_get(a, i); scr_arr_dense_put(a, i, scr_arr_dense_get(a, j)); scr_arr_dense_put(a, j, slot); - uint8_t state = a->present[i]; - a->present[i] = a->present[j]; - a->present[j] = state; + uint8_t state = *scr_arr_state_in(states, i); + *scr_arr_state_in(states, i) = *scr_arr_state_in(states, j); + *scr_arr_state_in(states, j) = state; } scr_arr_refill(a); return scr_arr_retain(a); @@ -1967,8 +2069,8 @@ ScrArr *scr_arr_reverse(ScrArr *a) { ScrArrStorage old = scr_arr_take_storage(a); a->len = len; for (size_t i = 0; i < old.cap; i++) { - if (old.present[i]) - scr_arr_store_state_owned(a, len - 1 - i, scr_arr_storage_get(&old, i), old.present[i]); + if (*scr_arr_state_in(old.states, i)) + scr_arr_store_state_owned(a, len - 1 - i, scr_arr_storage_get(&old, i), *scr_arr_state_in(old.states, i)); } for (size_t i = 0; i < old.sparse_len; i++) { scr_arr_store_state_owned(a, len - 1 - old.sparse[i].index, @@ -2018,7 +2120,7 @@ uint8_t scr_arr_shift_state(ScrArr *a, uint64_t *slot_out) { if (a->len == 0) return SCR_ARR_HOLE; size_t old_len = a->len; if (scr_arr_is_dense(a)) { - uint8_t state = a->present[0]; + uint8_t state = *scr_arr_state_ptr(a, 0); if (slot_out && state == SCR_ARR_VALUE) *slot_out = scr_arr_dense_get(a, 0); /* The first slot becomes head room; the tail stays where it is. */ if (scr_arr_drop_front(a, 1)) return state; @@ -2035,8 +2137,8 @@ uint8_t scr_arr_shift_state(ScrArr *a, uint64_t *slot_out) { uint8_t state = scr_arr_state_at_storage(&old, 0, &s); a->len = 0; for (size_t i = 1; i < old.cap; i++) { - if (old.present[i]) - scr_arr_store_state_owned(a, i - 1, scr_arr_storage_get(&old, i), old.present[i]); + if (*scr_arr_state_in(old.states, i)) + scr_arr_store_state_owned(a, i - 1, scr_arr_storage_get(&old, i), *scr_arr_state_in(old.states, i)); } for (size_t i = 0; i < old.sparse_len; i++) { if (old.sparse[i].index > 0) { @@ -2094,7 +2196,7 @@ ScrArr *scr_arr_splice(ScrArr *a, double start, double deleteCount) { if (scr_arr_is_dense(a)) { if (n) { memcpy(out->data, scr_arr_dense_at(a, from), n * scr_arr_slot_width(a)); - memcpy(out->present, a->present + from, n); + memcpy(scr_arr_state_run(out, 0, n), scr_arr_state_run(a, from, n), n); } scr_arr_refill(out); /* A removed prefix becomes head room. */ @@ -2112,8 +2214,8 @@ ScrArr *scr_arr_splice(ScrArr *a, double start, double deleteCount) { ScrArrStorage old = scr_arr_take_storage(a); a->len = 0; for (size_t i = 0; i < old.cap; i++) { - if (old.present[i] == SCR_ARR_HOLE) continue; - uint8_t state = old.present[i]; + if (*scr_arr_state_in(old.states, i) == SCR_ARR_HOLE) continue; + uint8_t state = *scr_arr_state_in(old.states, i); uint64_t slot = scr_arr_storage_get(&old, i); size_t dst; if (i < from) { @@ -2171,7 +2273,7 @@ ScrArr *scr_arr_splice_insert(ScrArr *a, double start, double deleteCount, : scr_arr_new(a->elem, n); if (n) { memcpy(removed->data, scr_arr_dense_at(a, from), n * scr_arr_slot_width(a)); - memcpy(removed->present, a->present + from, n); + memcpy(scr_arr_state_run(removed, 0, n), scr_arr_state_run(a, from, n), n); } removed->len = n; scr_arr_refill(removed); @@ -2229,8 +2331,9 @@ void scr_arr_splice_drop(ScrArr *a, double start, double deleteCount, size_t cou removed = scr_mem_alloc(n * sizeof(uint64_t)); if (!removed) scr_arr_oom(); } + const uint8_t *states = scr_arr_states_end(a); for (size_t i = from; i < from + n; i++) { - if (a->present[i] == SCR_ARR_VALUE) removed[kept++] = a->data[i]; + if (*scr_arr_state_in(states, i) == SCR_ARR_VALUE) removed[kept++] = a->data[i]; } } size_t old_len = a->len; @@ -2245,7 +2348,7 @@ void scr_arr_splice_drop(ScrArr *a, double start, double deleteCount, size_t cou scr_arr_move_dense(a, from + count, from + n, old_len - from - n); if (count) { scr_arr_put_slots(a, from, count, slots); - memset(a->present + from, SCR_ARR_VALUE, count); + memset(scr_arr_state_run(a, from, count), SCR_ARR_VALUE, count); } if (next_len < old_len) scr_arr_clear_dense(a, next_len, old_len - next_len); a->len = next_len; @@ -2302,8 +2405,9 @@ static bool scr_arr_ref_eq(const ScrArr *a, uint64_t slot, void *v) { } double scr_arr_index_of_f64(ScrArr *a, double v) { + const uint8_t *states = scr_arr_states_end(a); for (size_t i = 0; i < a->cap && i < a->len; i++) { - if (a->present[i] == SCR_ARR_VALUE && scr_slot_to_f64(a->data[i]) == v) return (double)i; + if (*scr_arr_state_in(states, i) == SCR_ARR_VALUE && scr_slot_to_f64(a->data[i]) == v) return (double)i; } for (size_t i = 0; i < SCR_AX(a)->sparse_len; i++) { if (SCR_AX(a)->sparse[i].state == SCR_ARR_VALUE && scr_slot_to_f64(SCR_AX(a)->sparse[i].slot) == v) return (double)SCR_AX(a)->sparse[i].index; @@ -2312,8 +2416,9 @@ double scr_arr_index_of_f64(ScrArr *a, double v) { } double scr_arr_index_of_bool(ScrArr *a, bool v) { + const uint8_t *states = scr_arr_states_end(a); for (size_t i = 0; i < a->cap && i < a->len; i++) { - if (a->present[i] == SCR_ARR_VALUE && ((scr_arr_dense_get(a, i) != 0) == v)) return (double)i; + if (*scr_arr_state_in(states, i) == SCR_ARR_VALUE && ((scr_arr_dense_get(a, i) != 0) == v)) return (double)i; } for (size_t i = 0; i < SCR_AX(a)->sparse_len; i++) { if (SCR_AX(a)->sparse[i].state == SCR_ARR_VALUE && (SCR_AX(a)->sparse[i].slot != 0) == v) return (double)SCR_AX(a)->sparse[i].index; @@ -2322,8 +2427,9 @@ double scr_arr_index_of_bool(ScrArr *a, bool v) { } double scr_arr_index_of_ref(ScrArr *a, void *v) { + const uint8_t *states = scr_arr_states_end(a); for (size_t i = 0; i < a->cap && i < a->len; i++) { - if (a->present[i] == SCR_ARR_VALUE && scr_arr_ref_eq(a, a->data[i], v)) return (double)i; + if (*scr_arr_state_in(states, i) == SCR_ARR_VALUE && scr_arr_ref_eq(a, a->data[i], v)) return (double)i; } for (size_t i = 0; i < SCR_AX(a)->sparse_len; i++) { if (SCR_AX(a)->sparse[i].state == SCR_ARR_VALUE && scr_arr_ref_eq(a, SCR_AX(a)->sparse[i].slot, v)) return (double)SCR_AX(a)->sparse[i].index; @@ -2332,8 +2438,9 @@ double scr_arr_index_of_ref(ScrArr *a, void *v) { } bool scr_arr_includes_f64(ScrArr *a, double v) { + const uint8_t *states = scr_arr_states_end(a); for (size_t i = 0; i < a->cap && i < a->len; i++) { - if (a->present[i] != SCR_ARR_VALUE) continue; + if (*scr_arr_state_in(states, i) != SCR_ARR_VALUE) continue; double x = scr_slot_to_f64(a->data[i]); if (x == v || (x != x && v != v)) return true; /* SameValueZero: NaN hits */ } @@ -2353,8 +2460,9 @@ bool scr_arr_includes_ref(ScrArr *a, void *v) { if (a->elem == SCR_ELEM_REF && a->ops->retain == scr_dyn_retain_v) { const ScrDyn *needle = (const ScrDyn *)v; if (needle->kind == SCR_DYN_NUM && needle->v.num != needle->v.num) { + const uint8_t *states = scr_arr_states_end(a); for (size_t i = 0; i < a->cap && i < a->len; i++) { - if (a->present[i] != SCR_ARR_VALUE) continue; + if (*scr_arr_state_in(states, i) != SCR_ARR_VALUE) continue; const ScrDyn *d = (const ScrDyn *)scr_slot_to_ptr(a->data[i]); if (d->kind == SCR_DYN_NUM && d->v.num != d->v.num) return true; } @@ -2416,6 +2524,13 @@ ScrArr *scr_arr_slice(ScrArr *a, double start, double end) { ? scr_arr_new_ops(a->ops, hint ? hint : 1) : scr_arr_new(a->elem, hint ? hint : 1); out->len = n; + if (to <= a->cap && to <= a->filled && a->elem != SCR_ELEM_BOOL && !scr_elem_is_ref(a->elem)) { + /* Numbers below the source's hole-free prefix: values only. */ + scr_arr_copy_slots(out->data, a->data + from, n); + scr_arr_set_states(scr_arr_states_end_w(out, 8) - n, SCR_ARR_VALUE, n); + out->filled = (uint32_t)n; + return out; + } if (to <= a->cap) { scr_arr_copy_dense(out, 0, a, from, n, false, false); /* States copied from below the source's bound are present values. */ @@ -2424,9 +2539,10 @@ ScrArr *scr_arr_slice(ScrArr *a, double start, double end) { scr_arr_refill(out); return out; } + const uint8_t *states = scr_arr_states_end(a); for (size_t i = 0; i < a->cap && i < to; i++) { - if (i < from || a->present[i] == SCR_ARR_HOLE) continue; - uint8_t state = a->present[i]; + if (i < from || *scr_arr_state_in(states, i) == SCR_ARR_HOLE) continue; + uint8_t state = *scr_arr_state_in(states, i); scr_arr_store_state_owned(out, i - from, scr_arr_retain_state_slot(a, scr_arr_dense_get(a, i), state), state); @@ -2552,9 +2668,10 @@ ScrArr *scr_arr_sort_primitive(ScrArr *a, bool copy) { if (a->len == 0) return copy ? scr_arr_new(a->elem, 0) : scr_arr_retain(a); size_t dense = a->len < a->cap ? a->len : a->cap; size_t count = 0, undefined_count = 0; + const uint8_t *states = scr_arr_states_end(a); for (size_t i = 0; i < dense; i++) { - count += a->present[i] == SCR_ARR_VALUE; - undefined_count += a->present[i] == SCR_ARR_UNDEFINED; + count += *scr_arr_state_in(states, i) == SCR_ARR_VALUE; + undefined_count += *scr_arr_state_in(states, i) == SCR_ARR_UNDEFINED; } for (size_t i = 0; i < SCR_AX(a)->sparse_len; i++) { count += SCR_AX(a)->sparse[i].state == SCR_ARR_VALUE; @@ -2565,7 +2682,7 @@ ScrArr *scr_arr_sort_primitive(ScrArr *a, bool copy) { if (count && !values) scr_arr_oom(); size_t used = 0; for (size_t i = 0; i < dense; i++) - if (a->present[i] == SCR_ARR_VALUE) values[used++] = scr_arr_dense_get(a, i); + if (*scr_arr_state_in(states, i) == SCR_ARR_VALUE) values[used++] = scr_arr_dense_get(a, i); for (size_t i = 0; i < SCR_AX(a)->sparse_len; i++) if (SCR_AX(a)->sparse[i].state == SCR_ARR_VALUE) values[used++] = SCR_AX(a)->sparse[i].slot; ScrNumberSortKey *numbers = NULL; @@ -2591,8 +2708,8 @@ ScrArr *scr_arr_sort_primitive(ScrArr *a, bool copy) { if (!copy && scr_arr_is_dense(a)) { if (count) scr_arr_put_slots(a, 0, count, values); scr_arr_cut(a, count); - memset(a->present, SCR_ARR_VALUE, count); - memset(a->present + count, SCR_ARR_UNDEFINED, undefined_count); + memset(scr_arr_state_run(a, 0, count), SCR_ARR_VALUE, count); + memset(scr_arr_state_run(a, count, undefined_count), SCR_ARR_UNDEFINED, undefined_count); scr_arr_clear_dense(a, count + undefined_count, a->len - count - undefined_count); if (undefined_count) memset(scr_arr_dense_at(a, count), 0, undefined_count * scr_arr_slot_width(a)); @@ -4211,9 +4328,9 @@ void scr_arr_sort_values(ScrArr *a, double count, ScrClosure *f, uint32_t arity, static bool scr_arr_dense_values(const ScrArr *a) { if (!scr_arr_is_dense(a)) return false; if (a->filled == a->len) return true; - const uint8_t *present = a->present; + const uint8_t *states = scr_arr_states_end(a); bool all = true; - for (size_t i = 0; i < a->len; i++) all &= present[i] == SCR_ARR_VALUE; + for (size_t i = 0; i < a->len; i++) all &= *scr_arr_state_in(states, i) == SCR_ARR_VALUE; return all; } @@ -4285,16 +4402,16 @@ bool scr_arr_sort_dense(ScrArr *a, ScrClosure *f, uint32_t arity, bool owned, in } else if (a->len >= n && a->cap >= n) { if (!retained) { memcpy(a->data, src, n * sizeof(*src)); - memset(a->present, SCR_ARR_VALUE, n); + memset(scr_arr_state_run(a, 0, n), SCR_ARR_VALUE, n); } else { /* Publish each new edge before releasing the value it replaces: a * release can collect cycles. The buffer still holds the slots not * yet written. */ for (size_t i = 0; i < n; i++) { - bool held = a->present[i] == SCR_ARR_VALUE; + bool held = *scr_arr_state_ptr(a, i) == SCR_ARR_VALUE; uint64_t old = held ? a->data[i] : 0; a->data[i] = src[i]; - a->present[i] = SCR_ARR_VALUE; + *scr_arr_state_ptr(a, i) = SCR_ARR_VALUE; if (held) scr_elem_release(a, old); } } diff --git a/packages/runtime/src/scr_array.h b/packages/runtime/src/scr_array.h index 5289f8c13..909f9ae8c 100644 --- a/packages/runtime/src/scr_array.h +++ b/packages/runtime/src/scr_array.h @@ -64,8 +64,10 @@ typedef struct ScrArrOps { typedef struct ScrArr { size_t rc; /* SIZE_MAX = immortal (unused for arrays; kept per convention) */ - size_t len; - size_t cap; /* dense capacity; sparse entries cover indices >= cap */ + /* Length and dense capacity are at most SCR_ARR_MAX_LENGTH (2^32 - 1); + * sparse entries cover indices >= cap. */ + uint32_t len; + uint32_t cap; /* One ScrElemKind. */ uint8_t elem; /* 1 exactly when ops->trace is non-NULL: the element type carries a cycle @@ -88,10 +90,10 @@ typedef struct ScrArr { /* SCR_ELEM_REF only; NULL for every other element kind. */ const ScrArrOps *ops; /* Dense values: `cap` 8-byte slots, or `cap` bytes for SCR_ELEM_BOOL - * (scr_arr_slot_width). Sparse entries and properties keep 8-byte slots - * for every kind; a bool slot is 0 or 1 wherever it lives. */ + * (scr_arr_slot_width), followed in the same allocation by one state byte + * per slot in reverse slot order (scr_arr_states_end). Sparse entries and properties keep 8-byte + * slots for every kind; a bool slot is 0 or 1 wherever it lives. */ uint64_t *data; - uint8_t *present; /* one state per dense slot; heap arrays share data allocation */ /* Rarely used storage (sparse indices, noncanonical numeric properties, * match-result metadata) lives out of line: NULL until first needed, so * the common dense array stays small. */ @@ -114,20 +116,58 @@ typedef struct ScrArrExt { } ScrArrExt; /* The LLVM backend mirrors ScrArr as %ScrArr: its inline retain/release read * `cyc` (field 4, i8) to decide mark-live / candidate buffering exactly like - * scr_arr_retain and scr_arr_release; its inline element access uses filled - * (field 6, i32), data (field 8) and present (field 9). */ -_Static_assert(offsetof(ScrArr, cyc) == 3 * sizeof(size_t) + 1, + * scr_arr_retain and scr_arr_release; its inline element access uses len and + * cap (fields 1 and 2, i32), filled (field 6, i32) and data (field 8), and + * finds the state of slot i at data + cap * (slot width + 1) - 1 - i. */ +_Static_assert(offsetof(ScrArr, len) == sizeof(size_t) && sizeof(((ScrArr *)0)->len) == 4, + "emitted length reads load ScrArr.len as %ScrArr field 1, an i32"); +_Static_assert(offsetof(ScrArr, cap) == sizeof(size_t) + 4 && sizeof(((ScrArr *)0)->cap) == 4, + "emitted capacity reads load ScrArr.cap as %ScrArr field 2, an i32"); +_Static_assert(offsetof(ScrArr, cyc) == sizeof(size_t) + 9, "inline RC fast paths read ScrArr.cyc as %ScrArr field 4"); -_Static_assert(offsetof(ScrArr, filled) == 3 * sizeof(size_t) + 4, +_Static_assert(offsetof(ScrArr, filled) == sizeof(size_t) + 12, "inline element access reads ScrArr.filled as %ScrArr field 6"); -_Static_assert(offsetof(ScrArr, data) == 3 * sizeof(size_t) + 8 + sizeof(void *), +_Static_assert(offsetof(ScrArr, data) == sizeof(size_t) + 16 + sizeof(void *), "inline element reads use ScrArr.data as %ScrArr field 8"); +_Static_assert(sizeof(ScrArr) == sizeof(size_t) + 16 + 3 * sizeof(void *), + "ScrArr is 48 bytes on 64-bit targets"); /* Bytes per dense value slot. */ static inline size_t scr_arr_slot_width(const ScrArr *a) { return a->elem == SCR_ELEM_BOOL ? 1 : sizeof(uint64_t); } +/* The dense states, one byte per slot, share the values' allocation (inline + * slots included). They follow the `cap` value slots in reverse slot order: + * the state of slot i is the byte i + 1 before scr_arr_states_end(a). Removing + * slots from the front (head room: `data` advances, `cap` drops) leaves every + * remaining state at its address. The address is computed on integers, so a + * NULL `data` stays NULL. */ +static inline uint8_t *scr_arr_states_end(const ScrArr *a) { + return (uint8_t *)((uintptr_t)a->data + (uintptr_t)a->cap * (scr_arr_slot_width(a) + 1)); +} + +/* scr_arr_states_end for an array whose slots are known to be `width` + * bytes wide: a constant multiple, with no element-kind test. */ +static inline uint8_t *scr_arr_states_end_w(const ScrArr *a, size_t width) { + return (uint8_t *)((uintptr_t)a->data + (uintptr_t)a->cap * (width + 1)); +} + +/* The state of dense slot `index` (< cap), given the end of the states. */ +static inline uint8_t *scr_arr_state_in(const uint8_t *end, size_t index) { + return (uint8_t *)end - 1 - index; +} + +static inline uint8_t *scr_arr_state_ptr(const ScrArr *a, size_t index) { + return scr_arr_state_in(scr_arr_states_end(a), index); +} + +/* The states of dense slots [from, from + count) are contiguous, in reverse + * order; this is their first byte (the state of slot from + count - 1). */ +static inline uint8_t *scr_arr_state_run(const ScrArr *a, size_t from, size_t count) { + return scr_arr_states_end(a) - from - count; +} + /* The dense value at `index` (< cap) as an 8-byte slot, and its store. */ static inline uint64_t scr_arr_dense_get(const ScrArr *a, size_t index) { if (a->elem == SCR_ELEM_BOOL) return ((const uint8_t *)a->data)[index]; @@ -145,18 +185,18 @@ static inline void scr_arr_cut(ScrArr *a, size_t index) { if (a->filled > index) a->filled = (uint32_t)index; } -/* The first index in [f, end) whose state is not a present value (or end). - * Compares eight states per step: bulk copies (slice, concat, spread) end - * with this scan over the whole copied range. */ -static inline size_t scr_arr_value_run(const uint8_t *present, size_t f, size_t end) { +/* The first index in [f, end) whose state is not a present value (or end), + * given the end of the states. Compares eight states per step: bulk copies + * (slice, concat, spread) end with this scan over the whole copied range. */ +static inline size_t scr_arr_value_run(const uint8_t *states_end, size_t f, size_t end) { const uint64_t values = 0x0101010101010101ull * SCR_ARR_VALUE; while (end - f >= 8) { uint64_t word; - memcpy(&word, present + f, sizeof word); + memcpy(&word, states_end - f - 8, sizeof word); if (word != values) break; f += 8; } - while (f < end && present[f] == SCR_ARR_VALUE) f++; + while (f < end && *scr_arr_state_in(states_end, f) == SCR_ARR_VALUE) f++; return f; } @@ -165,7 +205,7 @@ static inline size_t scr_arr_value_run(const uint8_t *present, size_t f, size_t static inline void scr_arr_refill(ScrArr *a) { size_t end = a->len < a->cap ? a->len : a->cap; size_t f = a->filled; - if (f < end) f = scr_arr_value_run(a->present, f, end); + if (f < end) f = scr_arr_value_run(scr_arr_states_end(a), f, end); a->filled = (uint32_t)f; } diff --git a/packages/runtime/src/scr_bytes.c b/packages/runtime/src/scr_bytes.c index 64fc36484..999e75fb0 100644 --- a/packages/runtime/src/scr_bytes.c +++ b/packages/runtime/src/scr_bytes.c @@ -3116,7 +3116,7 @@ static void scr_bytes_fill_from_arr(uint8_t *out, ScrBytesElem elem, const ScrAr /* Sparse tails and explicit undefined retain ordinary iteration * semantics. Numeric array slots own no conversion callbacks. */ if (index < arr->cap) { - if (arr->present[index] == SCR_ARR_VALUE) + if (*scr_arr_state_ptr(arr, index) == SCR_ARR_VALUE) memcpy(&values[i], &arr->data[index], sizeof(double)); else values[i] = NAN; } else values[i] = scr_arr_get_number(arr, (double)index); diff --git a/packages/runtime/src/scr_console.c b/packages/runtime/src/scr_console.c index 1e031a2d0..0afb8981a 100644 --- a/packages/runtime/src/scr_console.c +++ b/packages/runtime/src/scr_console.c @@ -148,7 +148,7 @@ static void scr_enable_utf8_console_output(void) { } #endif -void scr_runtime_abi_v22(void) {} +void scr_runtime_abi_v23(void) {} void scr_init(void) { #ifdef SCR_WORKERS diff --git a/packages/runtime/src/scr_core.h b/packages/runtime/src/scr_core.h index 6d251e8ce..cdf1061d9 100644 --- a/packages/runtime/src/scr_core.h +++ b/packages/runtime/src/scr_core.h @@ -81,7 +81,7 @@ void scr_init(void); /* Program objects emitted by the bundled LLVM helper reference this symbol. * Its versioned spelling makes a mismatched manual runtime link fail before * the program can start. */ -void scr_runtime_abi_v22(void); +void scr_runtime_abi_v23(void); void scr_runtime_workers_v8(void); /* ── the trap funnel (scr_console.c; scr_library.c under -DSCR_LIB) ────── diff --git a/packages/runtime/src/scr_cycle.c b/packages/runtime/src/scr_cycle.c index fd5d64c81..d92699573 100644 --- a/packages/runtime/src/scr_cycle.c +++ b/packages/runtime/src/scr_cycle.c @@ -1190,8 +1190,10 @@ void scr_cyc_on_release(void *obj) { size_t end = arr->len < arr->cap ? arr->len : arr->cap; \ size_t filled = arr->filled < end ? arr->filled : end; \ for (size_t i = 0; i < filled; i++) visit((void *)(uintptr_t)arr->data[i], ctx); \ + const uint8_t *states = scr_arr_states_end(arr); \ for (size_t i = filled; i < end; i++) \ - if (arr->present[i] == SCR_ARR_VALUE) visit((void *)(uintptr_t)arr->data[i], ctx); \ + if (*scr_arr_state_in(states, i) == SCR_ARR_VALUE) \ + visit((void *)(uintptr_t)arr->data[i], ctx); \ } /* Visit every child of `obj` with VISIT, through ARRAY_WALK for a dense diff --git a/packages/runtime/src/scr_itable.c b/packages/runtime/src/scr_itable.c index 4842bf79c..7b5202b81 100644 --- a/packages/runtime/src/scr_itable.c +++ b/packages/runtime/src/scr_itable.c @@ -472,7 +472,7 @@ static ScrArrState scr_itable_element(ScrITableElements *w, size_t i, uint64_t * } if (i < a->cap) { *slot = a->data[i]; - return (ScrArrState)a->present[i]; + return (ScrArrState)*scr_arr_state_ptr(a, i); } if (a->ext) { while (w->sparse < a->ext->sparse_len && a->ext->sparse[w->sparse].index < i) w->sparse++; diff --git a/packages/runtime/src/scr_itable.test.c b/packages/runtime/src/scr_itable.test.c index 5e5f22bc9..f6d98654a 100644 --- a/packages/runtime/src/scr_itable.test.c +++ b/packages/runtime/src/scr_itable.test.c @@ -54,8 +54,10 @@ static ScrArr *make_f64s(const double *values, size_t n) { a->len = a->cap = n; a->filled = (uint32_t)n; a->elem = SCR_ELEM_F64; - a->data = calloc(n ? n : 1, sizeof(uint64_t)); + /* Values, then one state byte per slot (scr_arr_states_end). */ + a->data = calloc(n ? n : 1, sizeof(uint64_t) + 1); for (size_t i = 0; i < n; i++) memcpy(&a->data[i], &values[i], sizeof(double)); + memset(scr_arr_state_run(a, 0, n), SCR_ARR_VALUE, n); return a; } @@ -65,8 +67,9 @@ static ScrArr *make_strs(ScrStr *const *values, size_t n) { a->filled = (uint32_t)n; a->elem = SCR_ELEM_STR; free(a->data); - a->data = calloc(n, sizeof(uint64_t)); + a->data = calloc(n ? n : 1, sizeof(uint64_t) + 1); for (size_t i = 0; i < n; i++) a->data[i] = (uint64_t)(uintptr_t)values[i]; + memset(scr_arr_state_run(a, 0, n), SCR_ARR_VALUE, n); return a; } @@ -157,10 +160,7 @@ int main(void) { ScrArr *shorter = make_f64s((double[]){1, 0.0}, 2); ScrArr *holed = make_f64s((double[]){1, 0.0, 0.0}, 3); holed->filled = 1; - holed->present = calloc(3, 1); - holed->present[0] = SCR_ARR_VALUE; - holed->present[1] = SCR_ARR_VALUE; - holed->present[2] = SCR_ARR_HOLE; + *scr_arr_state_ptr(holed, 2) = SCR_ARR_HOLE; ScrArr *empty_list = make_f64s(NULL, 0); const ScrITableEntry *l1 = scr_itable_insert_arr(lists, one, 1); assert(scr_itable_find_arr(lists, same) == l1 && scr_itable_insert_arr(lists, same, 2) == l1); @@ -207,7 +207,6 @@ int main(void) { ScrArr *arrays[] = {one, same, shorter, holed, empty_list, big, longest, huge, p1, p2, p3, heavy}; for (unsigned i = 0; i < sizeof arrays / sizeof *arrays; i++) { free(arrays[i]->data); - free(arrays[i]->present); free(arrays[i]); } free(ab); diff --git a/packages/runtime/src/scr_lib.c b/packages/runtime/src/scr_lib.c index 014115ef0..23f575d93 100644 --- a/packages/runtime/src/scr_lib.c +++ b/packages/runtime/src/scr_lib.c @@ -11745,10 +11745,11 @@ static bool scr_set_to_arr_sparse(const ScrMap *s, ScrArr *out) { ScrArr *scr_set_to_arr_f64(const ScrMap *s) { ScrArr *out = scr_arr_new(SCR_ELEM_F64, s->nlive); if (scr_set_to_arr_sparse(s, out)) return out; + uint8_t *states = scr_arr_states_end(out); for (size_t i = 0; i < s->nentries; i++) { if (!s->entries[i].hash) continue; out->data[out->len] = s->entries[i].key; - out->present[out->len++] = SCR_ARR_VALUE; + *scr_arr_state_in(states, out->len++) = SCR_ARR_VALUE; } return out; } @@ -11756,10 +11757,11 @@ ScrArr *scr_set_to_arr_f64(const ScrMap *s) { ScrArr *scr_set_to_arr_bool(const ScrMap *s) { ScrArr *out = scr_arr_new(SCR_ELEM_BOOL, s->nlive); if (scr_set_to_arr_sparse(s, out)) return out; + uint8_t *states = scr_arr_states_end(out); for (size_t i = 0; i < s->nentries; i++) { if (!s->entries[i].hash) continue; scr_arr_dense_put(out, out->len, s->entries[i].key != 0); - out->present[out->len++] = SCR_ARR_VALUE; + *scr_arr_state_in(states, out->len++) = SCR_ARR_VALUE; } return out; } @@ -11767,11 +11769,12 @@ ScrArr *scr_set_to_arr_bool(const ScrMap *s) { ScrArr *scr_set_to_arr_str(const ScrMap *s) { ScrArr *out = scr_arr_new(SCR_ELEM_STR, s->nlive); if (scr_set_to_arr_sparse(s, out)) return out; + uint8_t *states = scr_arr_states_end(out); for (size_t i = 0; i < s->nentries; i++) { if (!s->entries[i].hash) continue; scr_str_retain((ScrStr *)(uintptr_t)s->entries[i].key); out->data[out->len] = s->entries[i].key; - out->present[out->len++] = SCR_ARR_VALUE; + *scr_arr_state_in(states, out->len++) = SCR_ARR_VALUE; } return out; } @@ -11779,11 +11782,12 @@ ScrArr *scr_set_to_arr_str(const ScrMap *s) { ScrArr *scr_set_to_arr_ref(const ScrMap *s, const ScrArrOps *ops) { ScrArr *out = scr_arr_new_ops(ops, s->nlive); if (scr_set_to_arr_sparse(s, out)) return out; + uint8_t *states = scr_arr_states_end(out); for (size_t i = 0; i < s->nentries; i++) { if (!s->entries[i].hash) continue; ops->retain((void *)(uintptr_t)s->entries[i].key); out->data[out->len] = s->entries[i].key; - out->present[out->len++] = SCR_ARR_VALUE; + *scr_arr_state_in(states, out->len++) = SCR_ARR_VALUE; } return out; } diff --git a/packages/runtime/src/scr_map.c b/packages/runtime/src/scr_map.c index 28922296e..647f559dc 100644 --- a/packages/runtime/src/scr_map.c +++ b/packages/runtime/src/scr_map.c @@ -1504,7 +1504,7 @@ void scr_set_add_all(ScrMap *set, ScrArr *values) { /* Packed values stay owned by the seed array throughout construction. * Avoid taking a temporary owner only to release it after insertion. * Sparse/missing positions retain the existing accessor semantics. */ - if (i < values->cap && values->present[i] == SCR_ARR_VALUE) { + if (i < values->cap && *scr_arr_state_ptr(values, i) == SCR_ARR_VALUE) { uint64_t key = scr_arr_dense_get(values, i); if (set->key_kind == SCR_MAP_KEY_F64) key = scr_map_f64_bits(scr_map_slot_to_f64(key)); scr_map_set(set, scr_map_hash_key(set, key), key, 0); diff --git a/packages/runtime/src/scr_publish.c b/packages/runtime/src/scr_publish.c index 9dedc6844..e95f8767d 100644 --- a/packages/runtime/src/scr_publish.c +++ b/packages/runtime/src/scr_publish.c @@ -196,8 +196,9 @@ void scr_pub_arr(void *v, ScrPub *ctx, ScrPubFn elem) { ScrPubFn fn = a->elem == SCR_ELEM_STR ? scr_pub_str_fn : a->elem == SCR_ELEM_REF ? elem : NULL; if (!fn) return; size_t dense = a->len < a->cap ? a->len : a->cap; + const uint8_t *states = scr_arr_states_end(a); for (size_t i = 0; i < dense; i++) - if (!a->present || a->present[i]) scr_pub_slot(ctx, a->data[i], fn); + if (!states || *scr_arr_state_in(states, i)) scr_pub_slot(ctx, a->data[i], fn); if (a->ext) for (size_t i = 0; i < a->ext->sparse_len; i++) if (a->ext->sparse[i].state) scr_pub_slot(ctx, a->ext->sparse[i].slot, fn); diff --git a/packages/runtime/src/scr_regex.c b/packages/runtime/src/scr_regex.c index fa104058f..e4867ddf3 100644 --- a/packages/runtime/src/scr_regex.c +++ b/packages/runtime/src/scr_regex.c @@ -691,7 +691,8 @@ static ScrDyn *scr_regex_groups_of(const ScrArr *a, uint8_t *bc) { const char *name = groupnames; for (int k = 1; k < capture_count; k++) { size_t length = strlen(name); - bool matched = (size_t)k < a->len && (size_t)k < a->cap && a->present[k] == SCR_ARR_VALUE; + bool matched = (size_t)k < a->len && (size_t)k < a->cap && + *scr_arr_state_ptr(a, (size_t)k) == SCR_ARR_VALUE; /* ES2025 duplicates sit in distinct alternatives: at most one * participates, and it wins over an earlier undefined. */ if (length && (matched || !scr_dyn_obj_get(groups, name, length))) @@ -803,7 +804,7 @@ static uint64_t scr_regex_mask_bits(double mask) { * present undefined (SCR_RE_NO_SPAN when lazy), a lazy capture of an ASCII * subject is a span word, and every other capture a fresh string. */ static inline void scr_regex_fill_slot(const ScrRegexInput *input, uint64_t *data, - uint8_t *present, int k, uint64_t mask, uint64_t lazy, + uint8_t *states, int k, uint64_t mask, uint64_t lazy, bool spans) { const uint8_t *cs = input->capture[2 * k], *ce = input->capture[2 * k + 1]; bool deferred = k < 64 && (lazy >> k & 1); @@ -820,7 +821,7 @@ static inline void scr_regex_fill_slot(const ScrRegexInput *input, uint64_t *dat state = SCR_ARR_VALUE; } data[k] = word; - present[k] = state; + *scr_arr_state_in(states, (size_t)k) = state; } /* re.exec(s): a fresh string[] of [whole match, ...captures], or NULL @@ -851,29 +852,29 @@ static ScrArr *scr_regex_exec_core(ScrStr *s, ScrRegex *re, uint64_t mask, bool * they stay in registers across the slot stores. */ size_t n = (size_t)count + spans; uint64_t *data = out->data; - uint8_t *present = out->present; + uint8_t *states = scr_arr_states_end(out); if (count <= 64) { /* Captures neither selected nor lazy are present undefined with a * zero payload; only the selected and lazy ones need a decision. */ memset(data, 0, (size_t)count * sizeof *data); - memset(present, SCR_ARR_UNDEFINED, (size_t)count); + memset(states - count, SCR_ARR_UNDEFINED, (size_t)count); uint64_t visit = mask | lazy; if (count < 64) visit &= (UINT64_C(1) << count) - 1; while (visit) { int k = __builtin_ctzll(visit); visit &= visit - 1; - scr_regex_fill_slot(&input, data, present, k, mask, lazy, spans); + scr_regex_fill_slot(&input, data, states, k, mask, lazy, spans); } } else { for (int k = 0; k < count; k++) - scr_regex_fill_slot(&input, data, present, k, mask, lazy, spans); + scr_regex_fill_slot(&input, data, states, k, mask, lazy, spans); } if (spans) { data[count] = (uint64_t)(uintptr_t)scr_str_retain(s); - present[count] = SCR_ARR_VALUE; + *scr_arr_state_in(states, count) = SCR_ARR_VALUE; } size_t filled = 0; - while (filled < n && present[filled] == SCR_ARR_VALUE) filled++; + while (filled < n && *scr_arr_state_in(states, filled) == SCR_ARR_VALUE) filled++; out->filled = (uint32_t)filled; out->len = n; if (metadata) { @@ -904,14 +905,14 @@ static bool scr_regex_group_bytes(const ScrArr *m, double k, const char **p, siz /* Array lengths stay below 2^53, so signed conversions are exact. */ if (!(k >= 0 && k < (double)(int64_t)m->len)) return false; size_t i = (size_t)(int64_t)k; - if (m->present[i] == SCR_ARR_VALUE) { + if (*scr_arr_state_ptr(m, i) == SCR_ARR_VALUE) { const ScrStr *str = (const ScrStr *)(uintptr_t)m->data[i]; *p = str->data; *n = str->len; return true; } uint64_t span = m->data[i]; - if (m->present[i] != SCR_ARR_UNDEFINED || span == SCR_RE_NO_SPAN) return false; + if (*scr_arr_state_ptr(m, i) != SCR_ARR_UNDEFINED || span == SCR_RE_NO_SPAN) return false; const ScrStr *subject = (const ScrStr *)(uintptr_t)m->data[m->len - 1]; *p = subject->data + (span >> 32); *n = (size_t)((span & UINT32_MAX) - (span >> 32)); @@ -1045,21 +1046,22 @@ static ScrArr *scr_regex_match_all_core(ScrStr *s, ScrRegex *re, ScrArr *indices * directly, then the hole-free prefix is the run of captures from 0. */ ScrArr *row = scr_arr_new_inline(SCR_ELEM_STR, (size_t)capture_count); uint64_t *data = row->data; - uint8_t *present = row->present; + uint8_t *states = scr_arr_states_end_w(row, 8); for (int k = 0; k < capture_count; k++) { const uint8_t *cs = capture[2 * k], *ce = capture[2 * k + 1]; if (cs == NULL || ce == NULL) { data[k] = 0; - present[k] = SCR_ARR_UNDEFINED; + *scr_arr_state_in(states, (size_t)k) = SCR_ARR_UNDEFINED; } else { ScrStr *text = scr_regex_span(&input, scr_capture_index(&input, cs), scr_capture_index(&input, ce)); data[k] = (uint64_t)(uintptr_t)text; - present[k] = SCR_ARR_VALUE; + *scr_arr_state_in(states, (size_t)k) = SCR_ARR_VALUE; } } size_t filled = 0; - while (filled < (size_t)capture_count && present[filled] == SCR_ARR_VALUE) filled++; + while (filled < (size_t)capture_count && *scr_arr_state_in(states, filled) == SCR_ARR_VALUE) + filled++; row->filled = (uint32_t)filled; row->len = (size_t)capture_count; if (source) scr_regex_defer_metadata(row, source, start); diff --git a/packages/runtime/src/scr_region.c b/packages/runtime/src/scr_region.c index cc094ef0b..75d4bb74b 100644 --- a/packages/runtime/src/scr_region.c +++ b/packages/runtime/src/scr_region.c @@ -805,8 +805,9 @@ static void scr_rgn_walk_arr(ScrRgnWalk *w, ScrArr *a) { ScrRgnSlotKind k = a->elem == SCR_ELEM_ARR ? (ScrRgnSlotKind){SCR_RGN_W_ARR, NULL} : scr_rgn_slot_kind(a->ops ? a->ops->release : NULL); size_t dense = a->len < a->cap ? a->len : a->cap; + const uint8_t *states = scr_arr_states_end(a); for (size_t i = 0; i < dense; i++) - if (!a->present || a->present[i] == SCR_ARR_VALUE) scr_rgn_slot_as(w, scr_rgn_slot_ptr(a->data[i]), k); + if (!states || *scr_arr_state_in(states, i) == SCR_ARR_VALUE) scr_rgn_slot_as(w, scr_rgn_slot_ptr(a->data[i]), k); if (!a->ext) return; for (size_t i = 0; i < a->ext->sparse_len; i++) if (a->ext->sparse[i].state == SCR_ARR_VALUE) scr_rgn_slot_as(w, scr_rgn_slot_ptr(a->ext->sparse[i].slot), k); diff --git a/packages/runtime/test/test_array.c b/packages/runtime/test/test_array.c index 1f86be80e..a3ad4735b 100644 --- a/packages/runtime/test/test_array.c +++ b/packages/runtime/test/test_array.c @@ -1283,7 +1283,7 @@ static void test_exact_construction(void) { static bool filled_ok(const ScrArr *a) { if (a->filled > a->len || a->filled > a->cap) return false; for (size_t i = 0; i < a->filled; i++) - if (a->present[i] != SCR_ARR_VALUE) return false; + if (*scr_arr_state_ptr(a, i) != SCR_ARR_VALUE) return false; return true; } @@ -1358,7 +1358,7 @@ static void test_filled_prefix(void) { scr_arr_release(scr_arr_sort_primitive(b, false)); check(filled_ok(b), "sorting moves holes past the bound"); for (size_t i = 0; i < b->len; i++) { - if (i < b->cap && b->present[i] != SCR_ARR_VALUE) break; + if (i < b->cap && *scr_arr_state_ptr(b, i) != SCR_ARR_VALUE) break; check(i < b->filled, "sorted values form the bound"); } while (scr_arr_len(b) > 0) { @@ -1378,11 +1378,11 @@ static void test_filled_prefix(void) { } /* Boolean arrays store one byte per dense element, followed by the states in - * the same allocation. Growth moves the states past the larger value area, + * the same allocation. Growth moves the states to the end of the larger area, * densifying absorbs sparse entries into bytes, and every moving and copying * operation keeps values, holes and present undefined in place. */ static bool bool_layout_ok(const ScrArr *a) { - return a->cap == 0 || a->present == (const uint8_t *)a->data + a->cap; + return a->cap == 0 || scr_arr_states_end(a) == (const uint8_t *)a->data + 2 * (size_t)a->cap; } static bool bool_states(const ScrArr *a, const char *want) { @@ -1495,7 +1495,8 @@ static void test_inline_slots(void) { check(!inline_slots(plain), "plain constructions keep separate storage"); scr_arr_release(plain); ScrArr *a = scr_arr_new_small(SCR_ELEM_F64); - check(inline_slots(a) && a->cap >= 4 && a->len == 0 && a->present == (uint8_t *)(a->data + a->cap), + check(inline_slots(a) && a->cap >= 4 && a->len == 0 && + scr_arr_states_end(a) == (uint8_t *)(a->data + a->cap) + a->cap, "small construction places slots and states in the array block"); size_t inline_cap = a->cap; for (size_t i = 0; i < inline_cap; i++) scr_arr_push_f64(a, (double)i); @@ -1503,12 +1504,13 @@ static void test_inline_slots(void) { scr_arr_push_f64(a, 100); check(!inline_slots(a) && a->cap > inline_cap && scr_arr_get_f64(a, 0) == 0 && scr_arr_get_f64(a, (double)inline_cap - 1) == (double)inline_cap - 1 && - scr_arr_get_f64(a, (double)inline_cap) == 100 && a->present == (uint8_t *)(a->data + a->cap), + scr_arr_get_f64(a, (double)inline_cap) == 100 && + scr_arr_states_end(a) == (uint8_t *)(a->data + a->cap) + a->cap, "growth moves inline slots to their own allocation"); scr_arr_release(a); ScrArr *flags = scr_arr_new_small(SCR_ELEM_BOOL); - check(inline_slots(flags) && flags->present == (uint8_t *)flags->data + flags->cap, + check(inline_slots(flags) && scr_arr_states_end(flags) == (uint8_t *)flags->data + 2 * (size_t)flags->cap, "boolean inline slots are one byte each"); scr_arr_push_bool(flags, true); scr_arr_set_bool(flags, 3, true); @@ -1887,6 +1889,10 @@ static void test_dense_slots(void) { /* shift and splice(0, n) advance dense storage into head room instead of * moving the tail; unshift reuses that room, growth gives it back, and a * drained array starts over at the front of its allocation. */ +/* SCR_ARR_HEAD_ROOM (scr_array.c): the array has vacated slots in front of + * `data`, and the slot just before `data` holds their count. */ +static bool head_room(const ScrArr *a) { return (a->spare & 2u) != 0; } + static void test_head_room(void) { #ifdef SCR_RC_AUDIT long strings0 = scr_str_live_count(); @@ -1899,7 +1905,8 @@ static void test_head_room(void) { check_f64(scr_arr_shift_f64(q), 0, "shift answers the first element"); check(q->data == block + 1 && q->cap == cap - 1 && q->len == 63, "shift advances the storage instead of moving the tail"); - check(q->present == (uint8_t *)(block + cap) + 1, "states keep following the whole value area"); + check(scr_arr_states_end(q) == (uint8_t *)(block + cap) + cap - 1 && head_room(q) && q->data[-1] == 1, + "the remaining states stay at the end of the allocation"); check(q->filled == 63, "shift keeps the hole-free prefix"); for (int i = 1; i < 40; i++) scr_arr_shift_f64(q); check(q->data == block + 40 && q->len == 24 && scr_arr_get_f64(q, 0) == 40 && @@ -1974,7 +1981,7 @@ static void test_head_room(void) { bool ordered = rev->len == 1000; for (int i = 0; i < 1000 && ordered; i++) ordered = scr_arr_get_f64(rev, i) == 999 - i; check(ordered, "unshift run keeps reverse order"); - size_t room = (size_t)(rev->present - (uint8_t *)(rev->data + rev->cap)); + size_t room = head_room(rev) ? (size_t)rev->data[-1] : 0; check(room >= 4 && rev->cap >= rev->len, "an unshift run reserves head room"); uint64_t *rblock = rev->data; scr_arr_unshift_f64(rev, -1); @@ -1986,7 +1993,7 @@ static void test_head_room(void) { check(rev->data == rblock - 4 && scr_arr_get_f64(rev, 3) == -1, "unshift many order"); /* Storage swaps (typed views) first drop the head room. */ scr_arr_outline(rev); - check(rev->present == (uint8_t *)(rev->data + rev->cap) && scr_arr_get_f64(rev, 0) == 7 && + check(!head_room(rev) && scr_arr_get_f64(rev, 0) == 7 && scr_arr_get_f64(rev, 1003) == 0, "outline moves the elements to the front of their allocation"); scr_arr_release(rev); @@ -1996,7 +2003,7 @@ static void test_head_room(void) { for (int i = 0; i < 40; i++) scr_arr_push_f64(once, i); size_t once_cap = once->cap; scr_arr_unshift_f64(once, -1); - check(once->cap == once_cap && once->present == (uint8_t *)(once->data + once->cap), + check(once->cap == once_cap && !head_room(once), "a one-off unshift keeps the allocation"); scr_arr_release(once); diff --git a/packages/runtime/test/test_regex.c b/packages/runtime/test/test_regex.c index ae98d627b..6084c5102 100644 --- a/packages/runtime/test/test_regex.c +++ b/packages/runtime/test/test_regex.c @@ -331,7 +331,7 @@ static void test_exec_groups(void) { ScrArr *full = scr_regex_exec(s, &re_opt_group); ScrArr *lean = scr_regex_exec_groups(s, &re_opt_group, 2.0); check(full && lean && full->len == lean->len && lean->len == 2, "exec_groups keeps the capture count"); - check(lean->present[0] == SCR_ARR_UNDEFINED && lean->present[1] == SCR_ARR_UNDEFINED, + check(*scr_arr_state_ptr(lean, 0) == SCR_ARR_UNDEFINED && *scr_arr_state_ptr(lean, 1) == SCR_ARR_UNDEFINED, "exec_groups: unselected and nonparticipating captures are undefined"); check(lean->ext == NULL || lean->ext->metadata == NULL, "exec_groups: no index/input properties"); scr_arr_release(full); @@ -430,7 +430,7 @@ static void test_exec_spans(void) { * subjects, materialized on wide ones. */ ScrStr *ascii = S("xa12"); ScrArr *m = scr_regex_exec_spans(ascii, &re_pair, 6.0, 6.0); - check(m && m->len == 4 && m->present[1] == SCR_ARR_UNDEFINED, + check(m && m->len == 4 && *scr_arr_state_ptr(m, 1) == SCR_ARR_UNDEFINED, "exec_spans: ASCII subjects keep lazy captures as spans"); ScrStr *a = S("a"); ScrStr *twelve = S("12"); @@ -448,7 +448,7 @@ static void test_exec_spans(void) { scr_arr_release(m); ScrStr *wide = S("\xC3\xA9" "a7"); m = scr_regex_exec_spans(wide, &re_pair, 6.0, 6.0); - check(m && m->len == 3 && m->present[2] == SCR_ARR_VALUE && scr_regex_group_number(m, 2) == 7, + check(m && m->len == 3 && *scr_arr_state_ptr(m, 2) == SCR_ARR_VALUE && scr_regex_group_number(m, 2) == 7, "exec_spans: wide subjects materialize lazy captures"); scr_arr_release(m); m = scr_regex_exec_spans(ascii, &re_pair, 38.0, 2.0); diff --git a/tests/corpus/array-length-limits.ts b/tests/corpus/array-length-limits.ts new file mode 100644 index 000000000..d98d8e311 --- /dev/null +++ b/tests/corpus/array-length-limits.ts @@ -0,0 +1,94 @@ +// Array lengths and indices at the 2^32 - 1 limit, sparse writes past the +// dense capacity, holes and boolean arrays: the length and capacity fit in +// 32 bits, and the dense states follow the value slots. + +function report(label: string, run: () => unknown): void { + try { + console.log(label, run()); + } catch (e) { + console.log(label, e instanceof Error ? `${e.name}: ${e.message}` : e); + } +} + +const big: string[] = []; +big.length = 4294967295; +console.log("max length", big.length); +report("length 2^32", () => { + const a: number[] = []; + a.length = 4294967296; + return a.length; +}); +report("length -1", () => { + const a: number[] = []; + a.length = -1; + return a.length; +}); + +const edge: string[] = []; +edge[4294967294] = "last"; +console.log("max index", edge.length, edge[4294967294]); + +// Sparse writes beyond the dense capacity, then densifying appends. +const sparse: number[] = [1, 2, 3]; +sparse[1000] = 4; +sparse[50] = 5; +console.log("sparse", sparse.length, sparse[1000], sparse[50], 999 in sparse, sparse.indexOf(5)); +for (let i = 3; i < 60; i++) if (!(i in sparse)) sparse[i] = i; +console.log("filled", sparse.slice(0, 8).join(","), sparse[49], sparse[50], sparse.lastIndexOf(4)); +sparse.length = 10; +console.log("truncated", sparse.length, sparse.join(","), 50 in sparse); + +// Holes and their states across growth and splice. +const holes: (string | undefined)[] = ["a", , "c", undefined] as (string | undefined)[]; +console.log("holes", holes.length, 1 in holes, 3 in holes, holes.indexOf(undefined)); +holes.push("e", "f", "g", "h", "i"); +holes.splice(1, 2, "B"); +console.log("spliced", holes.length, JSON.stringify(holes), 1 in holes); +const copy = holes.slice(0, 3); +console.log("slice", copy.length, copy.join("|")); + +// Boolean arrays keep one byte per element; their states follow them. +const flags: boolean[] = []; +for (let i = 0; i < 40; i++) flags.push(i % 3 === 0); +flags[45] = true; +console.log("bools", flags.length, flags.filter((f) => f).length, 42 in flags, flags.indexOf(true, 40)); +flags.length = 5; +flags.unshift(false); +console.log("bools after", flags.join(","), flags.length); + +// Many one-element arrays (the common case the 48-byte header targets). +const ones: number[][] = []; +for (let i = 0; i < 1000; i++) ones.push([i]); +let total = 0; +for (const one of ones) total += one[0]! + one.length; +console.log("ones", ones.length, total, ones[999]!.concat(ones[0]!).join(",")); + +// Tagged templates hand out an immortal strings array. +function tag(strings: TemplateStringsArray, ...values: number[]): string { + return `${strings.length}:${strings.join("_")}:${values.join(",")}`; +} +console.log("tagged", tag`a${1}b${2}c`, tag`only`); + +// Small arrays filled past their inline slots, of numbers and of class +// instances that can form cycles. +class Node { + readonly id: number; + readonly next: Node[] = []; + constructor(id: number) { + this.id = id; + } +} +const grown: number[][] = []; +for (let n = 0; n < 10; n++) { + const a: number[] = []; + for (let i = 0; i < n; i++) a.push(i * n); + grown.push(a); +} +console.log("grown", grown.map((a) => a.join("-")).join(" ")); +const root = new Node(0); +for (let i = 1; i <= 9; i++) { + const child = new Node(i); + child.next.push(root); + root.next.push(child); + console.log("node", i, root.next.length, root.next.map((c) => c.id).join(""), child.next[0]!.id); +} diff --git a/tests/harness/corpus-costs.json b/tests/harness/corpus-costs.json index b18e2ab28..943904301 100644 --- a/tests/harness/corpus-costs.json +++ b/tests/harness/corpus-costs.json @@ -2393,6 +2393,7 @@ "array-from-checked.js": 3228, "array-from-value.mjs": 2658, "array-isarray-value/main.js": 2180, + "array-length-limits.ts": 4410, "array-numeric-own-keys.ts": 2770, "array-optional-number-write-index.ts": 2584, "array-primitive-ordering.ts": 3806, @@ -5490,6 +5491,7 @@ "array-from-checked.js": 3097, "array-from-value.mjs": 3428, "array-isarray-value/main.js": 3143, + "array-length-limits.ts": 3930, "array-numeric-own-keys.ts": 3632, "array-optional-number-write-index.ts": 3380, "array-primitive-ordering.ts": 3925, diff --git a/tests/harness/self-hosting-native-toolchain.test.ts b/tests/harness/self-hosting-native-toolchain.test.ts index fa71b5a08..9f6b82be2 100644 --- a/tests/harness/self-hosting-native-toolchain.test.ts +++ b/tests/harness/self-hosting-native-toolchain.test.ts @@ -98,7 +98,7 @@ for (const backend of ["llvm"] as const) { object_format: target.objectFormat, minimum_os: target.minimumOs, }, - runtime_abi: { version: 22, marker: "scr_runtime_abi_v22" }, + runtime_abi: { version: 23, marker: "scr_runtime_abi_v23" }, compiler: { command: "fixture", identity: "fixture", target: target.llvmTriple }, macros: { executable: [], excluded: [], sanitizer: "external-toolchain-required" }, flavors: {