diff --git a/packages/compiler/src/backend/llvm/array-search.test.ts b/packages/compiler/src/backend/llvm/array-search.test.ts new file mode 100644 index 000000000..8b644f708 --- /dev/null +++ b/packages/compiler/src/backend/llvm/array-search.test.ts @@ -0,0 +1,85 @@ +import { expect, test } from "vitest"; +import { + F64, + STRING, + VOID, + arrayOf, + type IrExpr, + type IrModule, + type IrType, +} from "../../ir/ir.js"; +import { emitLlvmModule } from "./emitter.js"; + +const loc = { file: "array-search.ts", start: 0, end: 0 }; +const ref = (localId: string, type: IrType): IrExpr => ({ kind: "varRef", localId, type, loc }); + +/** A module whose function `find` returns `a.method(v)` for an array `a` + * of `elem`. */ +function module(elem: IrType, method: "indexOf" | "includes"): IrModule { + const arr = arrayOf(elem); + const search: IrExpr = { + kind: "arrIntrinsic", + method, + receiver: ref("a", arr), + args: [ref("v", elem)], + type: method === "includes" ? { kind: "bool" } : F64, + loc, + }; + return { + irVersion: 16, + sourceFile: loc.file, + entry: "__main", + functions: [ + { name: "__main", params: [], returnType: VOID, locals: [], body: [], loc }, + { + name: "find", + params: [ + { name: "a", localId: "a", type: arr }, + { name: "v", localId: "v", type: elem }, + ], + returnType: search.type, + locals: [ + { id: "a", name: "a", type: arr, mutable: false }, + { id: "v", name: "v", type: elem, mutable: false }, + ], + body: [{ kind: "return", value: search, loc }], + loc, + }, + ], + }; +} + +function helper(llvm: string, name: string): string { + return new RegExp(`^define internal double @${name}\\([^]*?^}`, "m").exec(llvm)![0]; +} + +test.each([32, 64] as const)("string arrays search inline (%i-bit)", (bits) => { + const sz = bits === 64 ? "i64" : "i32"; + const llvm = emitLlvmModule(module(STRING, "indexOf"), { pointerBits: bits }); + expect(llvm).toMatch(/call double @sc_arr_index_of_str\(ptr .*, ptr .*\)/); + const fn = helper(llvm, "sc_arr_index_of_str"); + // Dense arrays only (filled == len); everything else is the runtime search. + expect(fn).toMatch(new RegExp(`icmp eq ${sz} %f, %len`)); + expect(fn).toContain("call double @scr_arr_index_of_ref(ptr %a, ptr %v)"); + // Pointer identity, the length, then up to 16 bytes as words; longer + // strings compare in the runtime. + expect(fn).toContain("icmp eq ptr %s, %v"); + expect(fn).toMatch(/load i64, ptr %sap, align 1/); + expect(fn).toContain("call zeroext i1 @scr_str_eq(ptr %s, ptr %v)"); +}); + +test.each([32, 64] as const)("reference arrays compare slots, four per step (%i-bit)", (bits) => { + const llvm = emitLlvmModule(module(arrayOf(F64), "includes"), { pointerBits: bits }); + expect(llvm).toMatch(/call double @sc_arr_index_of_ptr\(ptr .*, ptr .*\)/); + expect(llvm).toMatch(/fcmp oge double %.*, 0\.0/); + const fn = helper(llvm, "sc_arr_index_of_ptr"); + expect(fn.match(/icmp eq i64 %qs\d, %needle/g)).toHaveLength(4); + expect(fn).toContain("icmp eq i64 %s, %needle"); + expect(llvm).not.toContain("sc_arr_index_of_str"); +}); + +test("number arrays keep the runtime search", () => { + const llvm = emitLlvmModule(module(F64, "indexOf"), { pointerBits: 64 }); + expect(llvm).toContain("call double @scr_arr_index_of_f64("); + expect(llvm).not.toContain("sc_arr_index_of_"); +}); diff --git a/packages/compiler/src/backend/llvm/array-search.ts b/packages/compiler/src/backend/llvm/array-search.ts new file mode 100644 index 000000000..92a4870ec --- /dev/null +++ b/packages/compiler/src/backend/llvm/array-search.ts @@ -0,0 +1,194 @@ +/* Inline indexOf / includes over arrays of references. + * + * The runtime search (scr_arr_index_of_ref) checks each slot's state byte and + * dispatches on the element kind for every element. When the array's + * hole-free prefix covers its whole length (filled == len, so there are no + * holes and no sparse entries), the internal helpers below scan the dense + * slots directly, and LLVM inlines them at the call site: + * + * - sc_arr_index_of_ptr, for element types whose strict equality and + * SameValueZero are both pointer identity (class instances, records, + * arrays, maps, sets, typed arrays): 64-bit compares, four slots per + * branch. + * - sc_arr_index_of_str, for strings: pointer identity, then the length, + * then the bytes. Strings of up to 16 bytes compare as two words; a short + * string's word is the eight bytes ending at its last byte, which start + * in the string header when it is shorter than eight bytes and so stay + * inside the object, shifted down past the header bytes. Longer strings + * compare with scr_str_eq. + * + * Any other array takes the runtime search. Both helpers answer the index, + * or -1; includes compares the answer with 0, since for these element types + * SameValueZero and strict equality agree. Every target scriptc builds for + * is little-endian, which the short-string words assume. + */ +import type { IrType } from "../../ir/ir.js"; +import { FN_ATTRS } from "./shapes.js"; + +export type ArraySearchKind = "ptr" | "str"; + +/** The inline search for arrays of `elem`, or null for the runtime search. */ +export function arraySearchKind(elem: IrType): ArraySearchKind | null { + switch (elem.kind) { + case "object": + case "record": + case "array": + case "map": + case "set": + case "bytes": + return "ptr"; + case "string": + return "str"; + default: + return null; + } +} + +export function arraySearchHelperName(kind: ArraySearchKind): string { + return kind === "ptr" ? "sc_arr_index_of_ptr" : "sc_arr_index_of_str"; +} + +/** The definitions of the helpers in `kinds`; they declare their slow paths + * through `declare`. */ +export function emitArraySearchHelpers( + kinds: ReadonlySet, + sz: string, + declare: (line: string) => void, +): string[] { + const defs: string[] = []; + for (const kind of [...kinds].sort()) { + declare("declare double @scr_arr_index_of_ref(ptr, ptr)"); + const widen = sz === "i64" ? " %f = zext i32 %f32 to i64" : " %f = add i32 %f32, 0"; + defs.push( + `define internal double @${arraySearchHelperName(kind)}(ptr %a, ptr %v) ${FN_ATTRS} {`, + `entry:`, + ` %lenp = getelementptr inbounds %ScrArr, ptr %a, i32 0, i32 1`, + ` %len = load ${sz}, ptr %lenp`, + ` %fp = getelementptr inbounds %ScrArr, ptr %a, i32 0, i32 6`, + ` %f32 = load i32, ptr %fp`, + widen, + ` %dense = icmp eq ${sz} %f, %len`, + ` br i1 %dense, label %scan, label %slow`, + `scan:`, + ` %dp = getelementptr inbounds %ScrArr, ptr %a, i32 0, i32 8`, + ` %data = load ptr, ptr %dp`, + ); + if (kind === "ptr") { + defs.push( + ` %needle = ptrtoint ptr %v to i64`, + // Four slots per step with one branch, then one at a time. + ` br label %quad`, + `quad:`, + ` %q = phi ${sz} [ 0, %scan ], [ %qnext, %quadmiss ]`, + ` %qend = add ${sz} %q, 4`, + ` %qfits = icmp ule ${sz} %qend, %len`, + ` br i1 %qfits, label %quadbody, label %loop`, + `quadbody:`, + ...[0, 1, 2, 3].flatMap((k) => [ + ` %qi${k} = add ${sz} %q, ${k}`, + ` %qp${k} = getelementptr inbounds i64, ptr %data, ${sz} %qi${k}`, + ` %qs${k} = load i64, ptr %qp${k}`, + ` %qh${k} = icmp eq i64 %qs${k}, %needle`, + ]), + ` %qh01 = or i1 %qh0, %qh1`, + ` %qh23 = or i1 %qh2, %qh3`, + ` %qhit = or i1 %qh01, %qh23`, + ` br i1 %qhit, label %loop, label %quadmiss`, + `quadmiss:`, + ` %qnext = add ${sz} %q, 4`, + ` br label %quad`, + // The single-slot loop finishes the tail, or finds the hit inside a + // step that had one. + `loop:`, + ` %i = phi ${sz} [ %q, %quad ], [ %q, %quadbody ], [ %next, %cont ]`, + ` %done = icmp eq ${sz} %i, %len`, + ` br i1 %done, label %miss, label %body`, + `body:`, + ` %sp = getelementptr inbounds i64, ptr %data, ${sz} %i`, + ` %s = load i64, ptr %sp`, + ` %hit = icmp eq i64 %s, %needle`, + ` br i1 %hit, label %found, label %cont`, + ); + } else { + declare("declare zeroext i1 @scr_str_eq(ptr, ptr)"); + defs.push( + ` %vlp = getelementptr inbounds %ScrStr, ptr %v, i32 0, i32 1`, + ` %vlen = load ${sz}, ptr %vlp`, + ` %vdata = getelementptr inbounds %ScrStr, ptr %v, i32 1`, + ` br label %loop`, + `loop:`, + ` %i = phi ${sz} [ 0, %scan ], [ %next, %cont ]`, + ` %done = icmp eq ${sz} %i, %len`, + ` br i1 %done, label %miss, label %body`, + `body:`, + ` %sp = getelementptr inbounds i64, ptr %data, ${sz} %i`, + ` %sw = load i64, ptr %sp`, + ` %s = inttoptr i64 %sw to ptr`, + ` %same = icmp eq ptr %s, %v`, + ` br i1 %same, label %found, label %length`, + `length:`, + ` %slp = getelementptr inbounds %ScrStr, ptr %s, i32 0, i32 1`, + ` %slen = load ${sz}, ptr %slp`, + ` %samelen = icmp eq ${sz} %slen, %vlen`, + ` br i1 %samelen, label %bytes, label %cont`, + // Same length: dispatch on it. + `bytes:`, + ` %empty = icmp eq ${sz} %vlen, 0`, + ` br i1 %empty, label %found, label %nonempty`, + `nonempty:`, + ` %short = icmp ult ${sz} %vlen, 8`, + ` br i1 %short, label %tiny, label %wide`, + `tiny:`, + ` %sdata = getelementptr inbounds %ScrStr, ptr %s, i32 1`, + ` %back = sub ${sz} %vlen, 8`, + ` %sap = getelementptr inbounds i8, ptr %sdata, ${sz} %back`, + ` %vap = getelementptr inbounds i8, ptr %vdata, ${sz} %back`, + ` %sa = load i64, ptr %sap, align 1`, + ` %va = load i64, ptr %vap, align 1`, + ` %diff = xor i64 %sa, %va`, + sz === "i64" ? ` %vlen64 = add i64 %vlen, 0` : ` %vlen64 = zext i32 %vlen to i64`, + ` %bits = shl i64 %vlen64, 3`, + ` %drop = sub i64 64, %bits`, + ` %kept = lshr i64 %diff, %drop`, + ` %tinyeq = icmp eq i64 %kept, 0`, + ` br i1 %tinyeq, label %found, label %cont`, + `wide:`, + ` %fits = icmp ule ${sz} %vlen, 16`, + ` br i1 %fits, label %two, label %long`, + `two:`, + ` %sdata2 = getelementptr inbounds %ScrStr, ptr %s, i32 1`, + ` %s0 = load i64, ptr %sdata2, align 1`, + ` %v0 = load i64, ptr %vdata, align 1`, + ` %tail = sub ${sz} %vlen, 8`, + ` %s1p = getelementptr inbounds i8, ptr %sdata2, ${sz} %tail`, + ` %v1p = getelementptr inbounds i8, ptr %vdata, ${sz} %tail`, + ` %s1 = load i64, ptr %s1p, align 1`, + ` %v1 = load i64, ptr %v1p, align 1`, + ` %d0 = xor i64 %s0, %v0`, + ` %d1 = xor i64 %s1, %v1`, + ` %d = or i64 %d0, %d1`, + ` %twoeq = icmp eq i64 %d, 0`, + ` br i1 %twoeq, label %found, label %cont`, + `long:`, + ` %longeq = call zeroext i1 @scr_str_eq(ptr %s, ptr %v)`, + ` br i1 %longeq, label %found, label %cont`, + ); + } + defs.push( + `cont:`, + ` %next = add ${sz} %i, 1`, + ` br label %loop`, + `found:`, + ` %at = uitofp ${sz} %i to double`, + ` ret double %at`, + `miss:`, + ` ret double -1.0`, + `slow:`, + ` %r = call double @scr_arr_index_of_ref(ptr %a, ptr %v)`, + ` ret double %r`, + `}`, + ``, + ); + } + return defs; +} diff --git a/packages/compiler/src/backend/llvm/emitter.ts b/packages/compiler/src/backend/llvm/emitter.ts index 4760ff3b3..b5b8b5596 100644 --- a/packages/compiler/src/backend/llvm/emitter.ts +++ b/packages/compiler/src/backend/llvm/emitter.ts @@ -1,3 +1,4 @@ +import { type ArraySearchKind, emitArraySearchHelpers } from "./array-search.js"; import { emitLlvmLayouts } from "./layouts.js"; import { compileRegexMatcher, type RegexMatcherCode } from "./regex-matchers.js"; import { CYC_BUF, CYC_HDR, FIRST_EMITTED_CYC_KIND, inlineAllocSupported } from "./alloc.js"; @@ -770,6 +771,8 @@ export class LlEmitter { * typeKey → thunk symbol. */ readonly resolveThunks = new Map(); readonly resolveThunkDefs: string[] = []; + /** Inline array searches used by the module (array-search.ts). */ + readonly arraySearchKinds = new Set(); /** Typed-ref recipe constants by type key (lazy receivers). */ readonly typedRefRecipes = new Map(); /** ReadableStream.from adapters keep typed arrays by reference and box @@ -1977,6 +1980,7 @@ export class LlEmitter { // write/abort declarations). const helpers = [ ...this.helperDefs(), + ...emitArraySearchHelpers(this.arraySearchKinds, this.sizeType, (line) => this.declare(line)), ...emitClassMembershipHelper(this.classMeta, this.sizeType), ...(this.publishWalkersCache?.defs() ?? []), ...this.publishedClassNamesDefs(), diff --git a/packages/compiler/src/backend/llvm/expr-containers.ts b/packages/compiler/src/backend/llvm/expr-containers.ts index 17c2a4285..1d5a72753 100644 --- a/packages/compiler/src/backend/llvm/expr-containers.ts +++ b/packages/compiler/src/backend/llvm/expr-containers.ts @@ -1,4 +1,5 @@ /* Focused LLVM expression emission extracted from emitter.ts. */ +import { arraySearchHelperName, arraySearchKind } from "./array-search.js"; import { InternalCompilerError } from "../../errors.js"; import { isStableReceiverOperand, undefinedArmTag } from "../../ir/analysis.js"; import { type IrExpr, type IrType, isRefCounted, typeEquals, typeKey } from "../../ir/ir.js"; @@ -719,16 +720,38 @@ export function emitArrIntrinsic( case "indexOf": { // The needle is BORROWED (released with this statement's frame); // the ref variant dispatches on the array's element kind (strings - // by content, everything else by pointer). Strict equality. + // by content, everything else by pointer). Strict equality. Arrays of + // objects and strings scan inline (array-search.ts). const v = host.emitExpr(e.args[0]!); + const search = acc === "ref" ? arraySearchKind(elem) : null; + if (search !== null) { + host.arraySearchKinds.add(search); + const t = B.tmp(); + B.line( + `${t} = call double @${arraySearchHelperName(search)}(ptr ${r.name}, ptr ${v.name})`, + ); + return { name: t, type: e.type }; + } host.declare(`declare double @scr_arr_index_of_${acc}(ptr, ${accArg})`); const t = B.tmp(); B.line(`${t} = call double @scr_arr_index_of_${acc}(ptr ${r.name}, ${accTy} ${v.name})`); return { name: t, type: e.type }; } case "includes": { - // Borrowed needle, SameValueZero (NaN matches NaN). + // Borrowed needle, SameValueZero (NaN matches NaN). For the inline + // searches SameValueZero is strict equality. const v = host.emitExpr(e.args[0]!); + const search = acc === "ref" ? arraySearchKind(elem) : null; + if (search !== null) { + host.arraySearchKinds.add(search); + const at = B.tmp(), + t = B.tmp(); + B.line( + `${at} = call double @${arraySearchHelperName(search)}(ptr ${r.name}, ptr ${v.name})`, + ); + B.line(`${t} = fcmp oge double ${at}, 0.0`); + return { name: t, type: e.type }; + } host.declare(`declare zeroext i1 @scr_arr_includes_${acc}(ptr, ${accArg})`); const t = B.tmp(); B.line(`${t} = call zeroext i1 @scr_arr_includes_${acc}(ptr ${r.name}, ${accTy} ${v.name})`); diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5551-array-search-references.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5551-array-search-references.ts.json new file mode 100644 index 000000000..6c2612470 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5551-array-search-references.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/5551-array-search-references.ts" + ], + "diags": [] +} diff --git a/tests/corpus/5551-array-search-references.ts b/tests/corpus/5551-array-search-references.ts new file mode 100644 index 000000000..83270c8dc --- /dev/null +++ b/tests/corpus/5551-array-search-references.ts @@ -0,0 +1,80 @@ +// indexOf and includes on arrays of objects, records, arrays, maps, sets, +// typed arrays and strings: dense arrays, arrays with holes and sparse +// entries (the runtime search), duplicates (the first index), and strings of +// every length class built separately, differing at each position. +class Item { + id: number; + constructor(id: number) { + this.id = id; + } +} +const items: Item[] = []; +for (let i = 0; i < 37; i++) items.push(new Item(i)); +const stranger = new Item(99); +const objectResults: number[] = []; +for (let i = 0; i < 37; i++) objectResults.push(items.indexOf(items[i]!)); +objectResults.push(items.indexOf(stranger), items.includes(stranger) ? 1 : 0, items.includes(items[36]!) ? 1 : 0); +items.push(items[5]!); +objectResults.push(items.indexOf(items[5]!), items.lastIndexOf(items[5]!)); +console.log(objectResults.join(",")); + +// Holes and sparse entries. +const holey: Item[] = items.slice(0, 10); +holey.length = 20; +holey[15] = stranger; +console.log(holey.indexOf(stranger), holey.includes(stranger), holey.indexOf(items[3]!), holey.indexOf(items[12]!)); +const sparse: Item[] = [items[0]!, items[1]!]; +sparse[5000] = stranger; +console.log(sparse.indexOf(stranger), sparse.includes(items[1]!), sparse.indexOf(items[2]!)); +const emptied: Item[] = [items[0]!]; +emptied.pop(); +console.log(emptied.indexOf(items[0]!), emptied.includes(items[0]!), emptied.length); + +// Mutation between searches. +const moving = items.slice(0, 8); +moving.shift(); +moving.splice(2, 1); +moving.unshift(stranger); +console.log(moving.indexOf(stranger), moving.indexOf(items[0]!), moving.indexOf(items[3]!), moving.indexOf(items[7]!)); + +// Other reference element kinds. +type Point = { x: number; y: number }; +const points: Point[] = [{ x: 1, y: 2 }, { x: 1, y: 2 }]; +const p0 = points[0]!; +console.log(points.indexOf(p0), points.indexOf(points[1]!), points.indexOf({ x: 1, y: 2 }), points.includes(p0)); +const rows: number[][] = [[1], [2], [1]]; +console.log(rows.indexOf(rows[2]!), rows.includes([1]), rows.indexOf(rows[1]!)); +const maps: Map[] = [new Map(), new Map()]; +const sets: Set[] = [new Set([1]), new Set([1])]; +const buffers: Uint8Array[] = [new Uint8Array(2), new Uint8Array(2)]; +console.log(maps.indexOf(maps[1]!), sets.indexOf(sets[1]!), sets.includes(new Set([1])), buffers.indexOf(buffers[1]!)); + +// Strings: lengths 0 to 40, each built at run time, differing at each position. +function build(n: number, at: number, ch: string): string { + let s = ""; + for (let i = 0; i < n; i++) s += i === at ? ch : String.fromCharCode(97 + (i % 26)); + return s; +} +let out = ""; +for (let n = 0; n <= 40; n++) { + const words: string[] = []; + for (let at = 0; at < n; at++) words.push(build(n, at, "Z")); + words.push(build(n, -1, "")); + for (let at = 0; at < n; at++) { + const probe = build(n, at, "Z"); + out += words.indexOf(probe) === at ? "." : "x"; + } + out += words.indexOf(build(n, -1, "")) === n ? "." : "x"; + out += words.includes(build(n + 1, -1, "")) ? "x" : "."; + out += words.indexOf(build(n, n - 1, "Y")) === -1 || n === 0 ? "." : "x"; + out += "|"; +} +console.log(out); +const names = ["", "a", "é", "日本", "naïve", "naive", "😀", "abcdefgh", "abcdefghi", "abcdefghabcdefgh", "abcdefghabcdefghX"]; +const lookups = names.map((name) => (name + "!").slice(0, name.length)); +console.log(lookups.map((name) => names.indexOf(name)).join(","), names.includes("na" + "ïve"), names.indexOf("日"), names.indexOf("abcdefghabcdefgY")); +const dup = ["k", "k", "q", "k"]; +console.log(dup.indexOf("k"), dup.indexOf("q"), dup.includes("z"), dup.lastIndexOf("k")); +const holeyNames: string[] = ["a", "b"]; +holeyNames[4] = "e"; +console.log(holeyNames.indexOf("e"), holeyNames.includes("b"), holeyNames.indexOf("c"));