diff --git a/packages/compiler/src/backend/llvm/emit-module.ts b/packages/compiler/src/backend/llvm/emit-module.ts index 8145429f1..b1118f8bd 100644 --- a/packages/compiler/src/backend/llvm/emit-module.ts +++ b/packages/compiler/src/backend/llvm/emit-module.ts @@ -1212,6 +1212,126 @@ export function emitModuleParts(emitter: LlEmitter): string[] { return out; } +/** The out-of-line hashed probe of a number-key map (scr_map_find_f64): + * `keyLines` define the normalized key word %kb from the parameter, and + * `runtimeKey` is the double passed to the runtime lookup. A map with at + * most SCR_MAP_LINEAR_LIMIT entries keeps no bucket table: the absent-key + * filter is tested before its entries are scanned. A bucket table is + * probed without the filter: the filter keeps two of its 64 bits per key, + * so past a few dozen keys it rejects few absent keys, and the probe's + * first bucket read answers those. Compact tables tag each 24-bit entry + * index (+1) with the hash's top byte; wider tables take the runtime + * lookup. */ +function hashedNumberProbe( + name: string, + param: string, + sz: string, + keyLines: string[], + runtimeKey: string, +): string[] { + return [ + `define internal ptr @${name}(ptr %m, ${param}) noinline ${FN_ATTRS} {`, + `entry:`, + ...keyLines, + ` %x1s = lshr i64 %kb, 30`, + ` %x1 = xor i64 %kb, %x1s`, + ` %x2 = mul i64 %x1, -4658895280553007687`, + ` %x3s = lshr i64 %x2, 27`, + ` %x3 = xor i64 %x2, %x3s`, + ` %x4 = mul i64 %x3, -7723592293110705685`, + ` %x5s = lshr i64 %x4, 31`, + ` %x5 = xor i64 %x4, %x5s`, + ` %hz = icmp eq i64 %x5, 0`, + ` %h = select i1 %hz, i64 1, i64 %x5`, + ` %nbp = getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 13`, + ...(sz === "i64" + ? [` %nb32 = load i32, ptr %nbp`, ` %nb = zext i32 %nb32 to i64`] + : [` %nb = load i32, ptr %nbp`]), + ` %eps = getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 11`, + ` %ents = load ptr, ptr %eps`, + ` %linear = icmp eq ${sz} %nb, 0`, + ` br i1 %linear, label %filter, label %buckets`, + `filter:`, + ` %f1s = lshr i64 %h, 44`, + ` %f1a = and i64 %f1s, 63`, + ` %f1 = shl i64 1, %f1a`, + ` %f2s = lshr i64 %h, 50`, + ` %f2a = and i64 %f2s, 63`, + ` %f2 = shl i64 1, %f2a`, + ` %fb = or i64 %f1, %f2`, + ` %fp = getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 10`, + ` %filter.bits = load i64, ptr %fp`, + ` %fm = and i64 %filter.bits, %fb`, + ` %fin = icmp eq i64 %fm, %fb`, + ` br i1 %fin, label %scan, label %miss`, + // A number-key entry has no cached hash; a dead entry's key matches no key. + `scan:`, + ` %nep = getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 8`, + ...(sz === "i64" + ? [` %ne32 = load i32, ptr %nep`, ` %ne = zext i32 %ne32 to i64`] + : [` %ne = load i32, ptr %nep`]), + ` br label %scan.loop`, + `scan.loop:`, + ` %si = phi ${sz} [ 0, %scan ], [ %si.next, %scan.next ]`, + ` %sdone = icmp eq ${sz} %si, %ne`, + ` br i1 %sdone, label %miss, label %scan.entry`, + `scan.entry:`, + ` %se = getelementptr inbounds %ScrMapNumEntry, ptr %ents, ${sz} %si`, + ` %sk = load i64, ptr %se`, + ` %skq = icmp eq i64 %sk, %kb`, + ` br i1 %skq, label %scan.found, label %scan.next`, + `scan.found:`, + ` ret ptr %se`, + `scan.next:`, + ` %si.next = add ${sz} %si, 1`, + ` br label %scan.loop`, + `buckets:`, + ` %wide = icmp ugt ${sz} %nb, 16777216`, + ` br i1 %wide, label %runtime, label %probe`, + `probe:`, + ` %bp = getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 14`, + ` %bk = load ptr, ptr %bp`, + ` %mask = sub ${sz} %nb, 1`, + ` %ht = lshr i64 %h, 32`, + ` %ht32 = trunc i64 %ht to i32`, + ` %tag = and i32 %ht32, -16777216`, + sz === "i64" ? ` %hi = add i64 %h, 0` : ` %hi = trunc i64 %h to ${sz}`, + ` %i0 = and ${sz} %hi, %mask`, + ` br label %probe.loop`, + `probe.loop:`, + ` %pi = phi ${sz} [ %i0, %probe ], [ %pi.next, %probe.next ]`, + ` %bep = getelementptr inbounds i32, ptr %bk, ${sz} %pi`, + ` %b = load i32, ptr %bep`, + ` %bempty = icmp eq i32 %b, 0`, + ` br i1 %bempty, label %miss, label %probe.tag`, + `probe.tag:`, + ` %bt = and i32 %b, -16777216`, + ` %btq = icmp eq i32 %bt, %tag`, + ` br i1 %btq, label %probe.entry, label %probe.next`, + `probe.entry:`, + ` %bix = and i32 %b, 16777215`, + ` %bix1 = sub i32 %bix, 1`, + sz === "i32" ? ` %bi = add i32 %bix1, 0` : ` %bi = zext i32 %bix1 to ${sz}`, + ` %pe = getelementptr inbounds %ScrMapNumEntry, ptr %ents, ${sz} %bi`, + ` %pk = load i64, ptr %pe`, + ` %pkq = icmp eq i64 %pk, %kb`, + ` br i1 %pkq, label %probe.found, label %probe.next`, + `probe.found:`, + ` ret ptr %pe`, + `probe.next:`, + ` %pi.inc = add ${sz} %pi, 1`, + ` %pi.next = and ${sz} %pi.inc, %mask`, + ` br label %probe.loop`, + `miss:`, + ` ret ptr null`, + `runtime:`, + ` %r = call ptr @scr_map_entry_f64(ptr %m, double ${runtimeKey}) memory(read) nounwind willreturn`, + ` ret ptr %r`, + `}`, + ``, + ]; +} + /** The shared abort helpers (emitted only when referenced): the OOM * abort of untraced shape allocation and the invalid-union-tag abort — * both print the runtime ABI’s exact message on fd 2 and abort. */ @@ -1351,118 +1471,34 @@ export function helperDefs(emitter: LlEmitter): string[] { ` %dense.any = icmp ne i64 %nd, 0`, ` br i1 %dense.any, label %miss, label %slow`, `slow:`, - ` %hashed = call ptr @sc_map_entry_f64_hashed(ptr %m, double %k)`, + ` %hashed = call ptr @sc_map_entry_f64_hashed_int(ptr %m, i64 %ki)`, ` ret ptr %hashed`, `}`, ``, - // Hashing (scr_map_find_f64): the SameValueZero key word, its - // scr_key_mix hash (zero remapped to one) and the absent-key filter. - `define internal ptr @sc_map_entry_f64_hashed(ptr %m, double %k) noinline ${FN_ATTRS} {`, - `entry:`, - ` %nan = fcmp uno double %k, %k`, - ` %zero = fcmp oeq double %k, 0.0`, - ` %kz = select i1 %zero, double 0.0, double %k`, - ` %kzb = bitcast double %kz to i64`, - ` %kb = select i1 %nan, i64 9221120237041090560, i64 %kzb`, - ` %x1s = lshr i64 %kb, 30`, - ` %x1 = xor i64 %kb, %x1s`, - ` %x2 = mul i64 %x1, -4658895280553007687`, - ` %x3s = lshr i64 %x2, 27`, - ` %x3 = xor i64 %x2, %x3s`, - ` %x4 = mul i64 %x3, -7723592293110705685`, - ` %x5s = lshr i64 %x4, 31`, - ` %x5 = xor i64 %x4, %x5s`, - ` %hz = icmp eq i64 %x5, 0`, - ` %h = select i1 %hz, i64 1, i64 %x5`, - ` %f1s = lshr i64 %h, 44`, - ` %f1a = and i64 %f1s, 63`, - ` %f1 = shl i64 1, %f1a`, - ` %f2s = lshr i64 %h, 50`, - ` %f2a = and i64 %f2s, 63`, - ` %f2 = shl i64 1, %f2a`, - ` %fb = or i64 %f1, %f2`, - ` %fp = getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 10`, - ` %filter = load i64, ptr %fp`, - ` %fm = and i64 %filter, %fb`, - ` %fin = icmp eq i64 %fm, %fb`, - ` br i1 %fin, label %table, label %miss`, - `table:`, - ` %nbp = getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 13`, - ...(sz === "i64" - ? [` %nb32 = load i32, ptr %nbp`, ` %nb = zext i32 %nb32 to i64`] - : [` %nb = load i32, ptr %nbp`]), - ` %eps = getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 11`, - ` %ents = load ptr, ptr %eps`, - ` %linear = icmp eq ${sz} %nb, 0`, - ` br i1 %linear, label %scan, label %buckets`, - // Small maps keep no bucket table: scan the entries' keys. A number-key - // entry has no cached hash; a dead entry's key matches no key. - `scan:`, - ` %nep = getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 8`, - ...(sz === "i64" - ? [` %ne32 = load i32, ptr %nep`, ` %ne = zext i32 %ne32 to i64`] - : [` %ne = load i32, ptr %nep`]), - ` br label %scan.loop`, - `scan.loop:`, - ` %si = phi ${sz} [ 0, %scan ], [ %si.next, %scan.next ]`, - ` %sdone = icmp eq ${sz} %si, %ne`, - ` br i1 %sdone, label %miss, label %scan.entry`, - `scan.entry:`, - ` %se = getelementptr inbounds %ScrMapNumEntry, ptr %ents, ${sz} %si`, - ` %sk = load i64, ptr %se`, - ` %skq = icmp eq i64 %sk, %kb`, - ` br i1 %skq, label %scan.found, label %scan.next`, - `scan.found:`, - ` ret ptr %se`, - `scan.next:`, - ` %si.next = add ${sz} %si, 1`, - ` br label %scan.loop`, - // Compact tables tag each 24-bit entry index (+1) with the hash's - // top byte; wider tables take the runtime lookup. - `buckets:`, - ` %wide = icmp ugt ${sz} %nb, 16777216`, - ` br i1 %wide, label %runtime, label %probe`, - `probe:`, - ` %bp = getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 14`, - ` %bk = load ptr, ptr %bp`, - ` %mask = sub ${sz} %nb, 1`, - ` %ht = lshr i64 %h, 32`, - ` %ht32 = trunc i64 %ht to i32`, - ` %tag = and i32 %ht32, -16777216`, - sz === "i64" ? ` %hi = add i64 %h, 0` : ` %hi = trunc i64 %h to ${sz}`, - ` %i0 = and ${sz} %hi, %mask`, - ` br label %probe.loop`, - `probe.loop:`, - ` %pi = phi ${sz} [ %i0, %probe ], [ %pi.next, %probe.next ]`, - ` %bep = getelementptr inbounds i32, ptr %bk, ${sz} %pi`, - ` %b = load i32, ptr %bep`, - ` %bempty = icmp eq i32 %b, 0`, - ` br i1 %bempty, label %miss, label %probe.tag`, - `probe.tag:`, - ` %bt = and i32 %b, -16777216`, - ` %btq = icmp eq i32 %bt, %tag`, - ` br i1 %btq, label %probe.entry, label %probe.next`, - `probe.entry:`, - ` %bix = and i32 %b, 16777215`, - ` %bix1 = sub i32 %bix, 1`, - sz === "i32" ? ` %bi = add i32 %bix1, 0` : ` %bi = zext i32 %bix1 to ${sz}`, - ` %pe = getelementptr inbounds %ScrMapNumEntry, ptr %ents, ${sz} %bi`, - ` %pk = load i64, ptr %pe`, - ` %pkq = icmp eq i64 %pk, %kb`, - ` br i1 %pkq, label %probe.found, label %probe.next`, - `probe.found:`, - ` ret ptr %pe`, - `probe.next:`, - ` %pi.inc = add ${sz} %pi, 1`, - ` %pi.next = and ${sz} %pi.inc, %mask`, - ` br label %probe.loop`, - `miss:`, - ` ret ptr null`, - `runtime:`, - ` %r = call ptr @scr_map_entry_f64(ptr %m, double %k) memory(read) nounwind willreturn`, - ` ret ptr %r`, - `}`, - ``, + // Hashing (scr_map_find_f64): the SameValueZero key word and its + // scr_key_mix hash. sc_map_entry_f64_int's probe converts the integer + // view to the key word, which is never NaN or -0, and so skips the + // normalization. + ...hashedNumberProbe( + "sc_map_entry_f64_hashed", + "double %k", + sz, + [ + ` %nan = fcmp uno double %k, %k`, + ` %zero = fcmp oeq double %k, 0.0`, + ` %kz = select i1 %zero, double 0.0, double %k`, + ` %kzb = bitcast double %kz to i64`, + ` %kb = select i1 %nan, i64 9221120237041090560, i64 %kzb`, + ], + "%k", + ), + ...hashedNumberProbe( + "sc_map_entry_f64_hashed_int", + "i64 %ki", + sz, + [` %kf = sitofp i64 %ki to double`, ` %kb = bitcast double %kf to i64`], + "%kf", + ), ); } // Map/Set iteration steps read the entries array inline: an index is diff --git a/packages/compiler/src/backend/llvm/map-number-lookup.test.ts b/packages/compiler/src/backend/llvm/map-number-lookup.test.ts index 7457db42d..f7bd87af1 100644 --- a/packages/compiler/src/backend/llvm/map-number-lookup.test.ts +++ b/packages/compiler/src/backend/llvm/map-number-lookup.test.ts @@ -168,8 +168,10 @@ test("hashed number-key lookups probe inline and call the runtime only for wide expect(hashed).toContain("select i1 %nan, i64 9221120237041090560, i64 %kzb"); expect(hashed).toContain("mul i64 %x1, -4658895280553007687"); expect(hashed).toContain("mul i64 %x3, -7723592293110705685"); - // Filter (field 10), small-map scan of the entry count (field 8), and - // compact buckets (fields 13 and 14) tagged with the hash's top byte. + // Filter (field 10) before the small-map scan of the entry count (field + // 8) only, and compact buckets (fields 13 and 14) tagged with the hash's + // top byte. + expect(hashed).toContain("br i1 %linear, label %filter, label %buckets"); expect(hashed).toContain("getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 10"); expect(hashed).toContain(`load i32, ptr %nep`); expect(hashed).toContain(`load i32, ptr %nbp`); @@ -192,11 +194,23 @@ test("a key with an integer view tests the direct index on the integer", () => { integral.module, )![0]; // An unsigned compare also rejects negative keys; the hashed probe takes - // the double. + // the integer. expect(helper).toContain("icmp ult i64 %ki, %nd"); expect(helper).toContain(`getelementptr inbounds i32, ptr %dense, ${sz} %i`); - expect(helper).toContain("call ptr @sc_map_entry_f64_hashed(ptr %m, double %k)"); + expect(helper).toContain("call ptr @sc_map_entry_f64_hashed_int(ptr %m, i64 %ki)"); expect(helper).not.toContain("fptosi"); + // Its key word is the integer's double, which needs no NaN or -0 + // normalization; the runtime lookup gets that double. + const hashed = + /^define internal ptr @sc_map_entry_f64_hashed_int\(ptr %m, i64 %ki\) noinline [^]*?^}/m.exec( + integral.module, + )![0]; + expect(hashed).toContain("sitofp i64 %ki to double"); + expect(hashed).not.toContain("fcmp"); + expect(hashed).toContain("mul i64 %x1, -4658895280553007687"); + expect(hashed).toContain( + "call ptr @scr_map_entry_f64(ptr %m, double %kf) memory(read) nounwind willreturn", + ); // A fractional key and a key without a proof keep the double lookup. for (const key of [{ kind: "numLit", value: 0.5, type: F64, loc } as IrExpr, ref("k", F64)]) { const { work } = emitted(lookupFixture(key), bits); diff --git a/packages/compiler/src/backend/llvm/map-number-lookup.ts b/packages/compiler/src/backend/llvm/map-number-lookup.ts index afc8ec606..17a1563c3 100644 --- a/packages/compiler/src/backend/llvm/map-number-lookup.ts +++ b/packages/compiler/src/backend/llvm/map-number-lookup.ts @@ -4,9 +4,10 @@ * direct index in the runtime (scr_map.c). Reads go through the internal * sc_map_entry_f64 helper (emitter.ts helperDefs), which LLVM inlines: it * resolves directly indexed maps without a call and other maps through - * the internal sc_map_entry_f64_hashed probe, which hashes the key as the - * runtime does and defers to scr_map_entry_f64 only for wide bucket - * tables. Both only read memory. The generated code then reads the entry's + * the internal sc_map_entry_f64_hashed probe (sc_map_entry_f64_hashed_int + * for a key with an integer view), which hashes the key as the runtime + * does and defers to scr_map_entry_f64 only for wide bucket tables. Both + * only read memory. The generated code then reads the entry's * value slot itself and retains a reference value with the value type's own * retain, which is the adapter the collection was constructed with. * String-keyed reads use the same entry contract through the internal diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8331-map-number-hashed-integer-keys.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8331-map-number-hashed-integer-keys.ts.json new file mode 100644 index 000000000..86822b641 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8331-map-number-hashed-integer-keys.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/8331-map-number-hashed-integer-keys.ts" + ], + "diags": [] +} diff --git a/tests/corpus/8331-map-number-hashed-integer-keys.ts b/tests/corpus/8331-map-number-hashed-integer-keys.ts new file mode 100644 index 000000000..8af7293e9 --- /dev/null +++ b/tests/corpus/8331-map-number-hashed-integer-keys.ts @@ -0,0 +1,102 @@ +// Number-keyed Map and Set lookups whose keys are integers the compiler +// tracks as integers (loop counters, literals, integer arithmetic) on +// hashed storage, and lookups of absent keys on small bucket tables. + +// Keys spread past the direct index's range, so the maps hash them. +const spread = new Map(); +for (let i = 0; i < 500; i++) spread.set(i * 104729 - 26000000, i); +let found = 0, + absent = 0, + total = 0; +for (let i = -10; i < 520; i++) { + const v = spread.get(i * 104729 - 26000000); + if (v === undefined) absent++; + else { + found++; + total += v; + } +} +console.log("spread", spread.size, found, absent, total); +console.log("spread literals", spread.get(-26000000), spread.get(-25895271), spread.get(0), spread.has(-1)); + +// Zero stored through -0 and through an integer; read back through both. +const zeros = new Map(); +for (let i = 1; i <= 20; i++) zeros.set(i * 1000003, "k" + i); +zeros.set(-0, "minus zero"); +let z = 0; +console.log("zeros", zeros.size, zeros.get(z), zeros.get(0), zeros.get(-0), zeros.get(z * -1)); +zeros.set(z, "zero"); +console.log("zeros overwrite", zeros.size, zeros.get(-0), [...zeros.keys()].includes(0)); + +// Large and negative integer keys. +const wide = new Map(); +for (let i = 0; i < 64; i++) wide.set(2 ** 40 + i * 3, i); +for (let i = 0; i < 64; i++) wide.set(-(2 ** 33) - i, -i); +let wideUp = 0, + wideDown = 0, + wideMissing = 0; +for (let i = 0; i < 200; i++) { + const a = wide.get(2 ** 40 + i); + const b = wide.get(-(2 ** 33) - i); + if (a === undefined) wideMissing++; + else wideUp += a; + if (b === undefined) wideMissing++; + else wideDown += b; +} +console.log("wide", wide.size, wideUp, wideDown, wideMissing, wide.get(1099511627776), wide.get(-8589934592)); + +// Absent keys on bucket tables just past the linear scan, and as they grow. +for (const size of [9, 12, 16, 33, 64, 100, 257, 400]) { + const m = new Map(); + for (let i = 0; i < size; i++) m.set(i * 7919 + 1000000, i); + let hits = 0, + misses = 0; + for (let i = 0; i < size * 4; i++) { + if (m.has(i * 7919 + 1000000)) hits++; + else misses++; + if (m.get(i * 7919 + 1000001) !== undefined) hits += 1000; + } + for (let i = 0; i < size; i += 2) m.delete(i * 7919 + 1000000); + let after = 0; + for (let i = 0; i < size; i++) if (m.has(i * 7919 + 1000000)) after++; + console.log("size", size, hits, misses, m.size, after, m.get(1000000), m.get(1007919)); +} + +// Small maps that scan their entries, with integer keys. +const tiny = new Map(); +for (let i = 0; i < 8; i++) tiny.set(i * 5000011, "t" + i); +let tinyHits = 0; +for (let i = 0; i < 40; i++) if (tiny.has(i * 5000011)) tinyHits++; +console.log("tiny", tiny.size, tinyHits, tiny.get(0), tiny.get(35000077), tiny.has(5000012)); + +// Sets with integer keys. +const seen = new Set(); +let dupes = 0; +for (let i = 0; i < 3000; i++) { + const k = ((i * 2654435761) % 1000003) - 500000; + if (seen.has(k)) dupes++; + else seen.add(k); +} +let present = 0; +for (let i = -500000; i < 500003; i += 997) if (seen.has(i)) present++; +console.log("set", seen.size, dupes, present, seen.has(-500000), seen.has(500002)); + +// A counting loop like a k-mer histogram: get, then update the found entry. +class Count { + n = 0; +} +const counts = new Map(); +let key = 0; +for (let i = 0; i < 20000; i++) { + key = ((key * 4) % 4194304) + (i % 7 === 0 ? 3 : i % 3); + const c = counts.get(key); + if (c === undefined) { + const fresh = new Count(); + fresh.n = 1; + counts.set(key, fresh); + } else c.n++; +} +let most = 0, + mostKey = -1; +for (const [k, c] of counts) if (c.n > most) [most, mostKey] = [c.n, k]; +console.log("counts", counts.size, most, mostKey); diff --git a/tests/harness/corpus-costs/8331-map-number-hashed-integer-keys.ts.json b/tests/harness/corpus-costs/8331-map-number-hashed-integer-keys.ts.json new file mode 100644 index 000000000..961fe0dad --- /dev/null +++ b/tests/harness/corpus-costs/8331-map-number-hashed-integer-keys.ts.json @@ -0,0 +1 @@ +{ "plain": 3441, "san": 1752 }