Probe hashed number-key maps without normalizing integer keys (#2264)

This commit is contained in:
Chris Tate
2026-10-25 17:03:41 -05:00
committed by GitHub
6 changed files with 276 additions and 116 deletions
+145 -109
View File
@@ -1212,6 +1212,126 @@ export function emitModuleParts(emitter: LlEmitter): string[] {
return out;
}
/** The out-of-line hashed probe of a number-key map (scr_map_find_f64):
* `keyLines` define the normalized key word %kb from the parameter, and
* `runtimeKey` is the double passed to the runtime lookup. A map with at
* most SCR_MAP_LINEAR_LIMIT entries keeps no bucket table: the absent-key
* filter is tested before its entries are scanned. A bucket table is
* probed without the filter: the filter keeps two of its 64 bits per key,
* so past a few dozen keys it rejects few absent keys, and the probe's
* first bucket read answers those. Compact tables tag each 24-bit entry
* index (+1) with the hash's top byte; wider tables take the runtime
* lookup. */
function hashedNumberProbe(
name: string,
param: string,
sz: string,
keyLines: string[],
runtimeKey: string,
): string[] {
return [
`define internal ptr @${name}(ptr %m, ${param}) noinline ${FN_ATTRS} {`,
`entry:`,
...keyLines,
` %x1s = lshr i64 %kb, 30`,
` %x1 = xor i64 %kb, %x1s`,
` %x2 = mul i64 %x1, -4658895280553007687`,
` %x3s = lshr i64 %x2, 27`,
` %x3 = xor i64 %x2, %x3s`,
` %x4 = mul i64 %x3, -7723592293110705685`,
` %x5s = lshr i64 %x4, 31`,
` %x5 = xor i64 %x4, %x5s`,
` %hz = icmp eq i64 %x5, 0`,
` %h = select i1 %hz, i64 1, i64 %x5`,
` %nbp = getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 13`,
...(sz === "i64"
? [` %nb32 = load i32, ptr %nbp`, ` %nb = zext i32 %nb32 to i64`]
: [` %nb = load i32, ptr %nbp`]),
` %eps = getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 11`,
` %ents = load ptr, ptr %eps`,
` %linear = icmp eq ${sz} %nb, 0`,
` br i1 %linear, label %filter, label %buckets`,
`filter:`,
` %f1s = lshr i64 %h, 44`,
` %f1a = and i64 %f1s, 63`,
` %f1 = shl i64 1, %f1a`,
` %f2s = lshr i64 %h, 50`,
` %f2a = and i64 %f2s, 63`,
` %f2 = shl i64 1, %f2a`,
` %fb = or i64 %f1, %f2`,
` %fp = getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 10`,
` %filter.bits = load i64, ptr %fp`,
` %fm = and i64 %filter.bits, %fb`,
` %fin = icmp eq i64 %fm, %fb`,
` br i1 %fin, label %scan, label %miss`,
// A number-key entry has no cached hash; a dead entry's key matches no key.
`scan:`,
` %nep = getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 8`,
...(sz === "i64"
? [` %ne32 = load i32, ptr %nep`, ` %ne = zext i32 %ne32 to i64`]
: [` %ne = load i32, ptr %nep`]),
` br label %scan.loop`,
`scan.loop:`,
` %si = phi ${sz} [ 0, %scan ], [ %si.next, %scan.next ]`,
` %sdone = icmp eq ${sz} %si, %ne`,
` br i1 %sdone, label %miss, label %scan.entry`,
`scan.entry:`,
` %se = getelementptr inbounds %ScrMapNumEntry, ptr %ents, ${sz} %si`,
` %sk = load i64, ptr %se`,
` %skq = icmp eq i64 %sk, %kb`,
` br i1 %skq, label %scan.found, label %scan.next`,
`scan.found:`,
` ret ptr %se`,
`scan.next:`,
` %si.next = add ${sz} %si, 1`,
` br label %scan.loop`,
`buckets:`,
` %wide = icmp ugt ${sz} %nb, 16777216`,
` br i1 %wide, label %runtime, label %probe`,
`probe:`,
` %bp = getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 14`,
` %bk = load ptr, ptr %bp`,
` %mask = sub ${sz} %nb, 1`,
` %ht = lshr i64 %h, 32`,
` %ht32 = trunc i64 %ht to i32`,
` %tag = and i32 %ht32, -16777216`,
sz === "i64" ? ` %hi = add i64 %h, 0` : ` %hi = trunc i64 %h to ${sz}`,
` %i0 = and ${sz} %hi, %mask`,
` br label %probe.loop`,
`probe.loop:`,
` %pi = phi ${sz} [ %i0, %probe ], [ %pi.next, %probe.next ]`,
` %bep = getelementptr inbounds i32, ptr %bk, ${sz} %pi`,
` %b = load i32, ptr %bep`,
` %bempty = icmp eq i32 %b, 0`,
` br i1 %bempty, label %miss, label %probe.tag`,
`probe.tag:`,
` %bt = and i32 %b, -16777216`,
` %btq = icmp eq i32 %bt, %tag`,
` br i1 %btq, label %probe.entry, label %probe.next`,
`probe.entry:`,
` %bix = and i32 %b, 16777215`,
` %bix1 = sub i32 %bix, 1`,
sz === "i32" ? ` %bi = add i32 %bix1, 0` : ` %bi = zext i32 %bix1 to ${sz}`,
` %pe = getelementptr inbounds %ScrMapNumEntry, ptr %ents, ${sz} %bi`,
` %pk = load i64, ptr %pe`,
` %pkq = icmp eq i64 %pk, %kb`,
` br i1 %pkq, label %probe.found, label %probe.next`,
`probe.found:`,
` ret ptr %pe`,
`probe.next:`,
` %pi.inc = add ${sz} %pi, 1`,
` %pi.next = and ${sz} %pi.inc, %mask`,
` br label %probe.loop`,
`miss:`,
` ret ptr null`,
`runtime:`,
` %r = call ptr @scr_map_entry_f64(ptr %m, double ${runtimeKey}) memory(read) nounwind willreturn`,
` ret ptr %r`,
`}`,
``,
];
}
/** The shared abort helpers (emitted only when referenced): the OOM
* abort of untraced shape allocation and the invalid-union-tag abort —
* both print the runtime ABI’s exact message on fd 2 and abort. */
@@ -1351,118 +1471,34 @@ export function helperDefs(emitter: LlEmitter): string[] {
` %dense.any = icmp ne i64 %nd, 0`,
` br i1 %dense.any, label %miss, label %slow`,
`slow:`,
` %hashed = call ptr @sc_map_entry_f64_hashed(ptr %m, double %k)`,
` %hashed = call ptr @sc_map_entry_f64_hashed_int(ptr %m, i64 %ki)`,
` ret ptr %hashed`,
`}`,
``,
// Hashing (scr_map_find_f64): the SameValueZero key word, its
// scr_key_mix hash (zero remapped to one) and the absent-key filter.
`define internal ptr @sc_map_entry_f64_hashed(ptr %m, double %k) noinline ${FN_ATTRS} {`,
`entry:`,
` %nan = fcmp uno double %k, %k`,
` %zero = fcmp oeq double %k, 0.0`,
` %kz = select i1 %zero, double 0.0, double %k`,
` %kzb = bitcast double %kz to i64`,
` %kb = select i1 %nan, i64 9221120237041090560, i64 %kzb`,
` %x1s = lshr i64 %kb, 30`,
` %x1 = xor i64 %kb, %x1s`,
` %x2 = mul i64 %x1, -4658895280553007687`,
` %x3s = lshr i64 %x2, 27`,
` %x3 = xor i64 %x2, %x3s`,
` %x4 = mul i64 %x3, -7723592293110705685`,
` %x5s = lshr i64 %x4, 31`,
` %x5 = xor i64 %x4, %x5s`,
` %hz = icmp eq i64 %x5, 0`,
` %h = select i1 %hz, i64 1, i64 %x5`,
` %f1s = lshr i64 %h, 44`,
` %f1a = and i64 %f1s, 63`,
` %f1 = shl i64 1, %f1a`,
` %f2s = lshr i64 %h, 50`,
` %f2a = and i64 %f2s, 63`,
` %f2 = shl i64 1, %f2a`,
` %fb = or i64 %f1, %f2`,
` %fp = getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 10`,
` %filter = load i64, ptr %fp`,
` %fm = and i64 %filter, %fb`,
` %fin = icmp eq i64 %fm, %fb`,
` br i1 %fin, label %table, label %miss`,
`table:`,
` %nbp = getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 13`,
...(sz === "i64"
? [` %nb32 = load i32, ptr %nbp`, ` %nb = zext i32 %nb32 to i64`]
: [` %nb = load i32, ptr %nbp`]),
` %eps = getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 11`,
` %ents = load ptr, ptr %eps`,
` %linear = icmp eq ${sz} %nb, 0`,
` br i1 %linear, label %scan, label %buckets`,
// Small maps keep no bucket table: scan the entries' keys. A number-key
// entry has no cached hash; a dead entry's key matches no key.
`scan:`,
` %nep = getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 8`,
...(sz === "i64"
? [` %ne32 = load i32, ptr %nep`, ` %ne = zext i32 %ne32 to i64`]
: [` %ne = load i32, ptr %nep`]),
` br label %scan.loop`,
`scan.loop:`,
` %si = phi ${sz} [ 0, %scan ], [ %si.next, %scan.next ]`,
` %sdone = icmp eq ${sz} %si, %ne`,
` br i1 %sdone, label %miss, label %scan.entry`,
`scan.entry:`,
` %se = getelementptr inbounds %ScrMapNumEntry, ptr %ents, ${sz} %si`,
` %sk = load i64, ptr %se`,
` %skq = icmp eq i64 %sk, %kb`,
` br i1 %skq, label %scan.found, label %scan.next`,
`scan.found:`,
` ret ptr %se`,
`scan.next:`,
` %si.next = add ${sz} %si, 1`,
` br label %scan.loop`,
// Compact tables tag each 24-bit entry index (+1) with the hash's
// top byte; wider tables take the runtime lookup.
`buckets:`,
` %wide = icmp ugt ${sz} %nb, 16777216`,
` br i1 %wide, label %runtime, label %probe`,
`probe:`,
` %bp = getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 14`,
` %bk = load ptr, ptr %bp`,
` %mask = sub ${sz} %nb, 1`,
` %ht = lshr i64 %h, 32`,
` %ht32 = trunc i64 %ht to i32`,
` %tag = and i32 %ht32, -16777216`,
sz === "i64" ? ` %hi = add i64 %h, 0` : ` %hi = trunc i64 %h to ${sz}`,
` %i0 = and ${sz} %hi, %mask`,
` br label %probe.loop`,
`probe.loop:`,
` %pi = phi ${sz} [ %i0, %probe ], [ %pi.next, %probe.next ]`,
` %bep = getelementptr inbounds i32, ptr %bk, ${sz} %pi`,
` %b = load i32, ptr %bep`,
` %bempty = icmp eq i32 %b, 0`,
` br i1 %bempty, label %miss, label %probe.tag`,
`probe.tag:`,
` %bt = and i32 %b, -16777216`,
` %btq = icmp eq i32 %bt, %tag`,
` br i1 %btq, label %probe.entry, label %probe.next`,
`probe.entry:`,
` %bix = and i32 %b, 16777215`,
` %bix1 = sub i32 %bix, 1`,
sz === "i32" ? ` %bi = add i32 %bix1, 0` : ` %bi = zext i32 %bix1 to ${sz}`,
` %pe = getelementptr inbounds %ScrMapNumEntry, ptr %ents, ${sz} %bi`,
` %pk = load i64, ptr %pe`,
` %pkq = icmp eq i64 %pk, %kb`,
` br i1 %pkq, label %probe.found, label %probe.next`,
`probe.found:`,
` ret ptr %pe`,
`probe.next:`,
` %pi.inc = add ${sz} %pi, 1`,
` %pi.next = and ${sz} %pi.inc, %mask`,
` br label %probe.loop`,
`miss:`,
` ret ptr null`,
`runtime:`,
` %r = call ptr @scr_map_entry_f64(ptr %m, double %k) memory(read) nounwind willreturn`,
` ret ptr %r`,
`}`,
``,
// Hashing (scr_map_find_f64): the SameValueZero key word and its
// scr_key_mix hash. sc_map_entry_f64_int's probe converts the integer
// view to the key word, which is never NaN or -0, and so skips the
// normalization.
...hashedNumberProbe(
"sc_map_entry_f64_hashed",
"double %k",
sz,
[
` %nan = fcmp uno double %k, %k`,
` %zero = fcmp oeq double %k, 0.0`,
` %kz = select i1 %zero, double 0.0, double %k`,
` %kzb = bitcast double %kz to i64`,
` %kb = select i1 %nan, i64 9221120237041090560, i64 %kzb`,
],
"%k",
),
...hashedNumberProbe(
"sc_map_entry_f64_hashed_int",
"i64 %ki",
sz,
[` %kf = sitofp i64 %ki to double`, ` %kb = bitcast double %kf to i64`],
"%kf",
),
);
}
// Map/Set iteration steps read the entries array inline: an index is
@@ -168,8 +168,10 @@ test("hashed number-key lookups probe inline and call the runtime only for wide
expect(hashed).toContain("select i1 %nan, i64 9221120237041090560, i64 %kzb");
expect(hashed).toContain("mul i64 %x1, -4658895280553007687");
expect(hashed).toContain("mul i64 %x3, -7723592293110705685");
// Filter (field 10), small-map scan of the entry count (field 8), and
// compact buckets (fields 13 and 14) tagged with the hash's top byte.
// Filter (field 10) before the small-map scan of the entry count (field
// 8) only, and compact buckets (fields 13 and 14) tagged with the hash's
// top byte.
expect(hashed).toContain("br i1 %linear, label %filter, label %buckets");
expect(hashed).toContain("getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 10");
expect(hashed).toContain(`load i32, ptr %nep`);
expect(hashed).toContain(`load i32, ptr %nbp`);
@@ -192,11 +194,23 @@ test("a key with an integer view tests the direct index on the integer", () => {
integral.module,
)![0];
// An unsigned compare also rejects negative keys; the hashed probe takes
// the double.
// the integer.
expect(helper).toContain("icmp ult i64 %ki, %nd");
expect(helper).toContain(`getelementptr inbounds i32, ptr %dense, ${sz} %i`);
expect(helper).toContain("call ptr @sc_map_entry_f64_hashed(ptr %m, double %k)");
expect(helper).toContain("call ptr @sc_map_entry_f64_hashed_int(ptr %m, i64 %ki)");
expect(helper).not.toContain("fptosi");
// Its key word is the integer's double, which needs no NaN or -0
// normalization; the runtime lookup gets that double.
const hashed =
/^define internal ptr @sc_map_entry_f64_hashed_int\(ptr %m, i64 %ki\) noinline [^]*?^}/m.exec(
integral.module,
)![0];
expect(hashed).toContain("sitofp i64 %ki to double");
expect(hashed).not.toContain("fcmp");
expect(hashed).toContain("mul i64 %x1, -4658895280553007687");
expect(hashed).toContain(
"call ptr @scr_map_entry_f64(ptr %m, double %kf) memory(read) nounwind willreturn",
);
// A fractional key and a key without a proof keep the double lookup.
for (const key of [{ kind: "numLit", value: 0.5, type: F64, loc } as IrExpr, ref("k", F64)]) {
const { work } = emitted(lookupFixture(key), bits);
@@ -4,9 +4,10 @@
* direct index in the runtime (scr_map.c). Reads go through the internal
* sc_map_entry_f64 helper (emitter.ts helperDefs), which LLVM inlines: it
* resolves directly indexed maps without a call and other maps through
* the internal sc_map_entry_f64_hashed probe, which hashes the key as the
* runtime does and defers to scr_map_entry_f64 only for wide bucket
* tables. Both only read memory. The generated code then reads the entry's
* the internal sc_map_entry_f64_hashed probe (sc_map_entry_f64_hashed_int
* for a key with an integer view), which hashes the key as the runtime
* does and defers to scr_map_entry_f64 only for wide bucket tables. Both
* only read memory. The generated code then reads the entry's
* value slot itself and retains a reference value with the value type's own
* retain, which is the adapter the collection was constructed with.
* String-keyed reads use the same entry contract through the internal
@@ -0,0 +1,6 @@
{
"order": [
"<repo>/tests/corpus/8331-map-number-hashed-integer-keys.ts"
],
"diags": []
}
@@ -0,0 +1,102 @@
// Number-keyed Map and Set lookups whose keys are integers the compiler
// tracks as integers (loop counters, literals, integer arithmetic) on
// hashed storage, and lookups of absent keys on small bucket tables.
// Keys spread past the direct index's range, so the maps hash them.
const spread = new Map<number, number>();
for (let i = 0; i < 500; i++) spread.set(i * 104729 - 26000000, i);
let found = 0,
absent = 0,
total = 0;
for (let i = -10; i < 520; i++) {
const v = spread.get(i * 104729 - 26000000);
if (v === undefined) absent++;
else {
found++;
total += v;
}
}
console.log("spread", spread.size, found, absent, total);
console.log("spread literals", spread.get(-26000000), spread.get(-25895271), spread.get(0), spread.has(-1));
// Zero stored through -0 and through an integer; read back through both.
const zeros = new Map<number, string>();
for (let i = 1; i <= 20; i++) zeros.set(i * 1000003, "k" + i);
zeros.set(-0, "minus zero");
let z = 0;
console.log("zeros", zeros.size, zeros.get(z), zeros.get(0), zeros.get(-0), zeros.get(z * -1));
zeros.set(z, "zero");
console.log("zeros overwrite", zeros.size, zeros.get(-0), [...zeros.keys()].includes(0));
// Large and negative integer keys.
const wide = new Map<number, number>();
for (let i = 0; i < 64; i++) wide.set(2 ** 40 + i * 3, i);
for (let i = 0; i < 64; i++) wide.set(-(2 ** 33) - i, -i);
let wideUp = 0,
wideDown = 0,
wideMissing = 0;
for (let i = 0; i < 200; i++) {
const a = wide.get(2 ** 40 + i);
const b = wide.get(-(2 ** 33) - i);
if (a === undefined) wideMissing++;
else wideUp += a;
if (b === undefined) wideMissing++;
else wideDown += b;
}
console.log("wide", wide.size, wideUp, wideDown, wideMissing, wide.get(1099511627776), wide.get(-8589934592));
// Absent keys on bucket tables just past the linear scan, and as they grow.
for (const size of [9, 12, 16, 33, 64, 100, 257, 400]) {
const m = new Map<number, number>();
for (let i = 0; i < size; i++) m.set(i * 7919 + 1000000, i);
let hits = 0,
misses = 0;
for (let i = 0; i < size * 4; i++) {
if (m.has(i * 7919 + 1000000)) hits++;
else misses++;
if (m.get(i * 7919 + 1000001) !== undefined) hits += 1000;
}
for (let i = 0; i < size; i += 2) m.delete(i * 7919 + 1000000);
let after = 0;
for (let i = 0; i < size; i++) if (m.has(i * 7919 + 1000000)) after++;
console.log("size", size, hits, misses, m.size, after, m.get(1000000), m.get(1007919));
}
// Small maps that scan their entries, with integer keys.
const tiny = new Map<number, string>();
for (let i = 0; i < 8; i++) tiny.set(i * 5000011, "t" + i);
let tinyHits = 0;
for (let i = 0; i < 40; i++) if (tiny.has(i * 5000011)) tinyHits++;
console.log("tiny", tiny.size, tinyHits, tiny.get(0), tiny.get(35000077), tiny.has(5000012));
// Sets with integer keys.
const seen = new Set<number>();
let dupes = 0;
for (let i = 0; i < 3000; i++) {
const k = ((i * 2654435761) % 1000003) - 500000;
if (seen.has(k)) dupes++;
else seen.add(k);
}
let present = 0;
for (let i = -500000; i < 500003; i += 997) if (seen.has(i)) present++;
console.log("set", seen.size, dupes, present, seen.has(-500000), seen.has(500002));
// A counting loop like a k-mer histogram: get, then update the found entry.
class Count {
n = 0;
}
const counts = new Map<number, Count>();
let key = 0;
for (let i = 0; i < 20000; i++) {
key = ((key * 4) % 4194304) + (i % 7 === 0 ? 3 : i % 3);
const c = counts.get(key);
if (c === undefined) {
const fresh = new Count();
fresh.n = 1;
counts.set(key, fresh);
} else c.n++;
}
let most = 0,
mostKey = -1;
for (const [k, c] of counts) if (c.n > most) [most, mostKey] = [c.n, k];
console.log("counts", counts.size, most, mostKey);
@@ -0,0 +1 @@
{ "plain": 3441, "san": 1752 }