diff --git a/packages/compiler/src/backend/llvm/emit-module.ts b/packages/compiler/src/backend/llvm/emit-module.ts index 8fdadfdcf..e5c8c5d6d 100644 --- a/packages/compiler/src/backend/llvm/emit-module.ts +++ b/packages/compiler/src/backend/llvm/emit-module.ts @@ -35,6 +35,8 @@ import { emitClassMembershipHelper } from "./classes.js"; import { type RegexMatcherCode, compileRegexMatcher } from "./regex-matchers.js"; import { MAP_ITER_DECL, + MAP_ITER_NUM_DECL, + SCR_MAP_NUM_DEAD, NUMBER_MAP_ENTRY_DECL, STRING_MAP_ENTRY_DECL, } from "./map-number-lookup.js"; @@ -450,25 +452,29 @@ export function emitModuleParts(emitter: LlEmitter): string[] { // The ScrMap prefix { rc, key_kind, val_kind, traced } — the inline // RC fast paths test the traced byte (shapes.ts). ...(emitter.rcHelpers === null ? [] : [`%ScrMapRc = type { ${emitter.sizeType}, i8, i8, i8 }`]), - // The complete ScrMap and its { key, val, hash } entry, read by the - // inline number-key lookup (sc_map_entry_f64) and the inline - // iteration steps (sc_map_iter_entry): field 8 is the i32 entry + // The complete ScrMap and its entries, read by the inline number-key + // lookup (sc_map_entry_f64) and the inline iteration steps + // (sc_map_iter_entry, sc_map_iter_entry_num): field 8 is the i32 entry // count, field 10 the i64 absent-key filter, entries is field 11, // nbuckets and buckets fields 13 and 14, the direct index and its i32 // length fields 15 and 16. ...(emitter.decls.has(NUMBER_MAP_ENTRY_DECL) || emitter.decls.has(STRING_MAP_ENTRY_DECL) || - emitter.decls.has(MAP_ITER_DECL) + emitter.decls.has(MAP_ITER_DECL) || + emitter.decls.has(MAP_ITER_NUM_DECL) ? [ `%ScrMapIx = type { ${emitter.sizeType}, i8, i8, i8, i8, i32, ptr, ptr, i32, i32, i64, ptr, i32, i32, ptr, ptr, i32, i32 }`, ] : []), - // String-keyed reference reads load the value slot of the entry - // scr_map_entry_str returns (map-number-lookup.ts). + // The { key, val, hash } entry of every map but a number-key one, whose + // entry is { key, val } (scr_map.h ScrMapNumEntry). String-keyed + // reference reads load the value slot of the entry scr_map_entry_str + // returns (map-number-lookup.ts). ...(emitter.decls.has(NUMBER_MAP_ENTRY_DECL) || emitter.decls.has(STRING_MAP_ENTRY_DECL) || - emitter.decls.has(MAP_ITER_DECL) - ? [`%ScrMapEntry = type { i64, i64, i64 }`] + emitter.decls.has(MAP_ITER_DECL) || + emitter.decls.has(MAP_ITER_NUM_DECL) + ? [`%ScrMapEntry = type { i64, i64, i64 }`, `%ScrMapNumEntry = type { i64, i64 }`] : []), // The runtime error prefix { rc, vt, name, message, code, cause } and the // class-object shape { rc, pre, post, ctor, name } — field reads on @@ -1086,7 +1092,7 @@ export function helperDefs(emitter: LlEmitter): string[] { sz === "i32" ? ` %e = add i32 %e32, 0` : ` %e = zext i32 %e32 to ${sz}`, ` %ep = getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 11`, ` %entries = load ptr, ptr %ep`, - ` %found = getelementptr inbounds %ScrMapEntry, ptr %entries, ${sz} %e`, + ` %found = getelementptr inbounds %ScrMapNumEntry, ptr %entries, ${sz} %e`, ` ret ptr %found`, `miss:`, ` ret ptr null`, @@ -1138,7 +1144,8 @@ export function helperDefs(emitter: LlEmitter): string[] { ` %ents = load ptr, ptr %eps`, ` %linear = icmp eq ${sz} %nb, 0`, ` br i1 %linear, label %scan, label %buckets`, - // Small maps keep no bucket table: scan the cached entry hashes. + // Small maps keep no bucket table: scan the entries' keys. A number-key + // entry has no cached hash; a dead entry's key matches no key. `scan:`, ` %nep = getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 8`, ...(sz === "i64" @@ -1150,14 +1157,10 @@ export function helperDefs(emitter: LlEmitter): string[] { ` %sdone = icmp eq ${sz} %si, %ne`, ` br i1 %sdone, label %miss, label %scan.entry`, `scan.entry:`, - ` %se = getelementptr inbounds %ScrMapEntry, ptr %ents, ${sz} %si`, - ` %seh = getelementptr inbounds %ScrMapEntry, ptr %se, i32 0, i32 2`, - ` %sh = load i64, ptr %seh`, + ` %se = getelementptr inbounds %ScrMapNumEntry, ptr %ents, ${sz} %si`, ` %sk = load i64, ptr %se`, - ` %shq = icmp eq i64 %sh, %h`, ` %skq = icmp eq i64 %sk, %kb`, - ` %sq = and i1 %shq, %skq`, - ` br i1 %sq, label %scan.found, label %scan.next`, + ` br i1 %skq, label %scan.found, label %scan.next`, `scan.found:`, ` ret ptr %se`, `scan.next:`, @@ -1192,14 +1195,10 @@ export function helperDefs(emitter: LlEmitter): string[] { ` %bix = and i32 %b, 16777215`, ` %bix1 = sub i32 %bix, 1`, sz === "i32" ? ` %bi = add i32 %bix1, 0` : ` %bi = zext i32 %bix1 to ${sz}`, - ` %pe = getelementptr inbounds %ScrMapEntry, ptr %ents, ${sz} %bi`, - ` %peh = getelementptr inbounds %ScrMapEntry, ptr %pe, i32 0, i32 2`, - ` %ph = load i64, ptr %peh`, + ` %pe = getelementptr inbounds %ScrMapNumEntry, ptr %ents, ${sz} %bi`, ` %pk = load i64, ptr %pe`, - ` %phq = icmp eq i64 %ph, %h`, ` %pkq = icmp eq i64 %pk, %kb`, - ` %pq = and i1 %phq, %pkq`, - ` br i1 %pq, label %probe.found, label %probe.next`, + ` br i1 %pkq, label %probe.found, label %probe.next`, `probe.found:`, ` ret ptr %pe`, `probe.next:`, @@ -1217,36 +1216,66 @@ export function helperDefs(emitter: LlEmitter): string[] { } // Map/Set iteration steps read the entries array inline: an index is // live when it is below the entry count and its entry's cached hash is - // nonzero (a tombstone's is zero), as scr_map_iter_live decides. The - // entry count is read again at every step, so entries appended during - // the iteration are visited. - if (emitter.decls.has(MAP_ITER_DECL)) { + // nonzero (a tombstone's is zero), as scr_map_iter_live decides, or for + // a number-key map when its key is not SCR_MAP_NUM_DEAD. The entry count + // is read again at every step, so entries appended during the iteration + // are visited. + if (emitter.decls.has(MAP_ITER_DECL) || emitter.decls.has(MAP_ITER_NUM_DECL)) { const sz = emitter.sizeType; + if (emitter.decls.has(MAP_ITER_DECL)) + defs.push( + `define internal ptr @sc_map_iter_entry(ptr %m, double %i) ${FN_ATTRS} {`, + `entry:`, + ` %np = getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 8`, + ` %n = load i32, ptr %np`, + ` %nf = uitofp i32 %n to double`, + ` %lo = fcmp oge double %i, 0.0`, + ` %hi = fcmp olt double %i, %nf`, + ` %in = and i1 %lo, %hi`, + ` br i1 %in, label %load, label %none`, + `load:`, + ` %ix = fptoui double %i to ${sz}`, + ` %ep = getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 11`, + ` %entries = load ptr, ptr %ep`, + ` %e = getelementptr inbounds %ScrMapEntry, ptr %entries, ${sz} %ix`, + ` %hp = getelementptr inbounds %ScrMapEntry, ptr %e, i32 0, i32 2`, + ` %h = load i64, ptr %hp`, + ` %live = icmp ne i64 %h, 0`, + ` br i1 %live, label %yes, label %none`, + `yes:`, + ` ret ptr %e`, + `none:`, + ` ret ptr null`, + `}`, + ``, + ); + if (emitter.decls.has(MAP_ITER_NUM_DECL)) + defs.push( + `define internal ptr @sc_map_iter_entry_num(ptr %m, double %i) ${FN_ATTRS} {`, + `entry:`, + ` %np = getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 8`, + ` %n = load i32, ptr %np`, + ` %nf = uitofp i32 %n to double`, + ` %lo = fcmp oge double %i, 0.0`, + ` %hi = fcmp olt double %i, %nf`, + ` %in = and i1 %lo, %hi`, + ` br i1 %in, label %load, label %none`, + `load:`, + ` %ix = fptoui double %i to ${sz}`, + ` %ep = getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 11`, + ` %entries = load ptr, ptr %ep`, + ` %e = getelementptr inbounds %ScrMapNumEntry, ptr %entries, ${sz} %ix`, + ` %key = load i64, ptr %e`, + ` %live = icmp ne i64 %key, ${SCR_MAP_NUM_DEAD}`, + ` br i1 %live, label %yes, label %none`, + `yes:`, + ` ret ptr %e`, + `none:`, + ` ret ptr null`, + `}`, + ``, + ); defs.push( - `define internal ptr @sc_map_iter_entry(ptr %m, double %i) ${FN_ATTRS} {`, - `entry:`, - ` %np = getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 8`, - ` %n = load i32, ptr %np`, - ` %nf = uitofp i32 %n to double`, - ` %lo = fcmp oge double %i, 0.0`, - ` %hi = fcmp olt double %i, %nf`, - ` %in = and i1 %lo, %hi`, - ` br i1 %in, label %load, label %none`, - `load:`, - ` %ix = fptoui double %i to ${sz}`, - ` %ep = getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 11`, - ` %entries = load ptr, ptr %ep`, - ` %e = getelementptr inbounds %ScrMapEntry, ptr %entries, ${sz} %ix`, - ` %hp = getelementptr inbounds %ScrMapEntry, ptr %e, i32 0, i32 2`, - ` %h = load i64, ptr %hp`, - ` %live = icmp ne i64 %h, 0`, - ` br i1 %live, label %yes, label %none`, - `yes:`, - ` ret ptr %e`, - `none:`, - ` ret ptr null`, - `}`, - ``, `define internal double @sc_map_iter_count(ptr %m) ${FN_ATTRS} {`, `entry:`, ` %np = getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 8`, diff --git a/packages/compiler/src/backend/llvm/expr-containers.ts b/packages/compiler/src/backend/llvm/expr-containers.ts index afa69998d..945d83764 100644 --- a/packages/compiler/src/backend/llvm/expr-containers.ts +++ b/packages/compiler/src/backend/llvm/expr-containers.ts @@ -1430,7 +1430,7 @@ export function emitMapLikeIntrinsic( return { name: emitMapIterStart(host, r.name), type: e.type }; case "iterLive": { const i = host.emitExpr(e.args[0]!); - const entry = emitMapIterEntry(host, r.name, i.name); + const entry = emitMapIterEntry(host, r.name, i.name, kAcc === "f64"); const t = B.tmp(); B.line(`${t} = icmp ne ptr ${entry}, null`); return { name: t, type: e.type }; @@ -1441,10 +1441,16 @@ export function emitMapLikeIntrinsic( if (kAcc !== "ref") { const retTy = kAcc === "f64" ? "double" : kAcc === "bool" ? "zeroext i1" : "ptr"; const symbol = `scr_map_iter_key_${kAcc}`; - const t = emitMapIterSlot(host, r.name, i.name, 0, kAcc === "str" ? "ref" : kAcc, key, { - decl: `declare ${retTy} @${symbol}(ptr, double)`, - symbol, - }); + const t = emitMapIterSlot( + host, + r.name, + i.name, + 0, + kAcc === "str" ? "ref" : kAcc, + key, + { decl: `declare ${retTy} @${symbol}(ptr, double)`, symbol }, + kAcc === "f64", + ); return host.own({ name: t, type: e.type }); } const retTy = "ptr"; @@ -1459,10 +1465,19 @@ export function emitMapLikeIntrinsic( const i = host.emitExpr(e.args[0]!); const retTy = vAcc === "f64" ? "double" : vAcc === "bool" ? "i1" : "ptr"; const symbol = `scr_map_iter_val_${vAcc}`; - const t = emitMapIterSlot(host, r.name, i.name, 1, vAcc, receiverType.value, { - decl: `declare ${vAcc === "bool" ? "zeroext i1" : retTy} @${symbol}(ptr, double)`, - symbol, - }); + const t = emitMapIterSlot( + host, + r.name, + i.name, + 1, + vAcc, + receiverType.value, + { + decl: `declare ${vAcc === "bool" ? "zeroext i1" : retTy} @${symbol}(ptr, double)`, + symbol, + }, + kAcc === "f64", + ); return host.own({ name: t, type: e.type }); } case "iterEnter": diff --git a/packages/compiler/src/backend/llvm/map-number-lookup.ts b/packages/compiler/src/backend/llvm/map-number-lookup.ts index f0aa03497..dbe4e93c9 100644 --- a/packages/compiler/src/backend/llvm/map-number-lookup.ts +++ b/packages/compiler/src/backend/llvm/map-number-lookup.ts @@ -35,7 +35,8 @@ export function emitNumberMapEntry(host: LlvmEmitterContext, map: string, key: s return entry; } -/** Load a found entry's value slot as `type` (double, i64 or ptr). */ +/** Load a found entry's value slot as `type` (double, i64 or ptr). The value + * is at byte 8 in both entry layouts (%ScrMapEntry, %ScrMapNumEntry). */ export function emitNumberMapSlot(host: LlvmEmitterContext, entry: string, type: string): string { const B = host.B; const slot = B.tmp(), @@ -190,13 +191,25 @@ export function emitNumberMapSetScalar( /** Declared by the inline Map/Set iteration steps; its presence makes the * emitter define sc_map_iter_entry and sc_map_iter_count (emitter.ts). */ export const MAP_ITER_DECL = "declare zeroext i1 @scr_map_iter_live(ptr, double)"; +/** Declared by the iteration steps over a number-key Map/Set; its presence + * makes the emitter define sc_map_iter_entry_num and sc_map_iter_count. */ +export const MAP_ITER_NUM_DECL = "declare double @scr_map_iter_count(ptr)"; +/** scr_map.h SCR_MAP_NUM_DEAD: a dead number-key entry's key word. */ +export const SCR_MAP_NUM_DEAD = "9218868437227405313"; /** The entry at an iteration index, or null for a tombstone or an index at - * or past the entry count (the scr_map_iter_live test). */ -export function emitMapIterEntry(host: LlvmEmitterContext, map: string, index: string): string { - host.declare(MAP_ITER_DECL); + * or past the entry count (the scr_map_iter_live test). `num`: the map has + * number keys, whose entries are { key, val } (scr_map.h ScrMapNumEntry). */ +export function emitMapIterEntry( + host: LlvmEmitterContext, + map: string, + index: string, + num: boolean, +): string { + host.declare(num ? MAP_ITER_NUM_DECL : MAP_ITER_DECL); const entry = host.B.tmp(); - host.B.line(`${entry} = call ptr @sc_map_iter_entry(ptr ${map}, double ${index})`); + const helper = num ? "sc_map_iter_entry_num" : "sc_map_iter_entry"; + host.B.line(`${entry} = call ptr @${helper}(ptr ${map}, double ${index})`); return entry; } @@ -230,9 +243,10 @@ export function emitMapIterSlot( access: "f64" | "bool" | "ref", type: IrType, runtime: { decl: string; symbol: string }, + num: boolean, ): string { const B = host.B; - const entry = emitMapIterEntry(host, map, index); + const entry = emitMapIterEntry(host, map, index, num); const found = B.tmp(); B.line(`${found} = icmp ne ptr ${entry}, null`); const hit = B.newLabel("mapit.hit"), diff --git a/packages/runtime/src/scr_json.c b/packages/runtime/src/scr_json.c index d9c1e5fab..7e37d0501 100644 --- a/packages/runtime/src/scr_json.c +++ b/packages/runtime/src/scr_json.c @@ -10822,8 +10822,8 @@ void scr_jb_put_dyn_entries(ScrJsonBuf *b, void *rec, ScrMap *map, bool *first) uint32_t count = map->nentries; bool indexed = false; for (uint32_t i = 0; i < count && !indexed; i++) { - if (!map->entries[i].hash) continue; - const ScrStr *key = (const ScrStr *)(uintptr_t)map->entries[i].key; + if (!scr_map_entry_live(map, i)) continue; + const ScrStr *key = (const ScrStr *)(uintptr_t)scr_map_entry_at(map, i)->key; uint32_t index; indexed = scr_key_array_index(key->data, key->len, &index); } diff --git a/packages/runtime/src/scr_lib.c b/packages/runtime/src/scr_lib.c index 8d3b668c1..f7e5d52e7 100644 --- a/packages/runtime/src/scr_lib.c +++ b/packages/runtime/src/scr_lib.c @@ -12709,7 +12709,7 @@ static bool scr_set_to_arr_sparse(const ScrMap *s, ScrArr *out) { /* scr_arr_new reserves dense storage for the live count; keep the * ordinary append path in case an allocation ever declines it. */ for (size_t i = 0; i < s->nentries; i++) { - if (!s->entries[i].hash) continue; + if (!scr_map_entry_live(s, i)) continue; if (out->elem == SCR_ELEM_F64) scr_arr_push_f64(out, scr_map_iter_key_f64(s, (double)i)); else if (out->elem == SCR_ELEM_BOOL) scr_arr_push_bool(out, scr_map_iter_key_bool(s, (double)i)); else if (out->elem == SCR_ELEM_STR) scr_arr_push_ref(out, scr_map_iter_key_str(s, (double)i)); @@ -12723,8 +12723,8 @@ ScrArr *scr_set_to_arr_f64(const ScrMap *s) { if (scr_set_to_arr_sparse(s, out)) return out; uint8_t *states = scr_arr_states_end(out); for (size_t i = 0; i < s->nentries; i++) { - if (!s->entries[i].hash) continue; - out->data[out->len] = s->entries[i].key; + if (!scr_map_entry_live(s, i)) continue; + out->data[out->len] = scr_map_entry_at(s, i)->key; *scr_arr_state_in(states, out->len++) = SCR_ARR_VALUE; } return out; @@ -12735,8 +12735,8 @@ ScrArr *scr_set_to_arr_bool(const ScrMap *s) { if (scr_set_to_arr_sparse(s, out)) return out; uint8_t *states = scr_arr_states_end(out); for (size_t i = 0; i < s->nentries; i++) { - if (!s->entries[i].hash) continue; - scr_arr_dense_put(out, out->len, s->entries[i].key != 0); + if (!scr_map_entry_live(s, i)) continue; + scr_arr_dense_put(out, out->len, scr_map_entry_at(s, i)->key != 0); *scr_arr_state_in(states, out->len++) = SCR_ARR_VALUE; } return out; @@ -12747,9 +12747,9 @@ ScrArr *scr_set_to_arr_str(const ScrMap *s) { if (scr_set_to_arr_sparse(s, out)) return out; uint8_t *states = scr_arr_states_end(out); for (size_t i = 0; i < s->nentries; i++) { - if (!s->entries[i].hash) continue; - scr_str_retain((ScrStr *)(uintptr_t)s->entries[i].key); - out->data[out->len] = s->entries[i].key; + if (!scr_map_entry_live(s, i)) continue; + scr_str_retain((ScrStr *)(uintptr_t)scr_map_entry_at(s, i)->key); + out->data[out->len] = scr_map_entry_at(s, i)->key; *scr_arr_state_in(states, out->len++) = SCR_ARR_VALUE; } return out; @@ -12760,9 +12760,9 @@ ScrArr *scr_set_to_arr_ref(const ScrMap *s, const ScrArrOps *ops) { if (scr_set_to_arr_sparse(s, out)) return out; uint8_t *states = scr_arr_states_end(out); for (size_t i = 0; i < s->nentries; i++) { - if (!s->entries[i].hash) continue; - ops->retain((void *)(uintptr_t)s->entries[i].key); - out->data[out->len] = s->entries[i].key; + if (!scr_map_entry_live(s, i)) continue; + ops->retain((void *)(uintptr_t)scr_map_entry_at(s, i)->key); + out->data[out->len] = scr_map_entry_at(s, i)->key; *scr_arr_state_in(states, out->len++) = SCR_ARR_VALUE; } return out; diff --git a/packages/runtime/src/scr_map.c b/packages/runtime/src/scr_map.c index 647f559dc..ecd1a558b 100644 --- a/packages/runtime/src/scr_map.c +++ b/packages/runtime/src/scr_map.c @@ -215,6 +215,54 @@ static bool scr_map_key_eq(const ScrMap *m, uint64_t stored, uint64_t probe) { (ScrStr *)scr_map_slot_to_ptr(probe)); } +/* ── entry layout ────────────────────────────────────────────────────── + * A number-key map (SCR_MAP_KEY_F64) keeps 16-byte entries, the key and + * the value: its hash is the key word's scr_key_mix, which is cheap to + * recompute, and its stored keys are normalized (one NaN pattern, +0), so + * a dead entry marks its key with a NaN pattern that no stored key has. + * Every other map keeps 24-byte entries with the cached hash, zero for a + * dead entry. Both layouts put the key at byte 0 and the value at byte 8. */ +static inline bool scr_map_num(const ScrMap *m) { + return m->key_kind == SCR_MAP_KEY_F64; +} + +static inline size_t scr_map_entry_size(const ScrMap *m) { + return scr_map_num(m) ? sizeof(ScrMapNumEntry) : sizeof(ScrMapEntry); +} + +/* The entry at index e. A number-key entry has no hash field: read its key + * and value only (scr_map_hash_at gives the hash). */ +static inline ScrMapEntry *scr_map_at(const ScrMap *m, size_t e) { + return (ScrMapEntry *)((char *)m->entries + e * scr_map_entry_size(m)); +} + +static inline bool scr_map_live_at(const ScrMap *m, size_t e) { + const ScrMapEntry *x = scr_map_at(m, e); + return scr_map_num(m) ? x->key != SCR_MAP_NUM_DEAD : x->hash != 0; +} + +/* The entry's hash, nonzero when it is live and zero for a dead entry. */ +static inline uint64_t scr_map_hash_at(const ScrMap *m, size_t e) { + const ScrMapEntry *x = scr_map_at(m, e); + if (!scr_map_num(m)) return x->hash; + if (x->key == SCR_MAP_NUM_DEAD) return 0; + uint64_t hash = scr_map_hash_word(x->key); + return hash ? hash : 1; +} + +static inline void scr_map_kill_at(ScrMap *m, size_t e) { + ScrMapEntry *x = scr_map_at(m, e); + if (scr_map_num(m)) x->key = SCR_MAP_NUM_DEAD; + else x->hash = 0; +} + +static inline void scr_map_put_at(ScrMap *m, size_t e, uint64_t key, uint64_t val, uint64_t hash) { + ScrMapEntry *x = scr_map_at(m, e); + x->key = key; + x->val = val; + if (!scr_map_num(m)) x->hash = hash; +} + /* ── absent-key filter ───────────────────────────────────────────────── * Every live entry's hash sets two bits of m->filter, so a lookup whose * two bits are not both set is absent: most misses on small and medium @@ -233,7 +281,17 @@ static inline uint64_t scr_map_filter_bits(uint64_t hash) { * A zero entry hash marks a tombstone; cached hashes reject unrelated keys * before equality and survive table growth without hashing the keys again. * Remap zero to one so every live entry has a nonzero hash. */ -typedef enum { SCR_MAP_LOOKUP_STR, SCR_MAP_LOOKUP_SPAN, SCR_MAP_LOOKUP_WORD, SCR_MAP_LOOKUP_GENERIC } ScrMapLookup; +/* SCR_MAP_LOOKUP_NUM is a word compare on a number-key map, so the probe + * reads 16-byte entries; every other lookup is on a map with 24-byte + * entries. Knowing the layout from the lookup keeps the entry stride a + * constant in the inlined probes. */ +typedef enum { + SCR_MAP_LOOKUP_STR, + SCR_MAP_LOOKUP_SPAN, + SCR_MAP_LOOKUP_WORD, + SCR_MAP_LOOKUP_NUM, + SCR_MAP_LOOKUP_GENERIC +} ScrMapLookup; typedef struct { const char *bytes; size_t length; } ScrMapSpan; @@ -242,7 +300,7 @@ typedef struct { const char *bytes; size_t length; } ScrMapSpan; * while boxed keys retain their complete value/identity semantics. */ static inline bool scr_map_lookup_eq(const ScrMap *m, uint64_t stored, uint64_t key, ScrMapLookup lookup) { - if (lookup == SCR_MAP_LOOKUP_WORD) return stored == key; + if (lookup == SCR_MAP_LOOKUP_WORD || lookup == SCR_MAP_LOOKUP_NUM) return stored == key; if (lookup == SCR_MAP_LOOKUP_SPAN) { const ScrStr *a = scr_map_slot_to_ptr(stored); const ScrMapSpan *b = scr_map_slot_to_ptr(key); @@ -256,6 +314,23 @@ static inline bool scr_map_lookup_eq(const ScrMap *m, uint64_t stored, uint64_t return scr_map_key_eq(m, stored, key); } +/* The entry at index e of a map that `lookup` probes (see ScrMapLookup). */ +static inline __attribute__((always_inline)) ScrMapEntry *scr_map_at_with(const ScrMap *m, size_t e, + ScrMapLookup lookup) { + if (lookup == SCR_MAP_LOOKUP_NUM) return (ScrMapEntry *)((ScrMapNumEntry *)m->entries + e); + return &m->entries[e]; +} + +/* Whether entry e holds `key`, whose hash is `hash`. A number-key map + * compares the normalized key words alone; a dead entry's marker matches + * no key. */ +static inline __attribute__((always_inline)) bool scr_map_match_at( + const ScrMap *m, size_t e, uint64_t hash, uint64_t key, ScrMapLookup lookup) { + const ScrMapEntry *x = scr_map_at_with(m, e, lookup); + if (lookup == SCR_MAP_LOOKUP_NUM) return x->key == key; + return x->hash == hash && scr_map_lookup_eq(m, x->key, key, lookup); +} + /* Compact tables keep a hash fingerprint beside each 24-bit entry index. * Rejecting a collision here avoids loading an unrelated ordered entry. * Larger tables keep full-width indices; collection size remains limited @@ -305,14 +380,14 @@ static inline __attribute__((always_inline)) size_t scr_map_probe_with( if (!bucket) return i; if ((bucket & ~SCR_MAP_INDEX_MASK) != tag) continue; size_t b = (bucket & SCR_MAP_INDEX_MASK) - 1; - if (m->entries[b].hash == hash && scr_map_lookup_eq(m, m->entries[b].key, key, lookup)) return i; + if (scr_map_match_at(m, b, hash, key, lookup)) return i; } } const size_t *buckets = m->buckets; for (size_t i = hash & mask;; i = (i + 1) & mask) { size_t b = buckets[i]; if (b == SCR_MAP_EMPTY) return i; - if (m->entries[b].hash == hash && scr_map_lookup_eq(m, m->entries[b].key, key, lookup)) return i; + if (scr_map_match_at(m, b, hash, key, lookup)) return i; } } @@ -329,14 +404,14 @@ static inline __attribute__((always_inline)) size_t scr_map_probe_entry_with( if (!bucket) return SCR_MAP_EMPTY; if ((bucket & ~SCR_MAP_INDEX_MASK) != tag) continue; size_t b = (bucket & SCR_MAP_INDEX_MASK) - 1; - if (m->entries[b].hash == hash && scr_map_lookup_eq(m, m->entries[b].key, key, lookup)) return b; + if (scr_map_match_at(m, b, hash, key, lookup)) return b; } } const size_t *buckets = m->buckets; for (size_t i = hash & mask;; i = (i + 1) & mask) { size_t b = buckets[i]; if (b == SCR_MAP_EMPTY) return SCR_MAP_EMPTY; - if (m->entries[b].hash == hash && scr_map_lookup_eq(m, m->entries[b].key, key, lookup)) return b; + if (scr_map_match_at(m, b, hash, key, lookup)) return b; } } @@ -353,7 +428,7 @@ static inline __attribute__((always_inline)) size_t scr_map_find_with( if ((m->filter & bits) != bits) return SCR_MAP_EMPTY; if (m->nbuckets == 0) { for (size_t e = 0; e < m->nentries; e++) { - if (m->entries[e].hash == hash && scr_map_lookup_eq(m, m->entries[e].key, key, lookup)) return e; + if (scr_map_match_at(m, e, hash, key, lookup)) return e; } return SCR_MAP_EMPTY; } @@ -363,7 +438,7 @@ static inline __attribute__((always_inline)) size_t scr_map_find_with( static ScrMapLookup scr_map_lookup_kind(const ScrMap *m) { switch (m->key_kind) { case SCR_MAP_KEY_STR: return SCR_MAP_LOOKUP_STR; - case SCR_MAP_KEY_F64: + case SCR_MAP_KEY_F64: return SCR_MAP_LOOKUP_NUM; case SCR_MAP_KEY_BOOL: case SCR_MAP_KEY_REF: case SCR_MAP_KEY_NULL: @@ -410,6 +485,7 @@ static size_t scr_map_find(const ScrMap *m, uint64_t hash, uint64_t key) { switch (scr_map_lookup_kind(m)) { case SCR_MAP_LOOKUP_STR: return scr_map_find_with(m, hash, key, SCR_MAP_LOOKUP_STR); case SCR_MAP_LOOKUP_WORD: return scr_map_find_with(m, hash, key, SCR_MAP_LOOKUP_WORD); + case SCR_MAP_LOOKUP_NUM: return scr_map_find_with(m, hash, key, SCR_MAP_LOOKUP_NUM); default: return scr_map_find_with(m, hash, key, SCR_MAP_LOOKUP_GENERIC); } } @@ -418,12 +494,12 @@ static size_t scr_map_find_f64(const ScrMap *m, double key) { if (m->ndense) return scr_map_find_dense(m, key); if (m->nlive == 0) return SCR_MAP_EMPTY; uint64_t k = scr_map_f64_bits(key); - return scr_map_find_with(m, scr_map_hash_word(k), k, SCR_MAP_LOOKUP_WORD); + return scr_map_find_with(m, scr_map_hash_word(k), k, SCR_MAP_LOOKUP_NUM); } const ScrMapEntry *scr_map_entry_f64(const ScrMap *m, double key) { size_t e = scr_map_find_f64(m, key); - return e == SCR_MAP_EMPTY ? NULL : &m->entries[e]; + return e == SCR_MAP_EMPTY ? NULL : scr_map_at(m, e); } /* The key's hash is usually cached, so even the small linear scan rejects @@ -447,20 +523,20 @@ static size_t scr_map_find_span(const ScrMap *m, const char *bytes, size_t lengt bool scr_map_get_span_f64(const ScrMap *m, const char *bytes, size_t length, double *out) { size_t e = scr_map_find_span(m, bytes, length); if (e == SCR_MAP_EMPTY) return false; - *out = scr_map_slot_to_f64(m->entries[e].val); + *out = scr_map_slot_to_f64(scr_map_at(m, e)->val); return true; } bool scr_map_get_span_bool(const ScrMap *m, const char *bytes, size_t length, bool *out) { size_t e = scr_map_find_span(m, bytes, length); if (e == SCR_MAP_EMPTY) return false; - *out = m->entries[e].val != 0; + *out = scr_map_at(m, e)->val != 0; return true; } void *scr_map_get_span_ref(const ScrMap *m, const char *bytes, size_t length) { size_t e = scr_map_find_span(m, bytes, length); - return e == SCR_MAP_EMPTY ? NULL : m->ops->val_retain(scr_map_slot_to_ptr(m->entries[e].val)); + return e == SCR_MAP_EMPTY ? NULL : m->ops->val_retain(scr_map_slot_to_ptr(scr_map_at(m, e)->val)); } /* Rebuild the bucket table (size must be a power of two >= 2 * nentries): @@ -476,7 +552,7 @@ static void scr_map_rebuild_buckets(ScrMap *m, size_t nbuckets) { m->nbuckets = (uint32_t)nbuckets; size_t mask = nbuckets - 1; for (size_t e = 0; e < m->nentries; e++) { - uint64_t hash = m->entries[e].hash; + uint64_t hash = scr_map_hash_at(m, e); if (!hash) continue; size_t i = hash & mask; while (scr_map_bucket(m, i) != SCR_MAP_EMPTY) i = (i + 1) & mask; @@ -488,21 +564,23 @@ static void scr_map_rebuild_buckets(ScrMap *m, size_t nbuckets) { static void scr_map_fill_dense(ScrMap *m) { memset(m->dense, 0, m->ndense * sizeof *m->dense); for (size_t e = 0; e < m->nentries; e++) { - if (!m->entries[e].hash) continue; - m->dense[(size_t)scr_map_slot_to_f64(m->entries[e].key)] = (uint32_t)(e + 1); + if (!scr_map_live_at(m, e)) continue; + m->dense[(size_t)scr_map_slot_to_f64(scr_map_at(m, e)->key)] = (uint32_t)(e + 1); } } -/* A table of at least SCR_MAP_DENSE_MIN slots covering index `top`. */ +/* A table of at least SCR_MAP_DENSE_MIN slots covering index `top`, rounded + * up to a multiple of SCR_MAP_DENSE_STEP. Lookups only compare a key with + * ndense, so the length need not be a power of two. */ +#define SCR_MAP_DENSE_STEP 64 static size_t scr_map_dense_size(size_t top) { - size_t n = SCR_MAP_DENSE_MIN; - while (n <= top) n *= 2; - return n; + if (top < SCR_MAP_DENSE_MIN) return SCR_MAP_DENSE_MIN; + return (top / SCR_MAP_DENSE_STEP + 1) * SCR_MAP_DENSE_STEP; } /* The exclusive key bound a table may cover once it holds `live` entries. * Entry indices are stored as uint32_t + 1, and the table's length (a - * power of two above every covered key) fits ScrMap.ndense. */ + * multiple of SCR_MAP_DENSE_STEP covering every key) fits ScrMap.ndense. */ #define SCR_MAP_DENSE_MAX ((size_t)1 << 31) static size_t scr_map_dense_limit(size_t live) { if (live > SIZE_MAX / SCR_MAP_DENSE_SPREAD) return SCR_MAP_DENSE_MAX; @@ -526,8 +604,8 @@ static __attribute__((noinline)) void scr_map_enter_dense(ScrMap *m, double inco size_t limit = scr_map_dense_limit(m->nlive + 1), top, i; if (m->nentries >= UINT32_MAX - 1 || !scr_map_dense_slot(incoming, limit, &top)) return; for (size_t e = 0; e < m->nentries; e++) { - if (!m->entries[e].hash) continue; - if (!scr_map_dense_slot(scr_map_slot_to_f64(m->entries[e].key), limit, &i)) return; + if (!scr_map_live_at(m, e)) continue; + if (!scr_map_dense_slot(scr_map_slot_to_f64(scr_map_at(m, e)->key), limit, &i)) return; if (i > top) top = i; } scr_mem_free(m->buckets); @@ -537,11 +615,15 @@ static __attribute__((noinline)) void scr_map_enter_dense(ScrMap *m, double inco scr_map_fill_dense(m); } -/* Extend the table to cover a new integral key while it stays compact. */ +/* Extend the table to cover a new integral key while it stays compact. It + * grows by at least a quarter (within the key bound), so keys that arrive in + * increasing order reallocate a logarithmic number of times. */ static __attribute__((noinline)) bool scr_map_grow_dense(ScrMap *m, double k, size_t *out) { - size_t i; - if (!scr_map_dense_slot(k, scr_map_dense_limit(m->nlive + 1), &i)) return false; - size_t old = m->ndense, n = scr_map_dense_size(i); + size_t i, limit = scr_map_dense_limit(m->nlive + 1); + if (!scr_map_dense_slot(k, limit, &i)) return false; + size_t old = m->ndense, step = old + old / 4; + size_t n = scr_map_dense_size((step < limit ? step : limit) - 1); + if (n <= i) n = scr_map_dense_size(i); scr_map_dense_alloc(m, n); memset(m->dense + old, 0, (n - old) * sizeof *m->dense); *out = i; @@ -578,13 +660,14 @@ static void scr_map_compact(ScrMap *m, bool reindex) { uint64_t filter = 0; bool dense = m->ndense != 0; for (size_t r = 0; r < m->nentries; r++) { - if (!m->entries[r].hash) continue; + if (!scr_map_live_at(m, r)) continue; if (dense) { - size_t k = (size_t)scr_map_slot_to_f64(m->entries[r].key); + size_t k = (size_t)scr_map_slot_to_f64(scr_map_at(m, r)->key); if (k > top) top = k; } - filter |= scr_map_filter_bits(m->entries[r].hash); - m->entries[w++] = m->entries[r]; + filter |= scr_map_filter_bits(scr_map_hash_at(m, r)); + if (w != r) memcpy(scr_map_at(m, w), scr_map_at(m, r), scr_map_entry_size(m)); + w++; } m->nentries = (uint32_t)w; m->filter = filter; @@ -625,7 +708,7 @@ static void scr_map_grow_entries(ScrMap *m) { size_t cap = m->ecap == 0 ? 1 : m->ecap < 4 ? 4 : m->ecap; while (cap < m->nentries + 1) cap *= 2; if (cap > SCR_MAP_MAX_ENTRIES) scr_map_oom(); - ScrMapEntry *entries = scr_mem_realloc(m->entries, cap * sizeof *entries); + ScrMapEntry *entries = scr_mem_realloc(m->entries, cap * scr_map_entry_size(m)); if (!entries) scr_map_oom(); m->entries = entries; m->ecap = (uint32_t)cap; @@ -686,9 +769,9 @@ static void scr_map_trace(void *o, ScrTraceVisit visit, void *ctx) { ScrMap *m = (ScrMap *)o; bool keys = m->traced & SCR_MAP_TRACED_KEY, vals = m->traced & SCR_MAP_TRACED_VAL; for (size_t e = 0; e < m->nentries; e++) { - if (!m->entries[e].hash) continue; - if (keys) visit(scr_map_slot_to_ptr(m->entries[e].key), ctx); - if (vals) visit(scr_map_slot_to_ptr(m->entries[e].val), ctx); + if (!scr_map_live_at(m, e)) continue; + if (keys) visit(scr_map_slot_to_ptr(scr_map_at(m, e)->key), ctx); + if (vals) visit(scr_map_slot_to_ptr(scr_map_at(m, e)->val), ctx); } } @@ -699,9 +782,9 @@ static void scr_map_gcfree(void *o) { bool keys = m->traced & SCR_MAP_TRACED_KEY, vals = m->traced & SCR_MAP_TRACED_VAL; scr_weak_dispose_inline(m); for (size_t e = 0; e < m->nentries; e++) { - if (!m->entries[e].hash) continue; - if (!keys) scr_map_release_key(m, m->entries[e].key); - if (!vals) scr_map_release_val(m, m->entries[e].val); + if (!scr_map_live_at(m, e)) continue; + if (!keys) scr_map_release_key(m, scr_map_at(m, e)->key); + if (!vals) scr_map_release_val(m, scr_map_at(m, e)->val); } scr_mem_free(m->entries); scr_mem_free(m->buckets); @@ -754,9 +837,9 @@ static void scr_map_destroy(void *object) { ScrMap *m = object; scr_weak_dispose_inline(m); for (size_t e = 0; e < m->nentries; e++) { - if (!m->entries[e].hash) continue; - scr_map_release_key(m, m->entries[e].key); - scr_map_release_val(m, m->entries[e].val); + if (!scr_map_live_at(m, e)) continue; + scr_map_release_key(m, scr_map_at(m, e)->key); + scr_map_release_val(m, scr_map_at(m, e)->val); } scr_mem_free(m->entries); scr_mem_free(m->buckets); @@ -806,11 +889,12 @@ void scr_map_clear(ScrMap *m) { if (walked == 0) return; uint32_t *dense = m->ndense ? m->dense : NULL; for (size_t e = 0; e < walked; e++) { - if (!m->entries[e].hash) continue; - m->entries[e].hash = 0; - if (dense) dense[(size_t)scr_map_slot_to_f64(m->entries[e].key)] = 0; - scr_map_release_key(m, m->entries[e].key); - scr_map_release_val(m, m->entries[e].val); + if (!scr_map_live_at(m, e)) continue; + uint64_t key = scr_map_at(m, e)->key, val = scr_map_at(m, e)->val; + scr_map_kill_at(m, e); + if (dense) dense[(size_t)scr_map_slot_to_f64(key)] = 0; + scr_map_release_key(m, key); + scr_map_release_val(m, val); } m->nlive = 0; m->filter = 0; @@ -864,8 +948,8 @@ bool scr_map_has_str(const ScrMap *m, const ScrStr *key) { static bool scr_map_delete_found(ScrMap *m, size_t e) { if (e == SCR_MAP_EMPTY) return false; - uint64_t key = m->entries[e].key, val = m->entries[e].val; - m->entries[e].hash = 0; /* bucket slot stays: probe chains intact */ + uint64_t key = scr_map_at(m, e)->key, val = scr_map_at(m, e)->val; + scr_map_kill_at(m, e); /* bucket slot stays: probe chains intact */ m->nlive--; if (m->ndense) m->dense[(size_t)scr_map_slot_to_f64(key)] = 0; /* Each such compaction follows deletes of three quarters of the entries, @@ -926,15 +1010,13 @@ static void scr_map_append(ScrMap *m, uint64_t hash, uint64_t key, uint64_t val, while (scr_map_bucket(m, slot) != SCR_MAP_EMPTY) slot = (slot + 1) & mask; } size_t idx = m->nentries++; - m->entries[idx].key = key; - m->entries[idx].val = val; - m->entries[idx].hash = hash; + scr_map_put_at(m, idx, key, val, hash); m->filter |= scr_map_filter_bits(hash); m->nlive++; if (m->key_kind == SCR_MAP_KEY_STR) { ScrStr *k = scr_map_slot_to_ptr(key); /* key is borrowed */ if (__builtin_expect(scr_str_is_scratch(k), 0)) - m->entries[idx].key = scr_map_slot_from_ptr(scr_str_scratch_keep(k)); + scr_map_at(m, idx)->key = scr_map_slot_from_ptr(scr_str_scratch_keep(k)); else scr_str_retain(k); } else if (m->key_kind == SCR_MAP_KEY_DYN) { @@ -942,7 +1024,7 @@ static void scr_map_append(ScrMap *m, uint64_t hash, uint64_t key, uint64_t val, if (d->kind == SCR_DYN_NUM && d->v.num == 0) { /* Iteration returns +0 even when the inserted key was -0. Never * mutate the caller's shared box while normalizing a stored key. */ - m->entries[idx].key = scr_map_slot_from_ptr(scr_dyn_new_num(0)); + scr_map_at(m, idx)->key = scr_map_slot_from_ptr(scr_dyn_new_num(0)); } else { m->ops->key_retain(scr_map_slot_to_ptr(key)); } @@ -951,7 +1033,7 @@ static void scr_map_append(ScrMap *m, uint64_t hash, uint64_t key, uint64_t val, ((ScrUnion *)scr_map_slot_to_ptr(key))->slot == UINT64_C(0x8000000000000000)) { /* Normalize stored -0 without mutating the caller's immutable box. */ ScrUnion *u = scr_map_slot_to_ptr(key); - m->entries[idx].key = scr_map_slot_from_ptr(scr_union_new_f64(u->tag, 0)); + scr_map_at(m, idx)->key = scr_map_slot_from_ptr(scr_union_new_f64(u->tag, 0)); } else if (scr_map_ref_key(m->key_kind)) { m->ops->key_retain(scr_map_slot_to_ptr(key)); /* key is borrowed */ } @@ -971,8 +1053,9 @@ static inline __attribute__((always_inline)) void scr_map_set_with( e = scr_map_bucket(m, slot); } if (e != SCR_MAP_EMPTY) { - uint64_t old = m->entries[e].val; - m->entries[e].val = val; /* unlink before releasing (cycle collector) */ + ScrMapEntry *x = scr_map_at_with(m, e, lookup); + uint64_t old = x->val; + x->val = val; /* unlink before releasing (cycle collector) */ scr_map_release_val(m, old); return; } @@ -983,13 +1066,18 @@ static void scr_map_set_str_slot(ScrMap *m, uint64_t hash, uint64_t key, uint64_ scr_map_set_with(m, hash, key, val, SCR_MAP_LOOKUP_STR); } -/* One out-of-line copy of the word-key insert serves both the generic - * dispatcher and the number-key path below. */ +/* Out-of-line copies of the word-key insert: the generic dispatcher's, and + * the hashed number-key path's below. */ static __attribute__((noinline)) void scr_map_set_word(ScrMap *m, uint64_t hash, uint64_t key, uint64_t val) { scr_map_set_with(m, hash, key, val, SCR_MAP_LOOKUP_WORD); } +static __attribute__((noinline)) void scr_map_set_num(ScrMap *m, uint64_t hash, uint64_t key, + uint64_t val) { + scr_map_set_with(m, hash, key, val, SCR_MAP_LOOKUP_NUM); +} + /* A number-key map with at least this many entries moves to the direct * index when its keys allow it, so small integer keys are indexed before * the linear scan ends. */ @@ -1021,8 +1109,8 @@ static void scr_map_set_f64_slot(ScrMap *m, uint64_t key, uint64_t val) { scr_map_dense_slot(k, scr_map_dense_limit(m->nlive + 1), &slot)) { size_t e = scr_map_find_f64(m, k); if (e != SCR_MAP_EMPTY) { - uint64_t old = m->entries[e].val; - m->entries[e].val = val; /* unlink before releasing (cycle collector) */ + uint64_t old = scr_map_at(m, e)->val; + scr_map_at(m, e)->val = val; /* unlink before releasing (cycle collector) */ scr_map_release_val(m, old); return; } @@ -1033,8 +1121,8 @@ static void scr_map_set_f64_slot(ScrMap *m, uint64_t key, uint64_t val) { if (scr_map_dense_slot(k, m->ndense, &i) || scr_map_grow_dense(m, k, &i)) { uint32_t entry = m->dense[i]; if (entry) { - uint64_t old = m->entries[entry - 1].val; - m->entries[entry - 1].val = val; /* unlink before releasing (cycle collector) */ + uint64_t old = scr_map_at(m, entry - 1)->val; + scr_map_at(m, entry - 1)->val = val; /* unlink before releasing (cycle collector) */ scr_map_release_val(m, old); return; } @@ -1042,10 +1130,9 @@ static void scr_map_set_f64_slot(ScrMap *m, uint64_t key, uint64_t val) { scr_map_reserve_append(m); /* may compact, which refills the table */ size_t idx = m->nentries++; uint64_t hash = scr_map_hash_word(key); - m->entries[idx].key = key; - m->entries[idx].val = val; - m->entries[idx].hash = hash ? hash : 1; - m->filter |= scr_map_filter_bits(m->entries[idx].hash); + hash = hash ? hash : 1; + scr_map_put_at(m, idx, key, val, hash); + m->filter |= scr_map_filter_bits(hash); m->nlive++; m->dense[i] = (uint32_t)(idx + 1); return; @@ -1053,7 +1140,7 @@ static void scr_map_set_f64_slot(ScrMap *m, uint64_t key, uint64_t val) { } scr_map_leave_dense(m); } - scr_map_set_word(m, scr_map_hash_word(key), key, val); + scr_map_set_num(m, scr_map_hash_word(key), key, val); } static void scr_map_set(ScrMap *m, uint64_t hash, uint64_t key, uint64_t val) { @@ -1064,6 +1151,7 @@ static void scr_map_set(ScrMap *m, uint64_t hash, uint64_t key, uint64_t val) { switch (scr_map_lookup_kind(m)) { case SCR_MAP_LOOKUP_STR: scr_map_set_str_slot(m, hash, key, val); return; case SCR_MAP_LOOKUP_WORD: scr_map_set_word(m, hash, key, val); return; + case SCR_MAP_LOOKUP_NUM: scr_map_set_num(m, hash, key, val); return; default: scr_map_set_with(m, hash, key, val, SCR_MAP_LOOKUP_GENERIC); return; } } @@ -1120,41 +1208,41 @@ void scr_map_set_ref_ref(ScrMap *m, void *key, void *v) { bool scr_map_get_f64_f64(const ScrMap *m, double key, double *out) { size_t e = scr_map_find_f64(m, key); if (e == SCR_MAP_EMPTY) return false; - *out = scr_map_slot_to_f64(m->entries[e].val); + *out = scr_map_slot_to_f64(scr_map_at(m, e)->val); return true; } bool scr_map_get_f64_bool(const ScrMap *m, double key, bool *out) { size_t e = scr_map_find_f64(m, key); if (e == SCR_MAP_EMPTY) return false; - *out = m->entries[e].val != 0; + *out = scr_map_at(m, e)->val != 0; return true; } void *scr_map_get_f64_ref(const ScrMap *m, double key) { size_t e = scr_map_find_f64(m, key); if (e == SCR_MAP_EMPTY) return NULL; - return m->ops->val_retain(scr_map_slot_to_ptr(m->entries[e].val)); /* +1 */ + return m->ops->val_retain(scr_map_slot_to_ptr(scr_map_at(m, e)->val)); /* +1 */ } bool scr_map_get_str_f64(const ScrMap *m, const ScrStr *key, double *out) { size_t e = scr_map_find_str(m, key); if (e == SCR_MAP_EMPTY) return false; - *out = scr_map_slot_to_f64(m->entries[e].val); + *out = scr_map_slot_to_f64(scr_map_at(m, e)->val); return true; } bool scr_map_get_str_bool(const ScrMap *m, const ScrStr *key, bool *out) { size_t e = scr_map_find_str(m, key); if (e == SCR_MAP_EMPTY) return false; - *out = m->entries[e].val != 0; + *out = scr_map_at(m, e)->val != 0; return true; } static __attribute__((noinline)) const ScrMapEntry *scr_map_entry_str_slow(const ScrMap *m, const ScrStr *key) { size_t e = scr_map_find_str(m, key); - return e == SCR_MAP_EMPTY ? NULL : &m->entries[e]; + return e == SCR_MAP_EMPTY ? NULL : scr_map_at(m, e); } /* A stored key with the probe's hash but another identity: compare bytes. @@ -1210,27 +1298,27 @@ const ScrMapEntry *scr_map_entry_str(const ScrMap *m, const ScrStr *key) { void *scr_map_get_str_ref(const ScrMap *m, const ScrStr *key) { size_t e = scr_map_find_str(m, key); if (e == SCR_MAP_EMPTY) return NULL; - return m->ops->val_retain(scr_map_slot_to_ptr(m->entries[e].val)); /* +1 */ + return m->ops->val_retain(scr_map_slot_to_ptr(scr_map_at(m, e)->val)); /* +1 */ } bool scr_map_get_ref_f64(const ScrMap *m, const void *key, double *out) { size_t e = scr_map_find_ref(m, key); if (e == SCR_MAP_EMPTY) return false; - *out = scr_map_slot_to_f64(m->entries[e].val); + *out = scr_map_slot_to_f64(scr_map_at(m, e)->val); return true; } bool scr_map_get_ref_bool(const ScrMap *m, const void *key, bool *out) { size_t e = scr_map_find_ref(m, key); if (e == SCR_MAP_EMPTY) return false; - *out = m->entries[e].val != 0; + *out = scr_map_at(m, e)->val != 0; return true; } void *scr_map_get_ref_ref(const ScrMap *m, const void *key) { size_t e = scr_map_find_ref(m, key); if (e == SCR_MAP_EMPTY) return NULL; - return m->ops->val_retain(scr_map_slot_to_ptr(m->entries[e].val)); + return m->ops->val_retain(scr_map_slot_to_ptr(scr_map_at(m, e)->val)); } /* Boolean keys use unboxed 0/1 slots and the same probe path. */ @@ -1263,21 +1351,21 @@ void scr_map_set_bool_ref(ScrMap *m, bool key, void *value) { bool scr_map_get_bool_f64(const ScrMap *m, bool key, double *out) { size_t entry = scr_map_find_bool(m, key); if (entry == SCR_MAP_EMPTY) return false; - *out = scr_map_slot_to_f64(m->entries[entry].val); + *out = scr_map_slot_to_f64(scr_map_at(m, entry)->val); return true; } bool scr_map_get_bool_bool(const ScrMap *m, bool key, bool *out) { size_t entry = scr_map_find_bool(m, key); if (entry == SCR_MAP_EMPTY) return false; - *out = m->entries[entry].val != 0; + *out = scr_map_at(m, entry)->val != 0; return true; } void *scr_map_get_bool_ref(const ScrMap *m, bool key) { size_t entry = scr_map_find_bool(m, key); if (entry == SCR_MAP_EMPTY) return NULL; - return m->ops->val_retain(scr_map_slot_to_ptr(m->entries[entry].val)); + return m->ops->val_retain(scr_map_slot_to_ptr(scr_map_at(m, entry)->val)); } /* ── iteration primitives (the forEach desugar) ────────────────────────── */ @@ -1286,7 +1374,7 @@ double scr_map_iter_count(const ScrMap *m) { return (double)m->nentries; } bool scr_map_iter_live(const ScrMap *m, double i) { if (!(i >= 0) || i >= (double)m->nentries) return false; - return m->entries[(size_t)i].hash; + return scr_map_live_at(m, (size_t)i); } /* A walk that starts at or below head moves head to the live entry it @@ -1297,7 +1385,7 @@ double scr_map_iter_skip(ScrMap *m, double i) { if (!(i < (double)m->nentries)) return i; bool from_head = !(i > (double)m->head); size_t e = from_head ? m->head : (size_t)i; - while (e < m->nentries && !m->entries[e].hash) e++; + while (e < m->nentries && !scr_map_live_at(m, e)) e++; if (from_head && m->rc != SIZE_MAX) m->head = (uint32_t)e; return e == (size_t)i ? i : (double)e; } @@ -1307,10 +1395,10 @@ double scr_map_iter_start(ScrMap *m) { } static const ScrMapEntry *scr_map_iter_at(const ScrMap *m, double i) { - if (!(i >= 0) || i >= (double)m->nentries || !m->entries[(size_t)i].hash) { + if (!(i >= 0) || i >= (double)m->nentries || !scr_map_live_at(m, (size_t)i)) { scr_trap("scriptc: internal error: map iteration index out of range\n"); } - return &m->entries[(size_t)i]; + return scr_map_at(m, (size_t)i); } double scr_map_iter_key_f64(const ScrMap *m, double i) { @@ -1390,20 +1478,21 @@ ScrMap *scr_map_clone(const ScrMap *source, bool keys_only, const ScrMapOps *ops ops ? ops : source->ops); size_t count = source->nlive; if (!count) return out; - if (count > SIZE_MAX / sizeof(ScrMapEntry) || count > SIZE_MAX / 2 / sizeof(size_t)) scr_map_oom(); - out->entries = scr_mem_alloc(count * sizeof(ScrMapEntry)); + size_t entry_size = scr_map_entry_size(out); + if (count > SIZE_MAX / entry_size || count > SIZE_MAX / 2 / sizeof(size_t)) scr_map_oom(); + out->entries = scr_mem_alloc(count * entry_size); if (!out->entries) scr_map_oom(); out->ecap = (uint32_t)count; size_t top = 0; for (size_t i = 0; i < source->nentries; i++) { - const ScrMapEntry *entry = &source->entries[i]; - if (!entry->hash) continue; + if (!scr_map_live_at(source, i)) continue; + const ScrMapEntry *entry = scr_map_at(source, i); if (source->ndense) { size_t k = (size_t)scr_map_slot_to_f64(entry->key); if (k > top) top = k; } - ScrMapEntry *copy = &out->entries[out->nentries++]; - *copy = *entry; + ScrMapEntry *copy = scr_map_at(out, out->nentries++); + memcpy(copy, entry, entry_size); if (out->key_kind == SCR_MAP_KEY_STR) scr_str_retain(scr_map_slot_to_ptr(copy->key)); else if (scr_map_ref_key(out->key_kind)) out->ops->key_retain(scr_map_slot_to_ptr(copy->key)); if (keys_only) copy->val = 0; @@ -1442,7 +1531,7 @@ ScrMap *scr_map_clone_dyn(const ScrMap *source, bool keys_only) { ScrMap *out = keys_only ? scr_map_new(SCR_MAP_KEY_DYN, SCR_MAP_VAL_F64, &scr_map_dyn_set_ops) : scr_map_new(SCR_MAP_KEY_DYN, SCR_MAP_VAL_REF, &scr_map_dyn_map_ops); for (size_t i = 0; i < source->nentries; i++) { - if (!source->entries[i].hash) continue; + if (!scr_map_live_at(source, i)) continue; ScrDyn *key = scr_map_dyn_key(source, (double)i); uint64_t value = keys_only ? 0 : scr_map_slot_from_ptr(scr_map_dyn_value(source, (double)i)); scr_map_set(out, scr_map_hash_dyn(key), scr_map_slot_from_ptr(key), value); @@ -1453,8 +1542,8 @@ ScrMap *scr_map_clone_dyn(const ScrMap *source, bool keys_only) { void scr_map_values_into(ScrMap *set, const ScrMap *source, bool checked) { for (size_t i = 0; i < source->nentries; i++) { - if (!source->entries[i].hash) continue; - uint64_t value = checked ? scr_map_slot_from_ptr(scr_map_dyn_value(source, (double)i)) : source->entries[i].val; + if (!scr_map_live_at(source, i)) continue; + uint64_t value = checked ? scr_map_slot_from_ptr(scr_map_dyn_value(source, (double)i)) : scr_map_at(source, i)->val; if (set->key_kind == SCR_MAP_KEY_F64) value = scr_map_f64_bits(scr_map_slot_to_f64(value)); scr_map_set(set, scr_map_hash_key(set, value), value, 0); if (checked) scr_dyn_release(scr_map_slot_to_ptr(value)); @@ -1542,8 +1631,8 @@ ScrArr *scr_map_keys_js_order(const ScrMap *m) { ScrKeyIndex *indices = NULL; size_t count = 0; for (size_t i = 0; i < m->nentries; i++) { - if (!m->entries[i].hash) continue; - ScrStr *key = scr_map_slot_to_ptr(m->entries[i].key); + if (!scr_map_live_at(m, i)) continue; + ScrStr *key = scr_map_slot_to_ptr(scr_map_at(m, i)->key); uint32_t index; if (!scr_map_key_array_index(key, &index)) continue; if (!indices) { @@ -1554,13 +1643,13 @@ ScrArr *scr_map_keys_js_order(const ScrMap *m) { } scr_key_index_sort(indices, count); for (size_t i = 0; i < count; i++) { - ScrStr *key = scr_map_slot_to_ptr(m->entries[indices[i].entry].key); + ScrStr *key = scr_map_slot_to_ptr(scr_map_at(m, indices[i].entry)->key); scr_arr_push_ref(out, scr_str_retain(key)); } free(indices); for (size_t i = 0; i < m->nentries; i++) { - if (!m->entries[i].hash) continue; - ScrStr *key = scr_map_slot_to_ptr(m->entries[i].key); + if (!scr_map_live_at(m, i)) continue; + ScrStr *key = scr_map_slot_to_ptr(scr_map_at(m, i)->key); uint32_t index; if (!scr_map_key_array_index(key, &index)) scr_arr_push_ref(out, scr_str_retain(key)); } diff --git a/packages/runtime/src/scr_map.h b/packages/runtime/src/scr_map.h index 9ac2b0977..af421299d 100644 --- a/packages/runtime/src/scr_map.h +++ b/packages/runtime/src/scr_map.h @@ -57,6 +57,14 @@ typedef struct { uint64_t val; /* double bits, bool, or owned pointer */ uint64_t hash; /* nonzero cached hash; zero = tombstone (key/val released) */ } ScrMapEntry; +/* A number-key map's (SCR_MAP_KEY_F64) entry: the key and the value, with + * no cached hash. A dead entry's key is SCR_MAP_NUM_DEAD, a NaN pattern no + * stored key has (stored keys use the one canonical NaN). */ +typedef struct { + uint64_t key; + uint64_t val; +} ScrMapNumEntry; +#define SCR_MAP_NUM_DEAD UINT64_C(0x7ff0000000000001) /* Checked views use the original storage. Slot converters borrow on reads * and return owned references on writes, just like typed Map operations. */ @@ -132,15 +140,27 @@ typedef struct ScrMap { * the whole deleted prefix again. */ uint32_t head; } ScrMap; +/* The entry at index i of either layout, and whether it is live (code that + * walks a map's entries outside scr_map.c). Key and value are at bytes 0 + * and 8 in both layouts; a number-key entry has no hash field. */ +static inline const ScrMapEntry *scr_map_entry_at(const ScrMap *m, size_t i) { + size_t size = m->key_kind == SCR_MAP_KEY_F64 ? sizeof(ScrMapNumEntry) : sizeof(ScrMapEntry); + return (const ScrMapEntry *)((const char *)m->entries + i * size); +} +static inline bool scr_map_entry_live(const ScrMap *m, size_t i) { + const ScrMapEntry *e = scr_map_entry_at(m, i); + return m->key_kind == SCR_MAP_KEY_F64 ? e->key != SCR_MAP_NUM_DEAD : e->hash != 0; +} /* The LLVM backend reads the header through two types (emitter.ts). Its * inline retain/release test the traced byte as %ScrMapRc field 3, as * scr_map_retain and scr_map_release do. The inline number-key lookup * (sc_map_entry_f64) reads entries, dense and ndense as %ScrMapIx fields - * 11, 15 and 16, and entries as %ScrMapEntry { key, val, hash }; its hashed + * 11, 15 and 16, and entries as %ScrMapNumEntry { key, val }; its hashed * probe (sc_map_entry_f64_hashed) also reads the entry count, the i64 * filter, nbuckets and buckets as fields 8, 10, 13 and 14, and the inline - * iteration steps (sc_map_iter_entry) read the entry count and test each - * entry's hash, the third field. */ + * iteration steps read the entry count and test each entry: its hash, the + * third field (sc_map_iter_entry), or for a number-key map its key against + * SCR_MAP_NUM_DEAD (sc_map_iter_entry_num). */ _Static_assert(offsetof(ScrMap, traced) == sizeof(size_t) + 2, "inline RC fast paths read ScrMap.traced as %ScrMapRc field 3"); _Static_assert(offsetof(ScrMap, flags) == sizeof(size_t) + 3, @@ -166,7 +186,9 @@ _Static_assert(offsetof(ScrMap, dense) == offsetof(ScrMap, buckets) + sizeof(voi "inline number-key lookups read ScrMap.dense and ndense as %ScrMapIx fields 15 and 16"); _Static_assert(offsetof(ScrMapEntry, hash) == 16, "inline iteration reads ScrMapEntry.hash at byte 16"); _Static_assert(sizeof(ScrMapEntry) == 24 && offsetof(ScrMapEntry, val) == 8, - "inline number-key lookups read ScrMapEntry.val at byte 8"); + "inline lookups read ScrMapEntry.val at byte 8"); +_Static_assert(sizeof(ScrMapNumEntry) == 16 && offsetof(ScrMapNumEntry, val) == 8, + "inline number-key lookups and iteration read %ScrMapNumEntry { key, val }"); _Static_assert(sizeof(ScrMap) <= 11 * sizeof(uint64_t), "the ScrMap header stays within 88 bytes"); /* Entry for a number key, or NULL when absent. The pointer is valid until * the next mutation; generated code reads the value slot through it. */ diff --git a/packages/runtime/src/scr_map.test.c b/packages/runtime/src/scr_map.test.c index 144796615..d6f1163f4 100644 --- a/packages/runtime/src/scr_map.test.c +++ b/packages/runtime/src/scr_map.test.c @@ -189,7 +189,7 @@ static void modes(void) { scr_map_set_f64_f64(map, 5, 5); assert(map->ndense >= 16 && !map->nbuckets); for (int i = 6; i < 1000; i++) scr_map_set_f64_f64(map, i, i); - assert(map->ndense >= 1000 && map->ndense <= 2048); + assert(map->ndense >= 1000 && map->ndense <= 1000 + 1000 / 4 + 64); /* near the largest key */ scr_map_set_f64_f64(map, -0.0, 0); /* slot 0 */ assert(map->ndense); double out; @@ -412,7 +412,8 @@ static void delete_shrinks(void) { map = scr_map_new(SCR_MAP_KEY_F64, SCR_MAP_VAL_F64, NULL); for (size_t i = 0; i < 4096; i++) scr_map_set_f64_f64(map, (double)i, (double)i); for (size_t i = 1024; i < 4096; i++) assert(scr_map_delete_f64(map, (double)i)); - assert(map->ndense == 4096 && scr_map_iter_count(map) == 4096); + assert(map->ndense >= 4096 && map->ndense <= 4096 + 4096 / 4 + 64); + assert(scr_map_iter_count(map) == 4096); ScrMap *copy = scr_map_clone(map, false, NULL); assert(copy->ndense == 1024 && scr_map_size(copy) == 1024); double out; diff --git a/packages/runtime/src/scr_publish.c b/packages/runtime/src/scr_publish.c index 5e8a9b3be..0cbb02c8f 100644 --- a/packages/runtime/src/scr_publish.c +++ b/packages/runtime/src/scr_publish.c @@ -229,8 +229,8 @@ void scr_pub_map(void *v, ScrPub *ctx, ScrPubFn key, ScrPubFn val) { : NULL; ScrPubFn vfn = m->val_kind == SCR_MAP_VAL_REF ? val : NULL; for (size_t i = 0; i < m->nentries; i++) { - const ScrMapEntry *e = &m->entries[i]; - if (!e->hash) continue; /* tombstone */ + if (!scr_map_entry_live(m, i)) continue; /* tombstone */ + const ScrMapEntry *e = scr_map_entry_at(m, i); if (kfn) scr_pub_slot(ctx, e->key, kfn); if (vfn) scr_pub_slot(ctx, e->val, vfn); } diff --git a/packages/runtime/src/scr_region.c b/packages/runtime/src/scr_region.c index 455b23dbb..c9d6c5511 100644 --- a/packages/runtime/src/scr_region.c +++ b/packages/runtime/src/scr_region.c @@ -827,8 +827,8 @@ static void scr_rgn_walk_map(ScrRgnWalk *w, ScrMap *m) { : (ScrRgnSlotKind){SCR_RGN_S_NONE, NULL}; ScrRgnSlotKind vk = valref ? scr_rgn_slot_kind(m->ops->val_release) : (ScrRgnSlotKind){SCR_RGN_S_NONE, NULL}; for (size_t i = 0; i < m->nentries; i++) { - const ScrMapEntry *e = &m->entries[i]; - if (!e->hash) continue; /* tombstone */ + if (!scr_map_entry_live(m, i)) continue; /* tombstone */ + const ScrMapEntry *e = scr_map_entry_at(m, i); scr_rgn_slot_as(w, scr_rgn_slot_ptr(e->key), kk); scr_rgn_slot_as(w, scr_rgn_slot_ptr(e->val), vk); } diff --git a/packages/runtime/test/test_map.c b/packages/runtime/test/test_map.c index 8899974a8..9957a994c 100644 --- a/packages/runtime/test/test_map.c +++ b/packages/runtime/test/test_map.c @@ -685,9 +685,19 @@ static uint64_t filter_bits(uint64_t hash) { return (UINT64_C(1) << (hash >> 44 & 63)) | (UINT64_C(1) << (hash >> 50 & 63)); } +/* A live entry's hash: cached, or for a number-key entry (no hash field) + * the key word's mix, as scr_map.c computes it. */ +static uint64_t entry_hash(const ScrMap *m, size_t e) { + if (!scr_map_entry_live(m, e)) return 0; + const ScrMapEntry *entry = scr_map_entry_at(m, e); + if (m->key_kind != SCR_MAP_KEY_F64) return entry->hash; + uint64_t h = scr_key_mix(entry->key); + return h ? h : 1; +} + static bool filter_covers_live(const ScrMap *m) { for (size_t e = 0; e < m->nentries; e++) { - uint64_t h = m->entries[e].hash; + uint64_t h = entry_hash(m, e); if (h && (m->filter & filter_bits(h)) != filter_bits(h)) return false; } return true;