diff --git a/docs/content/docs/contracts.mdx b/docs/content/docs/contracts.mdx index d3bd3c7f4..d5949c4cf 100644 --- a/docs/content/docs/contracts.mdx +++ b/docs/content/docs/contracts.mdx @@ -349,7 +349,7 @@ class Identifier extends SyntaxNode { } ``` -On Linux, except in builds with sanitizers, the instances also take less memory. An executable keeps them in one address range below 32 GiB that it reserves at startup (only the pages in use take memory), and a field typed `C`, `C | undefined` or `C | null`, where `C` is a program-lifetime class, stores the instance's address in 4 bytes instead of 8, as does each element of an array of such a type. In the example, `parent` takes 4 bytes. +On Linux, except in builds with sanitizers, the instances also take less memory. An executable keeps them in one address range below 32 GiB that it reserves at startup (only the pages in use take memory), and a field typed `C`, `C | undefined` or `C | null`, where `C` is a program-lifetime class, stores the instance's address in 4 bytes instead of 8, as does each element of an array of such a type and each value of a `Map` with `string` keys and such a value type. In the example, `parent` takes 4 bytes. scriptc checks that the promise cannot change what the program computes (SC7008): diff --git a/packages/compiler/src/backend/llvm/emit-module.ts b/packages/compiler/src/backend/llvm/emit-module.ts index a87903f3a..dc881a995 100644 --- a/packages/compiler/src/backend/llvm/emit-module.ts +++ b/packages/compiler/src/backend/llvm/emit-module.ts @@ -38,6 +38,7 @@ import { MAP_ITER_NUM_DECL, SCR_MAP_NUM_DEAD, NUMBER_MAP_ENTRY_DECL, + STRING_MAP_ENTRY_C_DECL, STRING_MAP_ENTRY_DECL, } from "./map-number-lookup.js"; import { RUNTIME_ABI_MARKER } from "../runtime-abi.js"; @@ -463,6 +464,7 @@ export function emitModuleParts(emitter: LlEmitter): string[] { // length fields 15 and 16. ...(emitter.decls.has(NUMBER_MAP_ENTRY_DECL) || emitter.decls.has(STRING_MAP_ENTRY_DECL) || + emitter.decls.has(STRING_MAP_ENTRY_C_DECL) || emitter.decls.has(MAP_ITER_DECL) || emitter.decls.has(MAP_ITER_NUM_DECL) ? [ @@ -479,6 +481,11 @@ export function emitModuleParts(emitter: LlEmitter): string[] { emitter.decls.has(MAP_ITER_NUM_DECL) ? [`%ScrMapEntry = type { i64, i64, i64 }`, `%ScrMapNumEntry = type { i64, i64 }`] : []), + // A compact-value map's { key, hash, val } entry (scr_map.h ScrMapCEntry): + // a 32-bit hash and the value as an address over 8. + ...(emitter.decls.has(STRING_MAP_ENTRY_C_DECL) + ? [`%ScrMapCEntry = type { i64, i32, i32 }`] + : []), // The runtime error prefix { rc, vt, name, message, code, cause } and the // class-object shape { rc, pre, post, ctor, name } — field reads on // builtin errors and classval loads GEP through these. @@ -1301,36 +1308,69 @@ export function helperDefs(emitter: LlEmitter): string[] { // and the filter reads its bits 44..55, which are the capacity word's // own. A key without a cached hash, and every possible hit, take the // runtime lookup. - if (emitter.decls.has(STRING_MAP_ENTRY_DECL)) { + // A compact-value map's lookup (sc_map_entry_str_c) tests the same filter + // and calls scr_map_entry_str_c; its iteration step (sc_map_iter_entry_c) + // reads 16-byte entries and their 32-bit hash. + const stringMapEntry = (helper: string, runtime: string): string[] => [ + `define internal ptr @${helper}(ptr %m, ptr %k) ${FN_ATTRS} {`, + `entry:`, + ...(emitter.sizeType === "i64" + ? [ + ` %fp = getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 10`, + ` %filter = load i64, ptr %fp`, + ` %capp = getelementptr inbounds %ScrStr, ptr %k, i32 0, i32 2`, + ` %cap = load i64, ptr %capp`, + ` %cached = icmp uge i64 %cap, 4294967296`, + ` br i1 %cached, label %test, label %slow`, + `test:`, + ` %s1 = lshr i64 %cap, 44`, + ` %i1 = and i64 %s1, 63`, + ` %b1 = shl i64 1, %i1`, + ` %s2 = lshr i64 %cap, 50`, + ` %i2 = and i64 %s2, 63`, + ` %b2 = shl i64 1, %i2`, + ` %bits = or i64 %b1, %b2`, + ` %have = and i64 %filter, %bits`, + ` %maybe = icmp eq i64 %have, %bits`, + ` br i1 %maybe, label %slow, label %absent`, + `absent:`, + ` ret ptr null`, + ] + : [` br label %slow`]), + `slow:`, + ` %r = call ptr @${runtime}(ptr %m, ptr %k)`, + ` ret ptr %r`, + `}`, + ``, + ]; + if (emitter.decls.has(STRING_MAP_ENTRY_DECL)) + defs.push(...stringMapEntry("sc_map_entry_str", "scr_map_entry_str")); + if (emitter.decls.has(STRING_MAP_ENTRY_C_DECL)) { + const sz = emitter.sizeType; defs.push( - `define internal ptr @sc_map_entry_str(ptr %m, ptr %k) ${FN_ATTRS} {`, + ...stringMapEntry("sc_map_entry_str_c", "scr_map_entry_str_c"), + `define internal ptr @sc_map_iter_entry_c(ptr %m, double %i) ${FN_ATTRS} {`, `entry:`, - ...(emitter.sizeType === "i64" - ? [ - ` %fp = getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 10`, - ` %filter = load i64, ptr %fp`, - ` %capp = getelementptr inbounds %ScrStr, ptr %k, i32 0, i32 2`, - ` %cap = load i64, ptr %capp`, - ` %cached = icmp uge i64 %cap, 4294967296`, - ` br i1 %cached, label %test, label %slow`, - `test:`, - ` %s1 = lshr i64 %cap, 44`, - ` %i1 = and i64 %s1, 63`, - ` %b1 = shl i64 1, %i1`, - ` %s2 = lshr i64 %cap, 50`, - ` %i2 = and i64 %s2, 63`, - ` %b2 = shl i64 1, %i2`, - ` %bits = or i64 %b1, %b2`, - ` %have = and i64 %filter, %bits`, - ` %maybe = icmp eq i64 %have, %bits`, - ` br i1 %maybe, label %slow, label %absent`, - `absent:`, - ` ret ptr null`, - ] - : [` br label %slow`]), - `slow:`, - ` %r = call ptr @scr_map_entry_str(ptr %m, ptr %k)`, - ` ret ptr %r`, + ` %np = getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 8`, + ` %n = load i32, ptr %np`, + ` %nf = uitofp i32 %n to double`, + ` %lo = fcmp oge double %i, 0.0`, + ` %hi = fcmp olt double %i, %nf`, + ` %in = and i1 %lo, %hi`, + ` br i1 %in, label %load, label %none`, + `load:`, + ` %ix = fptoui double %i to ${sz}`, + ` %ep = getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 11`, + ` %entries = load ptr, ptr %ep`, + ` %e = getelementptr inbounds %ScrMapCEntry, ptr %entries, ${sz} %ix`, + ` %hp = getelementptr inbounds %ScrMapCEntry, ptr %e, i32 0, i32 1`, + ` %h = load i32, ptr %hp`, + ` %live = icmp ne i32 %h, 0`, + ` br i1 %live, label %yes, label %none`, + `yes:`, + ` ret ptr %e`, + `none:`, + ` ret ptr null`, `}`, ``, ); diff --git a/packages/compiler/src/backend/llvm/expr-containers.ts b/packages/compiler/src/backend/llvm/expr-containers.ts index 4ddb5e75d..ff50887b7 100644 --- a/packages/compiler/src/backend/llvm/expr-containers.ts +++ b/packages/compiler/src/backend/llvm/expr-containers.ts @@ -31,6 +31,7 @@ import { emitMapIterCount, emitMapIterEntry, emitMapIterSlot, + compactMapValues, mapEntryLayout, emitMapIterStart, emitNumberMapGetRef, @@ -1065,8 +1066,10 @@ function newCollection(host: LlvmEmitterContext, key: IrType, value: IrType): st const kind = mapKeyKindNum(key, arms); const m = B.tmp(); host.declare(`declare ptr @scr_map_new(i32, i32, ptr)`); + // SCR_MAP_VAL_CREF (3): string keys and compact values (compactMapValues). + const valKind = compactMapValues(host, key, value) ? 3 : mapValKindNum(value); B.line( - `${m} = call ptr @scr_map_new(i32 ${kind}, i32 ${mapValKindNum(value)}, ptr ${collectionOps(host, key, value)})`, + `${m} = call ptr @scr_map_new(i32 ${kind}, i32 ${valKind}, ptr ${collectionOps(host, key, value)})`, ); return m; } @@ -1129,6 +1132,9 @@ export function emitMapLikeIntrinsic( const key = receiverType.kind === "map" ? receiverType.key : receiverType.elem; const kAcc = mapKeyAccess(key); const kTy = mapKeyParamType(kAcc); + // The map keeps compact values in 16-byte entries (SCR_MAP_VAL_CREF). + const compact = receiverType.kind === "map" && compactMapValues(host, key, receiverType.value); + const layout = mapEntryLayout(key, compact); const method = e.method; // A published Map or Set rejects mutation (the guard precedes the // operands, which each method evaluates itself). @@ -1292,7 +1298,7 @@ export function emitMapLikeIntrinsic( if (numberKey !== null) { raw = emitNumberMapGetRef(host, r.name, numberKey, value); } else if (stringKey !== null) { - raw = emitStringMapGetRef(host, r.name, stringKey, value); + raw = emitStringMapGetRef(host, r.name, stringKey, value, compact); } else { host.declare(`declare ptr @scr_map_get_${kAcc}_ref(ptr, ${kTy})`); raw = B.tmp(); @@ -1370,7 +1376,7 @@ export function emitMapLikeIntrinsic( return host.wrapNullable(raw, raw, value, valueTag, e.type, undefTag); } if (stringKey !== null && isRefCounted(value)) { - const raw = emitStringMapGetRef(host, r.name, stringKey, value); + const raw = emitStringMapGetRef(host, r.name, stringKey, value, compact); return host.wrapNullable(raw, raw, value, valueTag, e.type, undefTag); } host.declare(`declare ptr @scr_map_get_${kAcc}_ref(ptr, ${kTy})`); @@ -1418,7 +1424,8 @@ export function emitMapLikeIntrinsic( case "has": { const k = borrowInputs ? host.emitReadReceiver(e.args[0]!) : host.emitExpr(e.args[0]!); if (kAcc === "f64") return { name: emitNumberMapHas(host, r.name, k.name), type: e.type }; - if (kAcc === "str") return { name: emitStringMapHas(host, r.name, k.name), type: e.type }; + if (kAcc === "str") + return { name: emitStringMapHas(host, r.name, k.name, compact), type: e.type }; host.declare(`declare zeroext i1 @scr_map_has_${kAcc}(ptr, ${kTy})`); const t = B.tmp(); B.line(`${t} = call zeroext i1 @scr_map_has_${kAcc}(ptr ${r.name}, ${kTy} ${k.name})`); @@ -1447,7 +1454,7 @@ export function emitMapLikeIntrinsic( return { name: emitMapIterStart(host, r.name), type: e.type }; case "iterLive": { const i = host.emitExpr(e.args[0]!); - const entry = emitMapIterEntry(host, r.name, i.name, mapEntryLayout(key)); + const entry = emitMapIterEntry(host, r.name, i.name, layout); const t = B.tmp(); B.line(`${t} = icmp ne ptr ${entry}, null`); return { name: t, type: e.type }; @@ -1466,7 +1473,7 @@ export function emitMapLikeIntrinsic( kAcc === "str" ? "ref" : kAcc, key, { decl: `declare ${retTy} @${symbol}(ptr, double)`, symbol }, - mapEntryLayout(key), + layout, ); return host.own({ name: t, type: e.type }); } @@ -1493,7 +1500,7 @@ export function emitMapLikeIntrinsic( decl: `declare ${vAcc === "bool" ? "zeroext i1" : retTy} @${symbol}(ptr, double)`, symbol, }, - mapEntryLayout(key), + layout, ); return host.own({ name: t, type: e.type }); } diff --git a/packages/compiler/src/backend/llvm/map-number-lookup.ts b/packages/compiler/src/backend/llvm/map-number-lookup.ts index 1244bd9c3..f483cc569 100644 --- a/packages/compiler/src/backend/llvm/map-number-lookup.ts +++ b/packages/compiler/src/backend/llvm/map-number-lookup.ts @@ -15,6 +15,12 @@ * retain inline rather than through the runtime's indirect adapter call, * and scalar reads and has() share the one lookup. * + * A string-key map whose values are compact class instances + * (SCR_MAP_VAL_CREF, LlEmitter.compactArrayElem of the value type) keeps + * 16-byte { key, hash, val } entries with the value as an address over 8: + * its reads go through sc_map_entry_str_c and sc_map_iter_entry_c, and + * decode the 4-byte value. + * * The iteration steps behind for-of, forEach and the other entry walks read * the entries array through the internal sc_map_iter_entry and * sc_map_iter_count helpers: a live entry's key or value loads in place, @@ -24,6 +30,7 @@ */ import type { IrType } from "../../ir/ir.js"; import type { LlvmEmitterContext } from "./expr-context.js"; +import { mapKeyAccess } from "./shapes.js"; export const NUMBER_MAP_ENTRY_DECL = "declare ptr @scr_map_entry_f64(ptr, double)"; @@ -47,18 +54,59 @@ export function emitNumberMapSlot(host: LlvmEmitterContext, entry: string, type: } export const STRING_MAP_ENTRY_DECL = "declare ptr @scr_map_entry_str(ptr, ptr)"; +/** Declared by reads of a compact-value map; its presence makes the emitter + * define sc_map_entry_str_c and sc_map_iter_entry_c (emit-module.ts). */ +export const STRING_MAP_ENTRY_C_DECL = "declare ptr @scr_map_entry_str_c(ptr, ptr)"; -/** The entry for a string `key`, or null when absent. Valid until the next mutation. */ -export function emitStringMapEntry(host: LlvmEmitterContext, map: string, key: string): string { - host.declare(STRING_MAP_ENTRY_DECL); +/** Whether a Map with key type `key` and value type `value` keeps compact + * values (SCR_MAP_VAL_CREF): string keys and a value type whose instances + * are compact (LlEmitter.compactArrayElem). */ +export function compactMapValues(host: LlvmEmitterContext, key: IrType, value: IrType): boolean { + // SCR_MAP_KEY_STR (shapes.ts mapKeyKindNum 1): string access, not a union. + return key.kind !== "union" && mapKeyAccess(key) === "str" && host.compactArrayElem(value); +} + +/** The entry for a string `key`, or null when absent. Valid until the next + * mutation. `compact`: the map keeps compact values (a %ScrMapCEntry). */ +export function emitStringMapEntry( + host: LlvmEmitterContext, + map: string, + key: string, + compact = false, +): string { + host.declare(compact ? STRING_MAP_ENTRY_C_DECL : STRING_MAP_ENTRY_DECL); const entry = host.B.tmp(); - host.B.line(`${entry} = call ptr @sc_map_entry_str(ptr ${map}, ptr ${key})`); + host.B.line( + `${entry} = call ptr @${compact ? "sc_map_entry_str_c" : "sc_map_entry_str"}(ptr ${map}, ptr ${key})`, + ); return entry; } +/** A found compact-value entry's value as a pointer: its 4-byte form, an + * address over 8 (scr_map.h ScrMapCEntry). */ +export function emitCompactMapSlot(host: LlvmEmitterContext, entry: string): string { + const B = host.B; + const slot = B.tmp(), + ref = B.tmp(), + wide = B.tmp(), + addr = B.tmp(), + value = B.tmp(); + B.line(`${slot} = getelementptr inbounds %ScrMapCEntry, ptr ${entry}, i32 0, i32 2`); + B.line(`${ref} = load i32, ptr ${slot}`); + B.line(`${wide} = zext i32 ${ref} to i64`); + B.line(`${addr} = shl nuw nsw i64 ${wide}, 3`); + B.line(`${value} = inttoptr i64 ${addr} to ptr`); + return value; +} + /** Map.has / Set.has under a string key. */ -export function emitStringMapHas(host: LlvmEmitterContext, map: string, key: string): string { - const entry = emitStringMapEntry(host, map, key); +export function emitStringMapHas( + host: LlvmEmitterContext, + map: string, + key: string, + compact = false, +): string { + const entry = emitStringMapEntry(host, map, key, compact); const found = host.B.tmp(); host.B.line(`${found} = icmp ne ptr ${entry}, null`); return found; @@ -92,12 +140,18 @@ export function emitStringMapGetRef( map: string, key: string, value: IrType, + compact = false, ): string { - return emitMapEntryGetRef(host, emitStringMapEntry(host, map, key), value); + return emitMapEntryGetRef(host, emitStringMapEntry(host, map, key, compact), value, compact); } /** The +1 value of a found entry, or null for a missing (null) entry. */ -function emitMapEntryGetRef(host: LlvmEmitterContext, entry: string, value: IrType): string { +function emitMapEntryGetRef( + host: LlvmEmitterContext, + entry: string, + value: IrType, + compact = false, +): string { const B = host.B; const found = B.tmp(); B.line(`${found} = icmp ne ptr ${entry}, null`); @@ -106,7 +160,10 @@ function emitMapEntryGetRef(host: LlvmEmitterContext, entry: string, value: IrTy join = B.newLabel("mapn.join"); B.condBr(found, hit, miss); B.startBlock(hit); - const retained = host.retainFrame(emitNumberMapSlot(host, entry, "ptr"), value); + const retained = host.retainFrame( + compact ? emitCompactMapSlot(host, entry) : emitNumberMapSlot(host, entry, "ptr"), + value, + ); B.br(join); B.startBlock(miss); B.br(join); @@ -199,20 +256,25 @@ export const SCR_MAP_NUM_DEAD = "9218868437227405313"; /** The entry layout a Map or Set of key type `key` stores: * - "num": number keys, whose entries are { key, val } (scr_map.h ScrMapNumEntry); + * - "centry": string keys with compact values (compactMapValues), whose + * entries are { key, hash, val } (ScrMapCEntry); * - "entry": every other typed key, whose entries are { key, val, hash }; * - "either": checked (`unknown`) keys. A `Map` or * `Set` cast from a checked value keeps the collection's own - * storage (scr_dyn_native_collection_check), which can have number keys. */ -export type MapEntryLayout = "num" | "entry" | "either"; + * storage (scr_dyn_native_collection_check), which can have number keys + * or compact values. */ +export type MapEntryLayout = "num" | "centry" | "entry" | "either"; -export function mapEntryLayout(key: IrType): MapEntryLayout { +export function mapEntryLayout(key: IrType, compact = false): MapEntryLayout { + if (compact) return "centry"; return key.kind === "f64" ? "num" : key.kind === "dyn" ? "either" : "entry"; } /** The entry at an iteration index, or null for a tombstone or an index at * or past the entry count (the scr_map_iter_live test). With the "either" * layout, the map's key kind (%ScrMapIx field 1, SCR_MAP_KEY_F64 = 0) picks - * the step at run time. */ + * the step at run time; in a program with compact class instances, so does + * its value kind (field 2, SCR_MAP_VAL_CREF = 3). */ export function emitMapIterEntry( host: LlvmEmitterContext, map: string, @@ -220,14 +282,20 @@ export function emitMapIterEntry( layout: MapEntryLayout, ): string { const B = host.B; - const step = (num: boolean): string => { - host.declare(num ? MAP_ITER_NUM_DECL : MAP_ITER_DECL); + const step = (kind: "num" | "centry" | "entry"): string => { + host.declare(kind === "num" ? MAP_ITER_NUM_DECL : MAP_ITER_DECL); + if (kind === "centry") host.declare(STRING_MAP_ENTRY_C_DECL); const entry = B.tmp(); - const helper = num ? "sc_map_iter_entry_num" : "sc_map_iter_entry"; + const helper = + kind === "num" + ? "sc_map_iter_entry_num" + : kind === "centry" + ? "sc_map_iter_entry_c" + : "sc_map_iter_entry"; B.line(`${entry} = call ptr @${helper}(ptr ${map}, double ${index})`); return entry; }; - if (layout !== "either") return step(layout === "num"); + if (layout !== "either") return step(layout); host.declare(MAP_ITER_DECL); const kindPtr = B.tmp(), kind = B.tmp(), @@ -240,16 +308,31 @@ export function emitMapIterEntry( join = B.newLabel("mapit.layout"); B.condBr(isNum, numLabel, entryLabel); B.startBlock(numLabel); - const numEntry = step(true); + const numEntry = step("num"); const numEnd = B.currentLabel(); B.br(join); B.startBlock(entryLabel); - const otherEntry = step(false); - const entryEnd = B.currentLabel(); + const arms: { value: string; label: string }[] = [{ value: numEntry, label: numEnd }]; + if (host.shapeHost.compactRegion?.() === true) { + const valPtr = B.tmp(), + valKind = B.tmp(), + isCompact = B.tmp(); + B.line(`${valPtr} = getelementptr inbounds %ScrMapIx, ptr ${map}, i32 0, i32 2`); + B.line(`${valKind} = load i8, ptr ${valPtr}`); + B.line(`${isCompact} = icmp eq i8 ${valKind}, 3`); + const compactLabel = B.newLabel("mapit.centry"), + wideLabel = B.newLabel("mapit.wide"); + B.condBr(isCompact, compactLabel, wideLabel); + B.startBlock(compactLabel); + arms.push({ value: step("centry"), label: B.currentLabel() }); + B.br(join); + B.startBlock(wideLabel); + } + arms.push({ value: step("entry"), label: B.currentLabel() }); B.br(join); B.startBlock(join); const entry = B.tmp(); - B.line(`${entry} = phi ptr [ ${numEntry}, %${numEnd} ], [ ${otherEntry}, %${entryEnd} ]`); + B.line(`${entry} = phi ptr ${arms.map((a) => `[ ${a.value}, %${a.label} ]`).join(", ")}`); return entry; } @@ -294,18 +377,24 @@ export function emitMapIterSlot( join = B.newLabel("mapit.join"); B.condBr(found, hit, miss); B.startBlock(hit); - const slot = B.tmp(); - B.line(`${slot} = getelementptr inbounds %ScrMapEntry, ptr ${entry}, i32 0, i32 ${field}`); const llTy = access === "f64" ? "double" : access === "bool" ? "i1" : "ptr"; - let loaded = B.tmp(); - if (access === "bool") { - const bits = loaded; - B.line(`${bits} = load i64, ptr ${slot}`); - loaded = B.tmp(); - B.line(`${loaded} = icmp ne i64 ${bits}, 0`); + let loaded: string; + if (layout === "centry" && field === 1) { + // A compact value; its type's retain emits no counting. + loaded = host.retainFrame(emitCompactMapSlot(host, entry), type); } else { - B.line(`${loaded} = load ${llTy}, ptr ${slot}`); - if (access === "ref") loaded = host.retainFrame(loaded, type); + const slot = B.tmp(); + B.line(`${slot} = getelementptr inbounds %ScrMapEntry, ptr ${entry}, i32 0, i32 ${field}`); + loaded = B.tmp(); + if (access === "bool") { + const bits = loaded; + B.line(`${bits} = load i64, ptr ${slot}`); + loaded = B.tmp(); + B.line(`${loaded} = icmp ne i64 ${bits}, 0`); + } else { + B.line(`${loaded} = load ${llTy}, ptr ${slot}`); + if (access === "ref") loaded = host.retainFrame(loaded, type); + } } const hitEnd = B.currentLabel(); B.br(join); diff --git a/packages/compiler/src/backend/llvm/map-read-lifetimes.ts b/packages/compiler/src/backend/llvm/map-read-lifetimes.ts index 005f88f13..6b58d314a 100644 --- a/packages/compiler/src/backend/llvm/map-read-lifetimes.ts +++ b/packages/compiler/src/backend/llvm/map-read-lifetimes.ts @@ -15,6 +15,7 @@ import { emitNumberMapGetScalar, emitStringMapGetScalar, emitStringMapGetRef, + compactMapValues, } from "./map-number-lookup.js"; import { mapKeyAccess, mapKeyParamType } from "./shapes.js"; import { rememberMapEntry } from "./map-entry-fusion.js"; @@ -323,7 +324,10 @@ export function emitStackMapRead(host: LlvmEmitterContext, read: LocalMapRead): } else if (isRefCounted(read.value)) { const stringKey = entryStringKey(key); let raw: string; - if (stringKey !== null) raw = emitStringMapGetRef(host, receiver.name, stringKey, read.value); + const mapType = read.receiver.type; + const compact = mapType.kind === "map" && compactMapValues(host, mapType.key, mapType.value); + if (stringKey !== null) + raw = emitStringMapGetRef(host, receiver.name, stringKey, read.value, compact); else { raw = B.tmp(); host.declare(`declare ptr @scr_map_get_${access}_ref(ptr, ${keyType})`); diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/4639-contract-program-lifetime-maps.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/4639-contract-program-lifetime-maps.ts.json new file mode 100644 index 000000000..5f7afb8e7 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/4639-contract-program-lifetime-maps.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/4639-contract-program-lifetime-maps.ts" + ], + "diags": [] +} diff --git a/packages/runtime/src/scr_map.c b/packages/runtime/src/scr_map.c index 99caa31a8..648d1bfc8 100644 --- a/packages/runtime/src/scr_map.c +++ b/packages/runtime/src/scr_map.c @@ -220,47 +220,106 @@ static bool scr_map_key_eq(const ScrMap *m, uint64_t stored, uint64_t probe) { * the value: its hash is the key word's scr_key_mix, which is cheap to * recompute, and its stored keys are normalized (one NaN pattern, +0), so * a dead entry marks its key with a NaN pattern that no stored key has. + * A SCR_MAP_VAL_CREF map (string keys, compact class values) keeps 16-byte + * entries: the key, its 32-bit hash and the value as an address over 8. * Every other map keeps 24-byte entries with the cached hash, zero for a - * dead entry. Both layouts put the key at byte 0 and the value at byte 8. */ + * dead entry. Every layout puts the key at byte 0; the 8-byte layouts put + * the value at byte 8. */ static inline bool scr_map_num(const ScrMap *m) { return m->key_kind == SCR_MAP_KEY_F64; } +static inline bool scr_map_cref(const ScrMap *m) { + return __builtin_expect(m->val_kind == SCR_MAP_VAL_CREF, 0); +} + static inline size_t scr_map_entry_size(const ScrMap *m) { - return scr_map_num(m) ? sizeof(ScrMapNumEntry) : sizeof(ScrMapEntry); + if (scr_map_num(m)) return sizeof(ScrMapNumEntry); + if (scr_map_cref(m)) return sizeof(ScrMapCEntry); + return sizeof(ScrMapEntry); } /* The entry at index e. A number-key entry has no hash field: read its key - * and value only (scr_map_hash_at gives the hash). */ + * and value only (scr_map_hash_at gives the hash); a SCR_MAP_VAL_CREF + * entry's value is read with scr_map_val_at. */ static inline ScrMapEntry *scr_map_at(const ScrMap *m, size_t e) { return (ScrMapEntry *)((char *)m->entries + e * scr_map_entry_size(m)); } +static inline ScrMapCEntry *scr_map_c_at(const ScrMap *m, size_t e) { + return (ScrMapCEntry *)m->entries + e; +} + static inline bool scr_map_live_at(const ScrMap *m, size_t e) { - const ScrMapEntry *x = scr_map_at(m, e); - return scr_map_num(m) ? x->key != SCR_MAP_NUM_DEAD : x->hash != 0; + if (scr_map_num(m)) return ((const ScrMapNumEntry *)m->entries)[e].key != SCR_MAP_NUM_DEAD; + if (scr_map_cref(m)) return scr_map_c_at(m, e)->hash != 0; + return m->entries[e].hash != 0; } /* The entry's hash, nonzero when it is live and zero for a dead entry. */ static inline uint64_t scr_map_hash_at(const ScrMap *m, size_t e) { - const ScrMapEntry *x = scr_map_at(m, e); - if (!scr_map_num(m)) return x->hash; - if (x->key == SCR_MAP_NUM_DEAD) return 0; - uint64_t hash = scr_map_hash_word(x->key); - return hash ? hash : 1; + if (scr_map_num(m)) { + uint64_t key = ((const ScrMapNumEntry *)m->entries)[e].key; + if (key == SCR_MAP_NUM_DEAD) return 0; + uint64_t hash = scr_map_hash_word(key); + return hash ? hash : 1; + } + if (scr_map_cref(m)) { + uint32_t h = scr_map_c_at(m, e)->hash; + return h ? scr_map_hash_widen(h) : 0; + } + return m->entries[e].hash; } static inline void scr_map_kill_at(ScrMap *m, size_t e) { - ScrMapEntry *x = scr_map_at(m, e); - if (scr_map_num(m)) x->key = SCR_MAP_NUM_DEAD; - else x->hash = 0; + if (scr_map_num(m)) ((ScrMapNumEntry *)m->entries)[e].key = SCR_MAP_NUM_DEAD; + else if (scr_map_cref(m)) scr_map_c_at(m, e)->hash = 0; + else m->entries[e].hash = 0; } +/* A compact value's 4-byte form and back (scr_cref_encode in scr_array.h). */ +static inline uint32_t scr_map_cref_encode(uint64_t val) { return (uint32_t)(val >> 3); } +static inline uint64_t scr_map_cref_decode(uint32_t val) { return (uint64_t)val << 3; } + +/* The value of entry e as an 8-byte slot, and its store. */ +static inline uint64_t scr_map_val_at(const ScrMap *m, size_t e) { + if (scr_map_num(m)) return ((const ScrMapNumEntry *)m->entries)[e].val; + if (scr_map_cref(m)) return scr_map_cref_decode(scr_map_c_at(m, e)->val); + return m->entries[e].val; +} + +static inline void scr_map_val_put(ScrMap *m, size_t e, uint64_t val) { + if (scr_map_num(m)) ((ScrMapNumEntry *)m->entries)[e].val = val; + else if (scr_map_cref(m)) scr_map_c_at(m, e)->val = scr_map_cref_encode(val); + else m->entries[e].val = val; +} + +/* A reference value as a +1 result: compact values are never counted. */ +static inline void *scr_map_val_retain(const ScrMap *m, uint64_t val) { + if (scr_map_cref(m)) return scr_map_slot_to_ptr(val); + return m->ops->val_retain(scr_map_slot_to_ptr(val)); +} + +/* A SCR_MAP_VAL_CREF map's keys are strings, whose hashes are widened + * 32-bit values: the entry keeps the low half. */ static inline void scr_map_put_at(ScrMap *m, size_t e, uint64_t key, uint64_t val, uint64_t hash) { - ScrMapEntry *x = scr_map_at(m, e); + if (scr_map_num(m)) { + ScrMapNumEntry *x = (ScrMapNumEntry *)m->entries + e; + x->key = key; + x->val = val; + return; + } + if (scr_map_cref(m)) { + ScrMapCEntry *x = scr_map_c_at(m, e); + x->key = key; + x->hash = (uint32_t)hash; + x->val = scr_map_cref_encode(val); + return; + } + ScrMapEntry *x = m->entries + e; x->key = key; x->val = val; - if (!scr_map_num(m)) x->hash = hash; + x->hash = hash; } /* ── absent-key filter ───────────────────────────────────────────────── @@ -282,17 +341,25 @@ static inline uint64_t scr_map_filter_bits(uint64_t hash) { * before equality and survive table growth without hashing the keys again. * Remap zero to one so every live entry has a nonzero hash. */ /* SCR_MAP_LOOKUP_NUM is a word compare on a number-key map, so the probe - * reads 16-byte entries; every other lookup is on a map with 24-byte - * entries. Knowing the layout from the lookup keeps the entry stride a - * constant in the inlined probes. */ + * reads 16-byte entries; SCR_MAP_LOOKUP_STR_C and SPAN_C are the string and + * span probes of a SCR_MAP_VAL_CREF map, whose 16-byte entries keep a + * 32-bit hash; every other lookup is on a map with 24-byte entries. Knowing + * the layout from the lookup keeps the entry stride a constant in the + * inlined probes. */ typedef enum { SCR_MAP_LOOKUP_STR, SCR_MAP_LOOKUP_SPAN, SCR_MAP_LOOKUP_WORD, SCR_MAP_LOOKUP_NUM, - SCR_MAP_LOOKUP_GENERIC + SCR_MAP_LOOKUP_GENERIC, + SCR_MAP_LOOKUP_STR_C, + SCR_MAP_LOOKUP_SPAN_C } ScrMapLookup; +static inline bool scr_map_lookup_c(ScrMapLookup lookup) { + return lookup == SCR_MAP_LOOKUP_STR_C || lookup == SCR_MAP_LOOKUP_SPAN_C; +} + typedef struct { const char *bytes; size_t length; } ScrMapSpan; /* Choose equality once per operation. Inlining the traversal with a constant @@ -301,12 +368,12 @@ typedef struct { const char *bytes; size_t length; } ScrMapSpan; static inline bool scr_map_lookup_eq(const ScrMap *m, uint64_t stored, uint64_t key, ScrMapLookup lookup) { if (lookup == SCR_MAP_LOOKUP_WORD || lookup == SCR_MAP_LOOKUP_NUM) return stored == key; - if (lookup == SCR_MAP_LOOKUP_SPAN) { + if (lookup == SCR_MAP_LOOKUP_SPAN || lookup == SCR_MAP_LOOKUP_SPAN_C) { const ScrStr *a = scr_map_slot_to_ptr(stored); const ScrMapSpan *b = scr_map_slot_to_ptr(key); return a->len == b->length && scr_key_equal(a->data, b->bytes, a->len); } - if (lookup == SCR_MAP_LOOKUP_STR) { + if (lookup == SCR_MAP_LOOKUP_STR || lookup == SCR_MAP_LOOKUP_STR_C) { const ScrStr *a = scr_map_slot_to_ptr(stored); const ScrStr *b = scr_map_slot_to_ptr(key); return a == b || (a->len == b->len && scr_key_equal(a->data, b->data, a->len)); @@ -318,6 +385,7 @@ static inline bool scr_map_lookup_eq(const ScrMap *m, uint64_t stored, uint64_t static inline __attribute__((always_inline)) ScrMapEntry *scr_map_at_with(const ScrMap *m, size_t e, ScrMapLookup lookup) { if (lookup == SCR_MAP_LOOKUP_NUM) return (ScrMapEntry *)((ScrMapNumEntry *)m->entries + e); + if (scr_map_lookup_c(lookup)) return (ScrMapEntry *)((ScrMapCEntry *)m->entries + e); return &m->entries[e]; } @@ -328,6 +396,8 @@ static inline __attribute__((always_inline)) bool scr_map_match_at( const ScrMap *m, size_t e, uint64_t hash, uint64_t key, ScrMapLookup lookup) { const ScrMapEntry *x = scr_map_at_with(m, e, lookup); if (lookup == SCR_MAP_LOOKUP_NUM) return x->key == key; + if (scr_map_lookup_c(lookup)) + return ((const ScrMapCEntry *)x)->hash == (uint32_t)hash && scr_map_lookup_eq(m, x->key, key, lookup); return x->hash == hash && scr_map_lookup_eq(m, x->key, key, lookup); } @@ -437,7 +507,7 @@ static inline __attribute__((always_inline)) size_t scr_map_find_with( static ScrMapLookup scr_map_lookup_kind(const ScrMap *m) { switch (m->key_kind) { - case SCR_MAP_KEY_STR: return SCR_MAP_LOOKUP_STR; + case SCR_MAP_KEY_STR: return scr_map_cref(m) ? SCR_MAP_LOOKUP_STR_C : SCR_MAP_LOOKUP_STR; case SCR_MAP_KEY_F64: return SCR_MAP_LOOKUP_NUM; case SCR_MAP_KEY_BOOL: case SCR_MAP_KEY_REF: @@ -484,6 +554,7 @@ static size_t scr_map_find(const ScrMap *m, uint64_t hash, uint64_t key) { if (m->ndense) return scr_map_find_dense(m, scr_map_slot_to_f64(key)); switch (scr_map_lookup_kind(m)) { case SCR_MAP_LOOKUP_STR: return scr_map_find_with(m, hash, key, SCR_MAP_LOOKUP_STR); + case SCR_MAP_LOOKUP_STR_C: return scr_map_find_with(m, hash, key, SCR_MAP_LOOKUP_STR_C); case SCR_MAP_LOOKUP_WORD: return scr_map_find_with(m, hash, key, SCR_MAP_LOOKUP_WORD); case SCR_MAP_LOOKUP_NUM: return scr_map_find_with(m, hash, key, SCR_MAP_LOOKUP_NUM); default: return scr_map_find_with(m, hash, key, SCR_MAP_LOOKUP_GENERIC); @@ -504,9 +575,24 @@ const ScrMapEntry *scr_map_entry_f64(const ScrMap *m, double key) { /* The key's hash is usually cached, so even the small linear scan rejects * entries by hash before touching their bytes. */ +/* The probes of a SCR_MAP_VAL_CREF map, out of line so that the inlined + * probes of other maps keep their size. */ +static __attribute__((noinline)) size_t scr_map_find_str_c(const ScrMap *m, const ScrStr *key) { + return scr_map_find_with(m, scr_map_hash_str(key), scr_map_slot_from_ptr((void *)key), + SCR_MAP_LOOKUP_STR_C); +} + +static __attribute__((noinline)) size_t scr_map_find_span_c(const ScrMap *m, const char *bytes, + size_t length) { + ScrMapSpan key = { bytes, length }; + return scr_map_find_with(m, scr_map_hash_bytes(bytes, length), scr_map_slot_from_ptr(&key), + SCR_MAP_LOOKUP_SPAN_C); +} + static inline __attribute__((always_inline)) size_t scr_map_find_str(const ScrMap *m, const ScrStr *key) { if (m->nlive == 0) return SCR_MAP_EMPTY; + if (scr_map_cref(m)) return scr_map_find_str_c(m, key); return scr_map_find_with(m, scr_map_hash_str(key), scr_map_slot_from_ptr((void *)key), SCR_MAP_LOOKUP_STR); } @@ -515,6 +601,7 @@ static inline __attribute__((always_inline)) size_t scr_map_find_str(const ScrMa * rules as a string key without allocating or copying the probe. */ static size_t scr_map_find_span(const ScrMap *m, const char *bytes, size_t length) { if (m->nlive == 0) return SCR_MAP_EMPTY; + if (scr_map_cref(m)) return scr_map_find_span_c(m, bytes, length); ScrMapSpan key = { bytes, length }; return scr_map_find_with(m, scr_map_hash_bytes(bytes, length), scr_map_slot_from_ptr(&key), SCR_MAP_LOOKUP_SPAN); @@ -523,20 +610,20 @@ static size_t scr_map_find_span(const ScrMap *m, const char *bytes, size_t lengt bool scr_map_get_span_f64(const ScrMap *m, const char *bytes, size_t length, double *out) { size_t e = scr_map_find_span(m, bytes, length); if (e == SCR_MAP_EMPTY) return false; - *out = scr_map_slot_to_f64(scr_map_at(m, e)->val); + *out = scr_map_slot_to_f64(scr_map_val_at(m, e)); return true; } bool scr_map_get_span_bool(const ScrMap *m, const char *bytes, size_t length, bool *out) { size_t e = scr_map_find_span(m, bytes, length); if (e == SCR_MAP_EMPTY) return false; - *out = scr_map_at(m, e)->val != 0; + *out = scr_map_val_at(m, e) != 0; return true; } void *scr_map_get_span_ref(const ScrMap *m, const char *bytes, size_t length) { size_t e = scr_map_find_span(m, bytes, length); - return e == SCR_MAP_EMPTY ? NULL : m->ops->val_retain(scr_map_slot_to_ptr(scr_map_at(m, e)->val)); + return e == SCR_MAP_EMPTY ? NULL : scr_map_val_retain(m, scr_map_val_at(m, e)); } /* Rebuild the bucket table (size must be a power of two >= 2 * nentries): @@ -666,7 +753,12 @@ static void scr_map_compact(ScrMap *m, bool reindex) { if (k > top) top = k; } filter |= scr_map_filter_bits(scr_map_hash_at(m, r)); - if (w != r) memcpy(scr_map_at(m, w), scr_map_at(m, r), scr_map_entry_size(m)); + if (w != r) { + /* A constant size per layout keeps the copy inline. */ + if (scr_map_num(m)) memcpy(scr_map_at(m, w), scr_map_at(m, r), sizeof(ScrMapNumEntry)); + else if (scr_map_cref(m)) memcpy(scr_map_at(m, w), scr_map_at(m, r), sizeof(ScrMapCEntry)); + else memcpy(scr_map_at(m, w), scr_map_at(m, r), sizeof(ScrMapEntry)); + } w++; } m->nentries = (uint32_t)w; @@ -778,7 +870,7 @@ static void scr_map_trace(void *o, ScrTraceVisit visit, void *ctx) { for (size_t e = 0; e < m->nentries; e++) { if (!scr_map_live_at(m, e)) continue; if (keys) visit(scr_map_slot_to_ptr(scr_map_at(m, e)->key), ctx); - if (vals) visit(scr_map_slot_to_ptr(scr_map_at(m, e)->val), ctx); + if (vals) visit(scr_map_slot_to_ptr(scr_map_val_at(m, e)), ctx); } } @@ -791,7 +883,7 @@ static void scr_map_gcfree(void *o) { for (size_t e = 0; e < m->nentries; e++) { if (!scr_map_live_at(m, e)) continue; if (!keys) scr_map_release_key(m, scr_map_at(m, e)->key); - if (!vals) scr_map_release_val(m, scr_map_at(m, e)->val); + if (!vals) scr_map_release_val(m, scr_map_val_at(m, e)); } scr_mem_free(m->entries); scr_mem_free(m->buckets); @@ -843,10 +935,15 @@ ScrMap *scr_map_retain(ScrMap *m) { static void scr_map_destroy(void *object) { ScrMap *m = object; scr_weak_dispose_inline(m); - for (size_t e = 0; e < m->nentries; e++) { - if (!scr_map_live_at(m, e)) continue; - scr_map_release_key(m, scr_map_at(m, e)->key); - scr_map_release_val(m, scr_map_at(m, e)->val); + /* Number and boolean keys, and values other than references, own + * nothing: a map of only those releases no entries. */ + bool keys = m->key_kind == SCR_MAP_KEY_STR || scr_map_ref_key(m->key_kind); + if (keys || m->val_kind == SCR_MAP_VAL_REF) { + for (size_t e = 0; e < m->nentries; e++) { + if (!scr_map_live_at(m, e)) continue; + scr_map_release_key(m, scr_map_at(m, e)->key); + scr_map_release_val(m, scr_map_val_at(m, e)); + } } scr_mem_free(m->entries); scr_mem_free(m->buckets); @@ -897,7 +994,7 @@ void scr_map_clear(ScrMap *m) { uint32_t *dense = m->ndense ? m->dense : NULL; for (size_t e = 0; e < walked; e++) { if (!scr_map_live_at(m, e)) continue; - uint64_t key = scr_map_at(m, e)->key, val = scr_map_at(m, e)->val; + uint64_t key = scr_map_at(m, e)->key, val = scr_map_val_at(m, e); scr_map_kill_at(m, e); if (dense) dense[(size_t)scr_map_slot_to_f64(key)] = 0; scr_map_release_key(m, key); @@ -955,7 +1052,7 @@ bool scr_map_has_str(const ScrMap *m, const ScrStr *key) { static bool scr_map_delete_found(ScrMap *m, size_t e) { if (e == SCR_MAP_EMPTY) return false; - uint64_t key = scr_map_at(m, e)->key, val = scr_map_at(m, e)->val; + uint64_t key = scr_map_at(m, e)->key, val = scr_map_val_at(m, e); scr_map_kill_at(m, e); /* bucket slot stays: probe chains intact */ m->nlive--; if (m->ndense) m->dense[(size_t)scr_map_slot_to_f64(key)] = 0; @@ -1060,6 +1157,11 @@ static inline __attribute__((always_inline)) void scr_map_set_with( e = scr_map_bucket(m, slot); } if (e != SCR_MAP_EMPTY) { + if (scr_map_lookup_c(lookup)) { + /* Compact values are never counted: nothing to release. */ + ((ScrMapCEntry *)m->entries)[e].val = scr_map_cref_encode(val); + return; + } ScrMapEntry *x = scr_map_at_with(m, e, lookup); uint64_t old = x->val; x->val = val; /* unlink before releasing (cycle collector) */ @@ -1069,7 +1171,18 @@ static inline __attribute__((always_inline)) void scr_map_set_with( scr_map_append(m, hash, key, val, slot); } +/* The insert of a SCR_MAP_VAL_CREF map, out of line so that the string-key + * insert of other maps keeps its size. */ +static __attribute__((noinline)) void scr_map_set_str_c(ScrMap *m, uint64_t hash, uint64_t key, + uint64_t val) { + scr_map_set_with(m, hash, key, val, SCR_MAP_LOOKUP_STR_C); +} + static void scr_map_set_str_slot(ScrMap *m, uint64_t hash, uint64_t key, uint64_t val) { + if (scr_map_cref(m)) { + scr_map_set_str_c(m, hash, key, val); + return; + } scr_map_set_with(m, hash, key, val, SCR_MAP_LOOKUP_STR); } @@ -1157,6 +1270,7 @@ static void scr_map_set(ScrMap *m, uint64_t hash, uint64_t key, uint64_t val) { } switch (scr_map_lookup_kind(m)) { case SCR_MAP_LOOKUP_STR: scr_map_set_str_slot(m, hash, key, val); return; + case SCR_MAP_LOOKUP_STR_C: scr_map_set_str_c(m, hash, key, val); return; case SCR_MAP_LOOKUP_WORD: scr_map_set_word(m, hash, key, val); return; case SCR_MAP_LOOKUP_NUM: scr_map_set_num(m, hash, key, val); return; default: scr_map_set_with(m, hash, key, val, SCR_MAP_LOOKUP_GENERIC); return; @@ -1246,10 +1360,14 @@ bool scr_map_get_str_bool(const ScrMap *m, const ScrStr *key, bool *out) { return true; } +/* scr_map_entry_str serves maps with 24-byte entries (generated code reads + * a SCR_MAP_VAL_CREF map through scr_map_entry_str_c). */ static __attribute__((noinline)) const ScrMapEntry *scr_map_entry_str_slow(const ScrMap *m, const ScrStr *key) { - size_t e = scr_map_find_str(m, key); - return e == SCR_MAP_EMPTY ? NULL : scr_map_at(m, e); + if (m->nlive == 0) return NULL; + size_t e = scr_map_find_with(m, scr_map_hash_str(key), scr_map_slot_from_ptr((void *)key), + SCR_MAP_LOOKUP_STR); + return e == SCR_MAP_EMPTY ? NULL : &m->entries[e]; } /* A stored key with the probe's hash but another identity: compare bytes. @@ -1302,10 +1420,60 @@ const ScrMapEntry *scr_map_entry_str(const ScrMap *m, const ScrStr *key) { return scr_map_entry_str_slow(m, key); } +static __attribute__((noinline)) const ScrMapCEntry *scr_map_entry_str_c_slow(const ScrMap *m, + const ScrStr *key) { + size_t e = scr_map_find_str(m, key); + return e == SCR_MAP_EMPTY ? NULL : scr_map_c_at(m, e); +} + +static __attribute__((noinline)) const ScrMapCEntry *scr_map_entry_str_c_bytes(const ScrMap *m, + const ScrStr *key, + const ScrMapCEntry *e) { + const ScrStr *stored = scr_map_slot_to_ptr(e->key); + if (stored->len == key->len && scr_key_equal(stored->data, key->data, key->len)) return e; + return scr_map_entry_str_c_slow(m, key); +} + +/* scr_map_entry_str over a SCR_MAP_VAL_CREF map's 16-byte entries, whose + * hash is the low half of the widened one. */ +const ScrMapCEntry *scr_map_entry_str_c(const ScrMap *m, const ScrStr *key) { +#if SCR_STR_HASH_CACHE + uint32_t h = key->hash; + size_t nbuckets = m->nbuckets; + if (__builtin_expect(h != 0 && scr_map_compact_buckets(nbuckets), 1)) { + uint64_t hash = scr_map_hash_widen(h); + uint64_t bits = scr_map_filter_bits(hash); + if ((m->filter & bits) != bits) return NULL; + const ScrMapCEntry *entries = (const ScrMapCEntry *)m->entries; + if (nbuckets == 0) { + for (size_t e = 0, n = m->nentries; e < n; e++) { + if (entries[e].hash != h) continue; + if (entries[e].key == scr_map_slot_from_ptr((void *)key)) return &entries[e]; + return scr_map_entry_str_c_bytes(m, key, &entries[e]); + } + return NULL; + } + const uint32_t *buckets = m->buckets; + uint32_t tag = (uint32_t)(hash >> 32 & UINT32_C(0xff000000)); + size_t mask = nbuckets - 1; + for (size_t i = hash & mask;; i = (i + 1) & mask) { + uint32_t bucket = buckets[i]; + if (!bucket) return NULL; + if ((bucket & ~SCR_MAP_INDEX_MASK) != tag) continue; + const ScrMapCEntry *e = &entries[(bucket & SCR_MAP_INDEX_MASK) - 1]; + if (e->hash != h) continue; + if (e->key == scr_map_slot_from_ptr((void *)key)) return e; + return scr_map_entry_str_c_bytes(m, key, e); + } + } +#endif + return scr_map_entry_str_c_slow(m, key); +} + void *scr_map_get_str_ref(const ScrMap *m, const ScrStr *key) { size_t e = scr_map_find_str(m, key); if (e == SCR_MAP_EMPTY) return NULL; - return m->ops->val_retain(scr_map_slot_to_ptr(scr_map_at(m, e)->val)); /* +1 */ + return scr_map_val_retain(m, scr_map_val_at(m, e)); /* +1 */ } bool scr_map_get_ref_f64(const ScrMap *m, const void *key, double *out) { @@ -1408,6 +1576,12 @@ static const ScrMapEntry *scr_map_iter_at(const ScrMap *m, double i) { return scr_map_at(m, (size_t)i); } +/* The value of the live entry at iteration index i as an 8-byte slot. */ +static uint64_t scr_map_iter_val_slot(const ScrMap *m, double i) { + (void)scr_map_iter_at(m, i); + return scr_map_val_at(m, (size_t)i); +} + double scr_map_iter_key_f64(const ScrMap *m, double i) { return scr_map_slot_to_f64(scr_map_iter_at(m, i)->key); } @@ -1442,7 +1616,7 @@ ScrDyn *scr_map_dyn_key(const ScrMap *map, double index) { } ScrDyn *scr_map_dyn_value(const ScrMap *map, double index) { - uint64_t slot = scr_map_iter_at(map, index)->val; + uint64_t slot = scr_map_iter_val_slot(map, index); return map->dyn_ops ? map->dyn_ops->val_box(slot) : scr_dyn_retain(scr_map_slot_to_ptr(slot)); } @@ -1491,6 +1665,9 @@ ScrMap *scr_map_clone(const ScrMap *source, bool keys_only, const ScrMapOps *ops if (!out->entries) scr_map_oom(); out->ecap = (uint32_t)count; size_t top = 0; + /* A key-only copy of a SCR_MAP_VAL_CREF map has the 24-byte layout: its + * entries are copied field by field. */ + bool same_layout = scr_map_cref(source) == scr_map_cref(out); for (size_t i = 0; i < source->nentries; i++) { if (!scr_map_live_at(source, i)) continue; const ScrMapEntry *entry = scr_map_at(source, i); @@ -1498,11 +1675,13 @@ ScrMap *scr_map_clone(const ScrMap *source, bool keys_only, const ScrMapOps *ops size_t k = (size_t)scr_map_slot_to_f64(entry->key); if (k > top) top = k; } - ScrMapEntry *copy = scr_map_at(out, out->nentries++); - memcpy(copy, entry, entry_size); + size_t e = out->nentries++; + ScrMapEntry *copy = scr_map_at(out, e); + if (__builtin_expect(same_layout, 1)) memcpy(copy, entry, entry_size); + else scr_map_put_at(out, e, entry->key, keys_only ? 0 : scr_map_val_at(source, i), scr_map_hash_at(source, i)); if (out->key_kind == SCR_MAP_KEY_STR) scr_str_retain(scr_map_slot_to_ptr(copy->key)); else if (scr_map_ref_key(out->key_kind)) out->ops->key_retain(scr_map_slot_to_ptr(copy->key)); - if (keys_only) copy->val = 0; + if (keys_only) scr_map_val_put(out, e, 0); else if (out->val_kind == SCR_MAP_VAL_REF) out->ops->val_retain(scr_map_slot_to_ptr(copy->val)); } out->nlive = (uint32_t)count; @@ -1550,7 +1729,7 @@ ScrMap *scr_map_clone_dyn(const ScrMap *source, bool keys_only) { void scr_map_values_into(ScrMap *set, const ScrMap *source, bool checked) { for (size_t i = 0; i < source->nentries; i++) { if (!scr_map_live_at(source, i)) continue; - uint64_t value = checked ? scr_map_slot_from_ptr(scr_map_dyn_value(source, (double)i)) : scr_map_at(source, i)->val; + uint64_t value = checked ? scr_map_slot_from_ptr(scr_map_dyn_value(source, (double)i)) : scr_map_val_at(source, i); if (set->key_kind == SCR_MAP_KEY_F64) value = scr_map_f64_bits(scr_map_slot_to_f64(value)); scr_map_set(set, scr_map_hash_key(set, value), value, 0); if (checked) scr_dyn_release(scr_map_slot_to_ptr(value)); @@ -1567,7 +1746,7 @@ ScrArr *scr_set_to_arr_dyn(const ScrMap *map) { } void *scr_map_iter_val_ref(const ScrMap *m, double i) { - return m->ops->val_retain(scr_map_slot_to_ptr(scr_map_iter_at(m, i)->val)); /* +1 */ + return scr_map_val_retain(m, scr_map_iter_val_slot(m, i)); /* +1 */ } /* A published (immortal) map never changes, so iterating it needs no diff --git a/packages/runtime/src/scr_map.h b/packages/runtime/src/scr_map.h index c61df9a90..af11d92d4 100644 --- a/packages/runtime/src/scr_map.h +++ b/packages/runtime/src/scr_map.h @@ -50,7 +50,11 @@ typedef enum { SCR_MAP_KEY_BIGINT, SCR_MAP_KEY_BOOL, SCR_MAP_KEY_UNION_VALUE, SCR_MAP_KEY_NULL, SCR_MAP_KEY_UNDEFINED /* tag-only union arms */ } ScrMapKeyKind; -typedef enum { SCR_MAP_VAL_F64, SCR_MAP_VAL_BOOL, SCR_MAP_VAL_REF } ScrMapValKind; +/* SCR_MAP_VAL_CREF: a string-key map whose values are compact class + * instances in the program-lifetime region (llvm/classes.ts + * LlClassMeta.compact). Its entries are ScrMapCEntry; the values are never + * counted or traced. */ +typedef enum { SCR_MAP_VAL_F64, SCR_MAP_VAL_BOOL, SCR_MAP_VAL_REF, SCR_MAP_VAL_CREF } ScrMapValKind; typedef struct { uint64_t key; /* double bits (normalized), ScrStr* (owned), or ref ptr (owned) */ @@ -65,6 +69,15 @@ typedef struct { uint64_t val; } ScrMapNumEntry; #define SCR_MAP_NUM_DEAD UINT64_C(0x7ff0000000000001) +/* A SCR_MAP_VAL_CREF map's entry: the string key, the key's 32-bit hash + * (its cached 64-bit hash is the same value in both halves, + * scr_map.c scr_map_hash_widen; zero = tombstone), and the value as an + * address over 8 (0 for NULL). */ +typedef struct { + uint64_t key; + uint32_t hash; + uint32_t val; +} ScrMapCEntry; /* Checked views use the original storage. Slot converters borrow on reads * and return owned references on writes, just like typed Map operations. */ @@ -140,16 +153,25 @@ typedef struct ScrMap { * the whole deleted prefix again. */ uint32_t head; } ScrMap; -/* The entry at index i of either layout, and whether it is live (code that - * walks a map's entries outside scr_map.c). Key and value are at bytes 0 - * and 8 in both layouts; a number-key entry has no hash field. */ +/* The entry at index i of any layout, whether it is live, and its value as + * an 8-byte slot (code that walks a map's entries outside scr_map.c). The + * key is at byte 0 in every layout; a number-key entry has no hash field, + * and a SCR_MAP_VAL_CREF entry keeps its value in 4 bytes. */ static inline const ScrMapEntry *scr_map_entry_at(const ScrMap *m, size_t i) { - size_t size = m->key_kind == SCR_MAP_KEY_F64 ? sizeof(ScrMapNumEntry) : sizeof(ScrMapEntry); + size_t size = m->key_kind == SCR_MAP_KEY_F64 ? sizeof(ScrMapNumEntry) + : m->val_kind == SCR_MAP_VAL_CREF ? sizeof(ScrMapCEntry) + : sizeof(ScrMapEntry); return (const ScrMapEntry *)((const char *)m->entries + i * size); } static inline bool scr_map_entry_live(const ScrMap *m, size_t i) { const ScrMapEntry *e = scr_map_entry_at(m, i); - return m->key_kind == SCR_MAP_KEY_F64 ? e->key != SCR_MAP_NUM_DEAD : e->hash != 0; + if (m->key_kind == SCR_MAP_KEY_F64) return e->key != SCR_MAP_NUM_DEAD; + if (m->val_kind == SCR_MAP_VAL_CREF) return ((const ScrMapCEntry *)e)->hash != 0; + return e->hash != 0; +} +static inline uint64_t scr_map_entry_val(const ScrMap *m, const ScrMapEntry *e) { + if (m->val_kind == SCR_MAP_VAL_CREF) return (uint64_t)((const ScrMapCEntry *)e)->val << 3; + return e->val; } /* The LLVM backend reads the header through two types (emitter.ts). Its * inline retain/release test the traced byte as %ScrMapRc field 3, as @@ -193,6 +215,9 @@ _Static_assert(sizeof(ScrMapEntry) == 24 && offsetof(ScrMapEntry, val) == 8, "inline lookups read ScrMapEntry.val at byte 8"); _Static_assert(sizeof(ScrMapNumEntry) == 16 && offsetof(ScrMapNumEntry, val) == 8, "inline number-key lookups and iteration read %ScrMapNumEntry { key, val }"); +_Static_assert(sizeof(ScrMapCEntry) == 16 && offsetof(ScrMapCEntry, hash) == 8 && + offsetof(ScrMapCEntry, val) == 12, + "inline compact-value lookups and iteration read %ScrMapCEntry { key, hash, val }"); _Static_assert(sizeof(ScrMap) <= 11 * sizeof(uint64_t), "the ScrMap header stays within 88 bytes"); /* Entry for a number key, or NULL when absent. The pointer is valid until * the next mutation; generated code reads the value slot through it. */ @@ -201,6 +226,9 @@ const ScrMapEntry *scr_map_entry_f64(const ScrMap *m, double key); * reference reads retain the value with its static type's retain instead * of the collection's adapter. */ const ScrMapEntry *scr_map_entry_str(const ScrMap *m, const ScrStr *key); +/* scr_map_entry_str for a SCR_MAP_VAL_CREF map: its entry is a + * ScrMapCEntry. */ +const ScrMapCEntry *scr_map_entry_str_c(const ScrMap *m, const ScrStr *key); /* String-key lookups borrow an exact byte span. Reference results return * +1; scalar outputs are written only on a hit, like the string getters. */ diff --git a/packages/runtime/src/scr_publish.c b/packages/runtime/src/scr_publish.c index 4ec993055..44f248b5b 100644 --- a/packages/runtime/src/scr_publish.c +++ b/packages/runtime/src/scr_publish.c @@ -249,12 +249,12 @@ void scr_pub_map(void *v, ScrPub *ctx, ScrPubFn key, ScrPubFn val) { m->key_kind == SCR_MAP_KEY_BIGINT || m->key_kind == SCR_MAP_KEY_UNION_VALUE ? key : NULL; - ScrPubFn vfn = m->val_kind == SCR_MAP_VAL_REF ? val : NULL; + ScrPubFn vfn = m->val_kind == SCR_MAP_VAL_REF || m->val_kind == SCR_MAP_VAL_CREF ? val : NULL; for (size_t i = 0; i < m->nentries; i++) { if (!scr_map_entry_live(m, i)) continue; /* tombstone */ const ScrMapEntry *e = scr_map_entry_at(m, i); if (kfn) scr_pub_slot(ctx, e->key, kfn); - if (vfn) scr_pub_slot(ctx, e->val, vfn); + if (vfn) scr_pub_slot(ctx, scr_map_entry_val(m, e), vfn); } } diff --git a/packages/runtime/test/test_map.c b/packages/runtime/test/test_map.c index 32bdb7be1..46805da1e 100644 --- a/packages/runtime/test/test_map.c +++ b/packages/runtime/test/test_map.c @@ -826,11 +826,96 @@ static void test_absent_key_filter(void) { for (int i = 0; i < UNIVERSE; i++) scr_str_release(names[i]); } +/* SCR_MAP_VAL_CREF: string keys with compact class values in 16-byte + * entries (a 32-bit hash, the value as an address over 8). The values here + * are stand-in addresses, never dereferenced. */ +static uint64_t cref_val(size_t i) { return (uint64_t)(i + 1) * 8; } + +static void test_compact_values(void) { + ScrMap *m = scr_map_new(SCR_MAP_KEY_STR, SCR_MAP_VAL_CREF, NULL); + enum { N = 2000 }; + char buf[32]; + ScrStr *keys[N]; + for (size_t i = 0; i < N; i++) { + snprintf(buf, sizeof buf, "k%zu", i); + keys[i] = S(buf); + scr_map_set_str_ref(m, keys[i], (void *)(uintptr_t)cref_val(i)); + } + check(scr_map_size(m) == N, "cref: size after inserts"); + bool all = true; + for (size_t i = 0; i < N; i++) { + all &= scr_map_get_str_ref(m, keys[i]) == (void *)(uintptr_t)cref_val(i); + const ScrMapCEntry *e = scr_map_entry_str_c(m, keys[i]); + all &= e && ((uint64_t)e->val << 3) == cref_val(i) && e->key == (uint64_t)(uintptr_t)keys[i]; + } + check(all, "cref: get and entry_str_c find every value"); + ScrStr *missing = S("absent"); + check(!scr_map_get_str_ref(m, missing) && !scr_map_entry_str_c(m, missing), "cref: absent key"); + check(scr_map_get_span_ref(m, "k7", 2) == (void *)(uintptr_t)cref_val(7), "cref: span lookup"); + /* A key object other than the stored one, equal bytes. */ + ScrStr *again = S("k42"); + check(scr_map_entry_str_c(m, again) && scr_map_get_str_ref(m, again) == (void *)(uintptr_t)cref_val(42), + "cref: equal key bytes, other object"); + scr_str_release(again); + for (size_t i = 0; i < N; i += 2) scr_map_set_str_ref(m, keys[i], (void *)(uintptr_t)cref_val(i + N)); + all = true; + for (size_t i = 0; i < N; i++) + all &= scr_map_get_str_ref(m, keys[i]) == (void *)(uintptr_t)cref_val(i % 2 ? i : i + N); + check(all && scr_map_size(m) == N, "cref: overwrite keeps the entry"); + for (size_t i = 0; i < N; i++) + if (i % 4) scr_map_delete_str(m, keys[i]); + check(scr_map_size(m) == N / 4, "cref: deletes"); + all = true; + for (size_t i = 0; i < N; i++) { + bool want = i % 4 == 0; + all &= scr_map_has_str(m, keys[i]) == want; + if (want) all &= scr_map_get_str_ref(m, keys[i]) == (void *)(uintptr_t)cref_val(i + N); + } + check(all, "cref: lookups after compaction"); + /* Iteration order and values. */ + size_t seen = 0; + all = true; + scr_map_iter_enter(m); + for (double i = scr_map_iter_start(m); i < scr_map_iter_count(m); i = scr_map_iter_skip(m, i + 1)) { + ScrStr *k = scr_map_iter_key_str(m, i); + void *v = scr_map_iter_val_ref(m, i); + size_t want = seen * 4; + all &= k->len == keys[want]->len && memcmp(k->data, keys[want]->data, k->len) == 0; + all &= v == (void *)(uintptr_t)cref_val(want + N); + scr_str_release(k); + seen++; + } + scr_map_iter_exit(m); + check(all && seen == N / 4, "cref: iteration order and values"); + /* Copies keep the layout; a key-only copy takes the 24-byte one. */ + ScrMap *copy = scr_map_clone(m, false, NULL); + ScrMap *set = scr_map_clone(m, true, NULL); + all = scr_map_size(copy) == N / 4 && scr_map_size(set) == N / 4; + for (size_t i = 0; i < N; i += 4) { + all &= scr_map_get_str_ref(copy, keys[i]) == (void *)(uintptr_t)cref_val(i + N); + all &= scr_map_has_str(set, keys[i]); + } + all &= !scr_map_has_str(set, keys[1]); + check(all, "cref: clone and key-only clone"); + scr_map_set_str_ref(copy, keys[1], (void *)(uintptr_t)cref_val(1)); + check(!scr_map_has_str(m, keys[1]) && scr_map_size(copy) == N / 4 + 1, "cref: copies are independent"); + scr_map_clear(m); + check(scr_map_size(m) == 0 && !scr_map_has_str(m, keys[0]), "cref: clear"); + scr_map_set_str_ref(m, keys[3], (void *)(uintptr_t)cref_val(3)); + check(scr_map_get_str_ref(m, keys[3]) == (void *)(uintptr_t)cref_val(3), "cref: reuse after clear"); + scr_map_release(copy); + scr_map_release(set); + scr_map_release(m); + scr_str_release(missing); + for (size_t i = 0; i < N; i++) scr_str_release(keys[i]); +} + int main(void) { test_key_bytes(); test_string_bucket_collisions(); test_string_keys(); test_string_entries(); + test_compact_values(); test_same_value_zero(); test_span_keys(); test_scratch_keys(); diff --git a/tests/corpus/4639-contract-program-lifetime-maps.ts b/tests/corpus/4639-contract-program-lifetime-maps.ts new file mode 100644 index 000000000..21f7da33c --- /dev/null +++ b/tests/corpus/4639-contract-program-lifetime-maps.ts @@ -0,0 +1,120 @@ +import type { ProgramLifetime } from "@scriptc/types"; + +// Maps from strings to ProgramLifetime class instances: get, set, has and +// delete, growth past the linear scan into bucket tables, deletion with +// compaction, clear, every iteration form, copies, Sets of keys and +// values, maps held in class fields, optional and nullable values, +// inspection, and a cast through unknown. Each read sees the same +// instance that was stored. + +class Sym implements ProgramLifetime { + readonly name: string; + readonly rank: number; + readonly members = new Map(); + constructor(name: string, rank: number) { + this.name = name; + this.rank = rank; + } +} + +class Scope implements ProgramLifetime { + readonly locals = new Map(); + readonly maybe = new Map(); + readonly nullable = new Map(); +} + +function show(m: ReadonlyMap): string { + return [...m].map(([k, v]) => `${k}=${v === undefined ? "u" : v === null ? "n" : v.name}`).join(","); +} + +const a = new Sym("a", 3); +const b = new Sym("b", 1); +const c = new Sym("c", 2); + +// get, set, has, delete, overwrite and size. +const m = new Map(); +m.set("x", a).set("y", b); +m.set("z", c); +console.log(show(m), m.size, m.get("y") === b, m.has("z"), m.has("w"), m.get("w")); +m.set("x", c); +console.log(show(m), m.get("x") === c, m.delete("y"), m.delete("y"), show(m)); +const seeded = new Map([ + ["p", a], + ["q", b], +]); +console.log(show(seeded), seeded.get("q")?.rank); + +// Growth into bucket tables, then deletes that compact the entries. +const big = new Map(); +const all: Sym[] = []; +for (let i = 0; i < 3000; i++) { + const s = new Sym(`s${i}`, i); + all.push(s); + big.set(s.name, s); +} +let same = 0; +for (let i = 0; i < 3000; i++) if (big.get(`s${i}`) === all[i]) same++; +console.log(big.size, same, big.get("s1234")?.rank, big.has("s3000")); +for (let i = 0; i < 3000; i++) if (i % 3 !== 0) big.delete(`s${i}`); +same = 0; +for (const [k, v] of big) if (v === all[Number(k.slice(1))] && v.rank % 3 === 0) same++; +console.log(big.size, same, big.get("s2997")?.name, big.get("s2998")); +for (let i = 0; i < 3000; i += 3) big.set(`s${i}`, all[(i + 1) % 3000]!); +console.log(big.get("s0")?.name, big.get("s2997")?.name, big.size); + +// Iteration forms. +const order: string[] = []; +m.forEach((v, k) => { + order.push(`${k}:${v.name}`); +}); +console.log(order.join(" ")); +console.log([...m.keys()].join(), [...m.values()].map((s) => s.name).join(), [...m.entries()].length); +console.log(Array.from(m.values(), (s) => s.rank).join(), Array.from(m).length); +for (const v of m.values()) v.members.set("owner", a); +console.log(c.members.get("owner") === a, a.members.size); + +// Mutation during iteration: entries added by the callback are visited. +const grow = new Map([["g0", a]]); +let visits = 0; +for (const [k] of grow) { + visits++; + if (visits < 4) grow.set(`${k}+`, b); +} +console.log(visits, show(grow)); + +// Copies, Sets of keys and values, clear. +const copy = new Map(m); +copy.set("extra", b); +console.log(show(m), show(copy), copy.get("x") === m.get("x")); +const keySet = new Set(m.keys()); +const valueSet = new Set(m.values()); +console.log([...keySet].join(), valueSet.has(c), valueSet.has(b), valueSet.size); +copy.clear(); +console.log(copy.size, copy.get("x"), show(m)); +copy.set("again", a); +console.log(show(copy)); + +// Maps in class fields, optional and nullable values. +const scope = new Scope(); +scope.locals.set("a", a); +scope.maybe.set("a", a).set("gone", undefined); +scope.nullable.set("b", b).set("none", null); +console.log(show(scope.locals), show(scope.maybe), show(scope.nullable)); +console.log(scope.maybe.has("gone"), scope.maybe.get("gone"), scope.nullable.get("none"), scope.nullable.get("b")?.name); +const v = scope.maybe.get("a"); +console.log(v === a, v?.rank); + +// Objects from entries, inspection and JSON. +const obj = Object.fromEntries(m); +console.log(Object.keys(obj).join(), obj["z"] === c); +console.log(new Map([["only", b]])); +console.log(JSON.stringify([...m.keys()]), JSON.stringify(Object.fromEntries([...m].map(([k, s]) => [k, s.rank])))); + +// Through unknown and back. +const u: unknown = m; +if (u instanceof Map) { + const back = u as Map; + const seen: string[] = []; + for (const [k, s] of back) seen.push(`${String(k)}:${(s as Sym).name}`); + console.log(back.size, seen.join(), back.get("x") === c); +} diff --git a/tests/harness/corpus-costs/4639-contract-program-lifetime-maps.ts.json b/tests/harness/corpus-costs/4639-contract-program-lifetime-maps.ts.json new file mode 100644 index 000000000..d12d0c3a4 --- /dev/null +++ b/tests/harness/corpus-costs/4639-contract-program-lifetime-maps.ts.json @@ -0,0 +1 @@ +{ "plain": 2500, "san": 3600 }