diff --git a/docs/content/docs/how-it-works.mdx b/docs/content/docs/how-it-works.mdx index ab85d515b..c899476ab 100644 --- a/docs/content/docs/how-it-works.mdx +++ b/docs/content/docs/how-it-works.mdx @@ -70,7 +70,7 @@ Without `-o`, artifacts are written to `.scriptc/` next to the input. Different ### External linking -Program objects define `main` and leave selected `scr_*` runtime functions undefined. A strong reference to `scr_runtime_abi_v19` checks runtime compatibility at link time. +Program objects define `main` and leave selected `scr_*` runtime functions undefined. A strong reference to `scr_runtime_abi_v20` checks runtime compatibility at link time. `--print=native-link-info` emits the program object and a JSON recipe containing the runtime pack and link order for external builds. This object ABI is experimental and requires an exact runtime version. See [Native Program Objects](/docs/native-objects) for examples. diff --git a/examples/native-object/README.md b/examples/native-object/README.md index 1ec17fe16..26cde6853 100644 --- a/examples/native-object/README.md +++ b/examples/native-object/README.md @@ -15,4 +15,4 @@ $ ./app-ld `cc` uses the C compiler as a linker driver. `ld` links the same precompiled runtime objects directly with the selected Apple SDK. The object defines `main`; it is a complete program object, not a library to load into another process. Use `scriptc build --lib --profile ...` for a host-callable static library. -The external object ABI is experimental. Always consume the runtime pack at the exact `runtime_pack.version` reported by the same scriptc installation. The object requires `scr_runtime_abi_v19`, so a mismatched runtime fails during the link instead of starting with an incompatible ABI. +The external object ABI is experimental. Always consume the runtime pack at the exact `runtime_pack.version` reported by the same scriptc installation. The object requires `scr_runtime_abi_v20`, so a mismatched runtime fails during the link instead of starting with an incompatible ABI. diff --git a/packages/cli/test/native-link-info.test.ts b/packages/cli/test/native-link-info.test.ts index 2edc2ff8b..cb30fbb17 100644 --- a/packages/cli/test/native-link-info.test.ts +++ b/packages/cli/test/native-link-info.test.ts @@ -72,7 +72,7 @@ describe.runIf(supported)("macOS arm64 native link info", () => { minimum_os: "14.0", }, program: { object, entry_symbol: "main" }, - runtime_abi: { version: 19, marker: "scr_runtime_abi_v19" }, + runtime_abi: { version: 20, marker: "scr_runtime_abi_v20" }, runtime_pack: { kind: "precompiled", package: "@scriptc/runtime-darwin-arm64", @@ -165,6 +165,6 @@ describe.runIf(supported)("macOS arm64 native link info", () => { () => null, (failure: { stderr?: string }) => failure, ); - expect(error?.stderr).toContain("scr_runtime_abi_v19"); + expect(error?.stderr).toContain("scr_runtime_abi_v20"); }); }); diff --git a/packages/compiler/src/backend/llvm/alloc.test.ts b/packages/compiler/src/backend/llvm/alloc.test.ts index 8f2458e14..771d3b09e 100644 --- a/packages/compiler/src/backend/llvm/alloc.test.ts +++ b/packages/compiler/src/backend/llvm/alloc.test.ts @@ -85,7 +85,17 @@ describe("emitted object allocation", () => { const cyc = emitObjectAlloc(h, "40", { type: "@ty" }); expectSingleDefinitions(cyc); - expect(cyc).toContain(" %sa.n = add i64 40, 16"); + expect(cyc).toContain(" %sa.n = add i64 40, 8"); + // The 8-byte header: the descriptor's i16 registry id, then zero color, + // generation and buffer state; an unregistered descriptor (id 0) takes + // the slow path. + expect(cyc).toContain( + " %sa.tidp = getelementptr inbounds { ptr, ptr, i32, i32 }, ptr @ty, i32 0, i32 3", + ); + expect(cyc).toContain(" %sa.tid = load atomic i32, ptr %sa.tidp acquire, align 4"); + expect(cyc).toContain(" %sa.small = and i1 %sa.reg, %sa.fits0"); + expect(cyc).toContain(" store i16 %sa.id, ptr %sa.blk ; ScrCycHdr.id"); + expect(cyc).toContain(" store i32 0, ptr %sa.bufp ; ScrCycHdr.buf"); expect(cyc).toContain(" %sa.s = call ptr @scr_cyc_alloc(i64 40, ptr @ty)"); expect(cyc).toContain(" store i64 %sa.live1, ptr @scr_cyc_live"); expect(h.decls).toContain( @@ -104,6 +114,12 @@ describe("emitted object allocation", () => { expect(sys).toContain(` call void @scr_rt_free(ptr ${blk})`); expect(lines).toContain(" call void %sf.hook(ptr %o)"); expect(lines.some((l) => l.includes("@scr_cyc_live"))).toBe(cyc); + if (cyc) { + // The generation is the i8 at obj-5. + expect(lines).toContain(" %sf.blk = getelementptr inbounds i8, ptr %o, i64 -8"); + expect(lines).toContain(" %sf.genp = getelementptr inbounds i8, ptr %sf.blk, i64 3"); + expect(lines).toContain(" %sf.old = icmp eq i8 %sf.gen, 2"); + } expect(lines.at(-1)).toBe("sf.done:"); } }); diff --git a/packages/compiler/src/backend/llvm/alloc.ts b/packages/compiler/src/backend/llvm/alloc.ts index 08a7e07ae..f221af74e 100644 --- a/packages/compiler/src/backend/llvm/alloc.ts +++ b/packages/compiler/src/backend/llvm/alloc.ts @@ -117,11 +117,16 @@ const SA_BUMP = 280; const SA_LIM = 536; const SA_STEP_SHIFT = 4; // 16-byte classes const SA_MAX = 512; -/** sizeof(ScrCycHdr); on 64-bit targets the type descriptor is its first - * word, and every other header word starts zero. */ -const CYC_HDR = 16; -/** ScrCycHdr.gen, an i8 at this header offset (scr_runtime.h asserts it). */ -const CYC_GEN = 9; +/** sizeof(ScrCycHdr): the descriptor's i16 registry id, i8 color and + * generation, and an i32 `buf` whose sign bit marks a buffered candidate + * (scr_runtime.h asserts the offsets). */ +export const CYC_HDR = 8; +/** ScrCycHdr.gen, an i8 at this header offset. */ +const CYC_GEN = 3; +/** ScrCycHdr.buf, an i32 at this header offset. */ +export const CYC_BUF = 4; +/** The emitted descriptor `{ trace, free_fn, kind, id }` (ScrCycType). */ +const CYC_TYPE = "{ ptr, ptr, i32, i32 }"; function declareState(host: AllocHost): void { if (host.threadLocalAlloc !== true) host.declare(`@scr_sa = external global ${SA_TYPE}`); @@ -144,12 +149,13 @@ export interface CycHeader { export const FIRST_EMITTED_CYC_KIND = 32; /** The definition of a shape's ScrCycType descriptor `{ trace, free_fn, - * kind }`, referenced by every header of that shape. `kind` indexes the - * collector's per-type statistics (scr_runtime.h); emitted shapes number - * themselves from FIRST_EMITTED_CYC_KIND (ShapeHost.nextCycKind) and 0 - * keeps none. */ + * kind, id }`, named by every header of that shape through its registry + * id. `kind` indexes the collector's per-type statistics (scr_runtime.h); + * emitted shapes number themselves from FIRST_EMITTED_CYC_KIND + * (ShapeHost.nextCycKind) and 0 keeps none. The runtime assigns `id` on the + * first allocation, so the descriptor is writable. */ export function cycTypeDescriptor(name: string, trace: string, free: string, kind = 0): string { - return `${name} = internal constant { ptr, ptr, i32 } { ptr ${trace}, ptr ${free}, i32 ${kind} }`; + return `${name} = internal global ${CYC_TYPE} { ptr ${trace}, ptr ${free}, i32 ${kind}, i32 0 }`; } /** @@ -203,11 +209,21 @@ function allocLines( } declareState(host); // Bytes of the block that the fast path itself stores after the clear: - // the header's descriptor word, or the object's leading words. - const keep = cyc ? 8 : stored; + // the header, or the object's leading words. + const keep = cyc ? CYC_HDR : stored; + // A type's first object registers its descriptor on the slow path. + const registered = cyc + ? [ + ` %sa.tidp = getelementptr inbounds ${CYC_TYPE}, ptr ${cyc.type}, i32 0, i32 3`, + ` %sa.tid = load atomic i32, ptr %sa.tidp acquire, align 4`, + ` %sa.reg = icmp ne i32 %sa.tid, 0`, + ` %sa.fits0 = icmp ule i64 %sa.n, ${SA_MAX}`, + ` %sa.small = and i1 %sa.reg, %sa.fits0`, + ] + : [` %sa.small = icmp ule i64 %sa.n, ${SA_MAX}`]; const lines = [ ` %sa.n = add i64 ${size}, ${cyc ? CYC_HDR : 0}`, - ` %sa.small = icmp ule i64 %sa.n, ${SA_MAX}`, + ...registered, ` br i1 %sa.small, label %sa.class, label %sa.slow`, `sa.class:`, ` %sa.nm1 = sub i64 %sa.n, 1`, @@ -253,7 +269,12 @@ function allocLines( lines.push( `sa.fast:`, ` %sa.blk = phi ptr [ %sa.b, %sa.pop ], [ %sa.p, %sa.carve ]`, - ` store ptr ${cyc.type}, ptr %sa.blk ; ScrCycHdr.type`, + ` %sa.id = trunc i32 %sa.tid to i16`, + ` store i16 %sa.id, ptr %sa.blk ; ScrCycHdr.id`, + ` %sa.cgp = getelementptr inbounds i8, ptr %sa.blk, i64 2`, + ` store i16 0, ptr %sa.cgp ; ScrCycHdr.color, gen`, + ` %sa.bufp = getelementptr inbounds i8, ptr %sa.blk, i64 ${CYC_BUF}`, + ` store i32 0, ptr %sa.bufp ; ScrCycHdr.buf`, ` %sa.live = load i64, ptr @scr_cyc_live`, ` %sa.live1 = add i64 %sa.live, 1`, ` store i64 %sa.live1, ptr @scr_cyc_live`, @@ -339,7 +360,7 @@ function freeLines(host: AllocHost, cyc: boolean): string[] { ` store i64 %sf.live1, ptr @scr_cyc_live`, ` %sf.blk = getelementptr inbounds i8, ptr %o, i64 -${CYC_HDR}`, ` %sf.genp = getelementptr inbounds i8, ptr %sf.blk, i64 ${CYC_GEN}`, - ` %sf.gen = load i8, ptr %sf.genp`, + ` %sf.gen = load i8, ptr %sf.genp ; ScrCycHdr.gen`, ` %sf.old = icmp eq i8 %sf.gen, 2`, ` br i1 %sf.old, label %sf.oldcount, label %sf.accounted`, `sf.oldcount:`, diff --git a/packages/compiler/src/backend/llvm/classes.test.ts b/packages/compiler/src/backend/llvm/classes.test.ts index 99a655133..309d8f071 100644 --- a/packages/compiler/src/backend/llvm/classes.test.ts +++ b/packages/compiler/src/backend/llvm/classes.test.ts @@ -79,7 +79,7 @@ test("releases split into an inlinable fast path and an out-of-line remainder", // takes the possible-root step without loading the vtable. const fast = body(mangleClassRelease("Node")); expect(fast).toContain("%shared = icmp ult i64 %rcm2, -3"); - expect(fast).toContain("load i16, ptr %bufp"); + expect(fast).toContain("load i32, ptr %bufp"); expect(fast).toContain(`call void @${mangleClassRelease("Node")}_slow(ptr %o)`); expect(fast).not.toContain("%ScrVt"); // Only the last reference dispatches, from a noinline remainder. @@ -136,7 +136,7 @@ test("ProgramLifetime hierarchies are never freed and take no reference counts", // The mortal Owner keeps its header, counts and possible-root step. expect(body(mangleClassNew("Owner"))).toContain("store i64 1, ptr %o"); expect(body(mangleClassNew("Owner"))).toContain("scr_cyc_alloc"); - expect(body(mangleClassRelease("Owner"))).toContain("load i16, ptr %bufp"); + expect(body(mangleClassRelease("Owner"))).toContain("load i32, ptr %bufp"); }); test("published ProgramLifetime instances use the published mark every other object uses", () => { diff --git a/packages/compiler/src/backend/llvm/emitter.ts b/packages/compiler/src/backend/llvm/emitter.ts index 4c2be9440..60b0a876a 100644 --- a/packages/compiler/src/backend/llvm/emitter.ts +++ b/packages/compiler/src/backend/llvm/emitter.ts @@ -1,6 +1,6 @@ import { emitLlvmLayouts } from "./layouts.js"; import { compileRegexMatcher, type RegexMatcherCode } from "./regex-matchers.js"; -import { FIRST_EMITTED_CYC_KIND, inlineAllocSupported } from "./alloc.js"; +import { CYC_BUF, CYC_HDR, FIRST_EMITTED_CYC_KIND, inlineAllocSupported } from "./alloc.js"; import { hasLoneSurrogate, llvmBytes as llBytes, wtf8Bytes } from "../literals.js"; import { InternalCompilerError } from "../../errors.js"; import { createHash } from "node:crypto"; @@ -550,7 +550,7 @@ export class LlEmitter { private debugScope: string | null = null; readonly sizeType: "i32" | "i64"; readonly ffiExtendNarrowIntegers: boolean; - readonly cycleBufferedOffset: number; + readonly cycleBufOffset: number; /** Emitted ScrCycType descriptors so far (their collector statistics * indices; see ShapeHost.nextCycKind). */ private cycKinds = 0; @@ -976,10 +976,8 @@ export class LlEmitter { this.runtimeAbiMarker = options.runtimeAbiMarker === true; this.detachedExit = options.detachedExit === true; this.rcHelpers = options.inlineRc === true ? new Set() : null; - // ScrCycHdr.buffered sits 6 bytes behind every object. The wasm32 - // header pads before its descriptor so its payload remains - // double-aligned without changing this ABI offset. - this.cycleBufferedOffset = 6; + // ScrCycHdr.buf sits 4 bytes behind every object on every target. + this.cycleBufOffset = CYC_HDR - CYC_BUF; this.ffiCallbackAdapters = allocateFfiCallbackAdapters(mod.ffiImports ?? []); this.ffiHasRetainedCallback = hasRetainedFfiCallback(mod.ffiImports ?? []); this.ffiHasForeignCallback = hasForeignFfiCallback(mod.ffiImports ?? []); @@ -1097,7 +1095,7 @@ export class LlEmitter { this.tracedUnions = traced.unions; this.shapeHost = { sizeType: this.sizeType, - cycleBufferedOffset: this.cycleBufferedOffset, + cycleBufOffset: this.cycleBufOffset, tracedShapes: this.tracedShapes, tracedUnions: this.tracedUnions, nullableUnions: this.nullableUnions, diff --git a/packages/compiler/src/backend/llvm/shapes.test.ts b/packages/compiler/src/backend/llvm/shapes.test.ts index f8199d555..563fdd5b5 100644 --- a/packages/compiler/src/backend/llvm/shapes.test.ts +++ b/packages/compiler/src/backend/llvm/shapes.test.ts @@ -23,7 +23,7 @@ function declarationHost( }, needOom() {}, sizeType: pointerBits === 32 ? "i32" : "i64", - cycleBufferedOffset: 6, + cycleBufOffset: 4, tracedShapes: new Set(), tracedUnions: new Set(), nullableUnions: new NullableUnions({ @@ -148,10 +148,11 @@ describe("LLVM runtime RC symbols", () => { expect(fn("sc_rc_release_map")).toContain("%ScrMapRc, ptr %o, i32 0, i32 8"); expect(fn("sc_rc_release_map")).toContain("br i1 %headered, label %slow, label %dec"); expect(fn("sc_rc_release_map")).not.toContain("store i32 1"); - // Unions: no color store, only the i16 `buffered` read at obj-6. + // Unions: no color store, only the sign of the i32 `buf` at obj-4. expect(fn("sc_rc_release_union")).not.toContain("store i32"); - expect(fn("sc_rc_release_union")).toContain("getelementptr i8, ptr %o, i64 -6"); - expect(fn("sc_rc_release_union")).toContain("load i16, ptr %bufp"); + expect(fn("sc_rc_release_union")).toContain("getelementptr i8, ptr %o, i64 -4"); + expect(fn("sc_rc_release_union")).toContain("load i32, ptr %bufp"); + expect(fn("sc_rc_release_union")).toContain("icmp slt i32 %buf, 0"); expect(fn("sc_rc_release_union")).toContain("call void @scr_cyc_on_release(ptr %o)"); // Promises tolerate NULL and always carry a header. expect(fn("sc_rc_retain_promise")).toContain("icmp eq ptr %o, null"); @@ -163,7 +164,7 @@ describe("LLVM runtime RC symbols", () => { releaseSym(host, { kind: "union", unionId: "u0" } as never); const ir = emitInlineRcHelpers(host).join("\n"); expect(ir).toContain("load i32, ptr %o"); - expect(ir).toContain("getelementptr i8, ptr %o, i32 -6"); + expect(ir).toContain("getelementptr i8, ptr %o, i32 -4"); }); }); @@ -182,7 +183,7 @@ describe("emitted object releases", () => { const start = defs.indexOf("root:"); const root = defs.slice(start, defs.indexOf("done:", start)); expect(root).not.toContain("store i32"); - expect(root).toContain("getelementptr i8, ptr %o, i64 -6"); + expect(root).toContain("getelementptr i8, ptr %o, i64 -4"); expect(root).toContain("br i1 %queued, label %done, label %enqueue"); expect(root.indexOf("enqueue:")).toBeLessThan( root.indexOf("call void @scr_cyc_on_release(ptr %o)"), diff --git a/packages/compiler/src/backend/llvm/shapes.ts b/packages/compiler/src/backend/llvm/shapes.ts index 4d0440b4c..7d41e7717 100644 --- a/packages/compiler/src/backend/llvm/shapes.ts +++ b/packages/compiler/src/backend/llvm/shapes.ts @@ -39,8 +39,8 @@ export interface ShapeHost { needOom(): void; /** The target C ABI's `size_t` / refcount integer type. */ readonly sizeType: "i32" | "i64"; - /** Byte offset from an object pointer back to ScrCycHdr.buffered. */ - readonly cycleBufferedOffset: number; + /** Byte offset from an object pointer back to ScrCycHdr.buf. */ + readonly cycleBufOffset: number; readonly tracedShapes: Set; readonly tracedUnions: Set; /** Unions represented as a nullable arm pointer (nullable-unions.ts): @@ -866,9 +866,9 @@ export function directChildReleaseLines( if (target.traced) { host.declare(`declare void @scr_cyc_on_dead(ptr)`); lines.push( - ` %${tag}.bufp = getelementptr i8, ptr ${value}, ${S} -${host.cycleBufferedOffset}`, - ` %${tag}.buf = load i16, ptr %${tag}.bufp`, - ` %${tag}.queued = icmp ne i16 %${tag}.buf, 0`, + ` %${tag}.bufp = getelementptr i8, ptr ${value}, ${S} -${host.cycleBufOffset}`, + ` %${tag}.buf = load i32, ptr %${tag}.bufp`, + ` %${tag}.queued = icmp slt i32 %${tag}.buf, 0`, ` br i1 %${tag}.queued, label %${tag}.unbuffer, label %${tag}.teardown`, `${tag}.unbuffer:`, ` call void @scr_cyc_on_dead(ptr ${value})`, @@ -1052,9 +1052,9 @@ export function releaseFastPath( } /** The possible-cycle-root step of a release that left a headered object - * alive: scr_cyc_on_release with its early return inlined. Read `buffered` - * (the i16 at obj-6; nonzero for a queued candidate or a - * tenured object) and call the runtime only to enqueue a new candidate. + * alive: scr_cyc_on_release with its early return inlined. Test the sign + * of `buf` (the i32 at obj-4; negative for a queued candidate or a tenured + * object) and call the runtime only to enqueue a new candidate. * Every emitted release of a headered object shares this sequence * (scr_runtime.h asserts the offset); it ends by branching to `done`. */ /** One traced field of an emitted trace function: load the slot and visit @@ -1084,9 +1084,9 @@ export function cycleRootLines(host: ShapeHost, done: string): string[] { const S = host.sizeType; host.declare(`declare void @scr_cyc_on_release(ptr)`); return [ - ` %bufp = getelementptr i8, ptr %o, ${S} -${host.cycleBufferedOffset}`, - ` %buf = load i16, ptr %bufp`, - ` %queued = icmp ne i16 %buf, 0`, + ` %bufp = getelementptr i8, ptr %o, ${S} -${host.cycleBufOffset}`, + ` %buf = load i32, ptr %bufp`, + ` %queued = icmp slt i32 %buf, 0`, ` br i1 %queued, label %${done}, label %enqueue`, `enqueue:`, ` call void @scr_cyc_on_release(ptr %o) ; buffer the candidate; may collect`, @@ -1102,9 +1102,9 @@ export function cycleDeadLines(host: ShapeHost, next: string): string[] { const S = host.sizeType; host.declare(`declare void @scr_cyc_on_dead(ptr)`); return [ - ` %dbufp = getelementptr i8, ptr %o, ${S} -${host.cycleBufferedOffset}`, - ` %dbuf = load i16, ptr %dbufp`, - ` %dqueued = icmp ne i16 %dbuf, 0`, + ` %dbufp = getelementptr i8, ptr %o, ${S} -${host.cycleBufOffset}`, + ` %dbuf = load i32, ptr %dbufp`, + ` %dqueued = icmp slt i32 %dbuf, 0`, ` br i1 %dqueued, label %unbuffer, label %${next}`, `unbuffer:`, ` call void @scr_cyc_on_dead(ptr %o)`, diff --git a/packages/compiler/src/backend/runtime-abi.ts b/packages/compiler/src/backend/runtime-abi.ts index fe13cde8c..60ca3b5e9 100644 --- a/packages/compiler/src/backend/runtime-abi.ts +++ b/packages/compiler/src/backend/runtime-abi.ts @@ -2,8 +2,8 @@ * of the link contract: a program object keeps this symbol undefined and the * matching runtime defines it, so incompatible manual links fail before the * program can start. */ -export const RUNTIME_ABI_VERSION = 19 as const; -export const RUNTIME_ABI_MARKER = "scr_runtime_abi_v19" as const; +export const RUNTIME_ABI_VERSION = 20 as const; +export const RUNTIME_ABI_MARKER = "scr_runtime_abi_v20" as const; /** Object consumption is useful today, but the complete scr_* surface may * still change before 1.0. The versioned marker prevents accidental mixing; diff --git a/packages/compiler/src/backend/runtime-pack-targets.test.ts b/packages/compiler/src/backend/runtime-pack-targets.test.ts index 655f2dfc4..473da193a 100644 --- a/packages/compiler/src/backend/runtime-pack-targets.test.ts +++ b/packages/compiler/src/backend/runtime-pack-targets.test.ts @@ -52,7 +52,7 @@ async function fixture(target: NativeTargetSpec, workers = false) { object_format: target.objectFormat, minimum_os: target.minimumOs, }, - runtime_abi: { version: 19, marker: "scr_runtime_abi_v19" }, + runtime_abi: { version: 20, marker: "scr_runtime_abi_v20" }, compiler: { command: "fixture", identity: "fixture", target: target.llvmTriple }, macros: { executable: [], excluded: ["SCR_LIB"], sanitizer: "external-toolchain-required" }, flavors: { diff --git a/packages/compiler/src/backend/runtime-pack.test.ts b/packages/compiler/src/backend/runtime-pack.test.ts index 4e6b5c6f8..bd9cffa1b 100644 --- a/packages/compiler/src/backend/runtime-pack.test.ts +++ b/packages/compiler/src/backend/runtime-pack.test.ts @@ -116,7 +116,7 @@ async function fixture(target: NativeTargetSpec = MACOS_ARM64_TARGET) { object_format: target.objectFormat, minimum_os: target.minimumOs, }, - runtime_abi: { version: 19, marker: "scr_runtime_abi_v19" }, + runtime_abi: { version: 20, marker: "scr_runtime_abi_v20" }, compiler: { command: "clang", identity: "fixture clang", diff --git a/packages/compiler/test/native-codegen-integration.test.ts b/packages/compiler/test/native-codegen-integration.test.ts index 0390ce658..260a1f87c 100644 --- a/packages/compiler/test/native-codegen-integration.test.ts +++ b/packages/compiler/test/native-codegen-integration.test.ts @@ -312,7 +312,7 @@ ${[0, 1, 2, 3, 4, 5].map((index) => ` call void @add${index}(i64 ${index + 2})` expect(await symbols(helperObject, ["-u"])).toEqual(await symbols(clangObject, ["-u"])); expect(await symbols(helperObject, ["-gU"])).toEqual(await symbols(clangObject, ["-gU"])); expect(await symbols(helperObject, ["-gU"])).toEqual(["0000000000000000 T _main"]); - expect(await symbols(helperObject, ["-u"])).toContain("_scr_runtime_abi_v19"); + expect(await symbols(helperObject, ["-u"])).toContain("_scr_runtime_abi_v20"); const clangExe = join(dir, "clang-program"); const linkDriver = join(dir, "clang-driver.c"); @@ -646,7 +646,7 @@ ${[0, 1, 2, 3, 4, 5].map((index) => ` call void @add${index}(i64 ${index + 2})` .split("\n"); expect(undefinedSymbols).toContain("_sf_scale"); expect(undefinedSymbols).toContain("_sf_callback_mix"); - expect(undefinedSymbols).toContain("_scr_runtime_abi_v19"); + expect(undefinedSymbols).toContain("_scr_runtime_abi_v20"); }); test("helper and clang object paths retain the same outbound FFI C ABI", async () => { diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5155-cycle-header-types.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5155-cycle-header-types.ts.json new file mode 100644 index 000000000..06b50c312 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5155-cycle-header-types.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/5155-cycle-header-types.ts" + ], + "diags": [] +} diff --git a/packages/runtime-darwin-arm64/scripts/build.mjs b/packages/runtime-darwin-arm64/scripts/build.mjs index 25da9a6a2..512785d2b 100644 --- a/packages/runtime-darwin-arm64/scripts/build.mjs +++ b/packages/runtime-darwin-arm64/scripts/build.mjs @@ -306,7 +306,7 @@ async function build() { package: packageManifest.name, version: packageManifest.version, target: RUNTIME_PACK_MATRIX.target, - runtime_abi: { version: 19, marker: "scr_runtime_abi_v19" }, + runtime_abi: { version: 20, marker: "scr_runtime_abi_v20" }, compiler: { command: compiler, identity: compilerVersion, diff --git a/packages/runtime-pack-common/scripts/build.mjs b/packages/runtime-pack-common/scripts/build.mjs index 6c487bb13..c52279ac1 100644 --- a/packages/runtime-pack-common/scripts/build.mjs +++ b/packages/runtime-pack-common/scripts/build.mjs @@ -402,7 +402,7 @@ async function build() { package: packageManifest.name, version: packageManifest.version, target: matrix.target, - runtime_abi: { version: 19, marker: "scr_runtime_abi_v19" }, + runtime_abi: { version: 20, marker: "scr_runtime_abi_v20" }, compiler: { command: compiler, identity: compilerVersion, diff --git a/packages/runtime/src/scr_array.c b/packages/runtime/src/scr_array.c index f54c4494c..090071c3d 100644 --- a/packages/runtime/src/scr_array.c +++ b/packages/runtime/src/scr_array.c @@ -887,7 +887,7 @@ static ScrArr *scr_arr_make_ops(const ScrArrOps *ops, size_t initial_cap, bool s small ? scr_arr_small_cap(SCR_ARR_INLINE_OFFSET + (cyc ? sizeof(ScrCycHdr) : 0), false) : 0; size_t size = inline_cap ? SCR_ARR_INLINE_OFFSET + inline_cap * SCR_ARR_SLOT_BYTES : sizeof(ScrArr); if (cyc) { - static const ScrCycType cyc_type = { &scr_arr_trace_v, &scr_arr_gc_free, SCR_CYC_KIND_ARRAY }; + static ScrCycType cyc_type = { &scr_arr_trace_v, &scr_arr_gc_free, SCR_CYC_KIND_ARRAY }; a = scr_cyc_alloc_inline(size, &cyc_type); } else { a = scr_mem_alloc(size); diff --git a/packages/runtime/src/scr_async.c b/packages/runtime/src/scr_async.c index f78ab41fb..087a68ce0 100644 --- a/packages/runtime/src/scr_async.c +++ b/packages/runtime/src/scr_async.c @@ -318,7 +318,7 @@ static void scr_promise_gcfree(void *o) { } ScrPromise *scr_promise_new(void) { - static const ScrCycType cyc_type = { &scr_promise_trace, &scr_promise_gcfree, SCR_CYC_KIND_PROMISE }; + static ScrCycType cyc_type = { &scr_promise_trace, &scr_promise_gcfree, SCR_CYC_KIND_PROMISE }; ScrPromise *p = scr_cyc_alloc(sizeof *p, &cyc_type); p->rc = 1; #ifdef SCR_RC_AUDIT diff --git a/packages/runtime/src/scr_closure.c b/packages/runtime/src/scr_closure.c index 66ee0efc1..ca5e29ced 100644 --- a/packages/runtime/src/scr_closure.c +++ b/packages/runtime/src/scr_closure.c @@ -70,7 +70,7 @@ static void scr_box_gcfree(void *o) { } ScrBox *scr_box_new(ScrBoxKind kind) { - static const ScrCycType cyc_type = { &scr_box_trace, &scr_box_gcfree, SCR_CYC_KIND_BOX }; + static ScrCycType cyc_type = { &scr_box_trace, &scr_box_gcfree, SCR_CYC_KIND_BOX }; ScrBox *b = scr_cyc_alloc_inline(sizeof(ScrBox), &cyc_type); b->rc = 1; b->kind = kind; @@ -186,7 +186,7 @@ static void scr_closure_gcfree(void *o) { } ScrClosure *scr_closure_new(void *fn, size_t ncaps) { - static const ScrCycType cyc_type = { &scr_closure_trace, &scr_closure_gcfree, SCR_CYC_KIND_CLOSURE }; + static ScrCycType cyc_type = { &scr_closure_trace, &scr_closure_gcfree, SCR_CYC_KIND_CLOSURE }; ScrClosure *c = scr_cyc_alloc_inline(sizeof(ScrClosure) + ncaps * sizeof(ScrBox *), &cyc_type); c->rc = 1; c->fn = fn; diff --git a/packages/runtime/src/scr_console.c b/packages/runtime/src/scr_console.c index 71c6f7fe8..1b7f485b8 100644 --- a/packages/runtime/src/scr_console.c +++ b/packages/runtime/src/scr_console.c @@ -126,7 +126,7 @@ static void scr_enable_utf8_console_output(void) { } #endif -void scr_runtime_abi_v19(void) {} +void scr_runtime_abi_v20(void) {} void scr_init(void) { #ifdef SCR_WORKERS diff --git a/packages/runtime/src/scr_cycle.c b/packages/runtime/src/scr_cycle.c index eef8f07b3..d8664fb52 100644 --- a/packages/runtime/src/scr_cycle.c +++ b/packages/runtime/src/scr_cycle.c @@ -266,15 +266,37 @@ SCR_TL size_t scr_cyc_live = 0; SCR_TL size_t scr_cyc_old_freed = 0; #endif -void *scr_cyc_alloc(size_t size, const ScrCycType *type) { +/* Descriptor copies by registry id. Id 0 stays empty (an unregistered type). */ +ScrCycType scr_cyc_types[SCR_CYC_MAX_TYPES]; +static uint32_t scr_cyc_ntypes = 1; + +/* Assign `type` its id on its first allocation; any thread may get there + * first, and every thread then sees the same id. A lost race leaves one + * table slot unused. */ +uint32_t scr_cyc_register(ScrCycType *type) { + uint32_t id = __atomic_load_n(&type->id, __ATOMIC_ACQUIRE); + if (id) return id; + uint32_t fresh = __atomic_fetch_add(&scr_cyc_ntypes, 1, __ATOMIC_RELAXED); + if (fresh >= SCR_CYC_MAX_TYPES) scr_trap("scriptc: too many cycle-collected types\n"); + scr_cyc_types[fresh] = *type; + scr_cyc_types[fresh].id = fresh; + __atomic_thread_fence(__ATOMIC_RELEASE); + uint32_t expected = 0; + if (__atomic_compare_exchange_n(&type->id, &expected, fresh, false, __ATOMIC_ACQ_REL, + __ATOMIC_ACQUIRE)) + return fresh; + return expected; +} + +void *scr_cyc_alloc(size_t size, ScrCycType *type) { return scr_cyc_alloc_inline(size, type); } void scr_cyc_free(void *obj) { scr_cyc_free_inline(obj); } static void scr_cyc_grow(ScrVec *vec) { - /* Candidate positions are 32-bit (ScrCycHdr.buf_index). */ - if (vec->cap > (size_t)UINT32_MAX / 2) scr_cyc_oom(); + /* Candidate positions are 31-bit (ScrCycHdr.buf). */ + if (vec->cap > (size_t)(SCR_CYC_BUFFERED_BIT - 1) / 2) scr_cyc_oom(); vec->cap = vec->cap ? vec->cap * 2 : 64; void **grown = realloc(vec->v, vec->cap * sizeof *grown); if (!grown) scr_cyc_oom(); @@ -723,20 +745,20 @@ static inline void scr_cyc_freed(const ScrCycType *type, unsigned gen) { void scr_cyc_on_dead(void *obj) { ScrCycHdr *h = scr_cyc_hdr(obj); - if (!h->buffered) return; + if (!scr_cyc_is_buffered(h)) return; /* O(1) removal: swap the last entry into the hole. A buffered object's * generation never changes (promotion runs only on objects the pass has * already drained from the buffers), so `gen` names the right one. */ - ScrVec *b = &scr_roots[h->gen]; - size_t i = h->buf_index; + ScrVec *b = &scr_roots[scr_cyc_gen(h)]; + size_t i = scr_cyc_buf_index(h); void *last = b->v[--b->n]; b->v[i] = last; - if (last != obj) scr_cyc_hdr(last)->buf_index = (uint32_t)i; - h->buffered = 0; + if (last != obj) scr_cyc_set_buffered(scr_cyc_hdr(last), (uint32_t)i); + scr_cyc_clear_buffered(h); scr_cyc_nbuffered--; /* A death must not consume room reserved for a future candidate. */ scr_cyc_trigger--; - if (h->gen != SCR_CYC_NURSERY && !scr_cyc_older_backlog()) + if (scr_cyc_gen(h) != SCR_CYC_NURSERY && !scr_cyc_older_backlog()) scr_cyc_scheduled_mature_age = 0; } @@ -747,12 +769,11 @@ void scr_cyc_on_release(void *obj) { ScrCycHdr *h = scr_cyc_hdr(obj); /* Releasing an existing candidate cannot advance the trigger. A pass * always re-arms after its teardowns, including any candidates they add. */ - if (h->buffered) return; - if (h->gen < SCR_CYC_OLD) h->gen = (uint8_t)scr_cyc_buffer_gen(h->type, h->gen); - ScrVec *b = &scr_roots[h->gen]; + if (scr_cyc_is_buffered(h)) return; + if (scr_cyc_gen(h) < SCR_CYC_OLD) scr_cyc_set_gen(h, scr_cyc_buffer_gen(scr_cyc_type(h), scr_cyc_gen(h))); + ScrVec *b = &scr_roots[scr_cyc_gen(h)]; if (b->n == b->cap) scr_cyc_grow(b); - h->buffered = 1; - h->buf_index = (uint32_t)b->n; + scr_cyc_set_buffered(h, (uint32_t)b->n); b->v[b->n++] = obj; scr_cyc_nbuffered++; /* Never re-entered: a teardown's releases of untraced children can buffer @@ -770,7 +791,7 @@ void scr_cyc_on_release(void *obj) { * a generation this pass does not collect. */ #define SCR_CYC_SKIP(child) \ ((child) == NULL || SCR_RC(child) == SIZE_MAX \ - || scr_cyc_hdr(child)->gen > scr_gen_limit) + || scr_cyc_gen(scr_cyc_hdr(child)) > scr_gen_limit) /* Trace callbacks forward an integer depth through their opaque context; * it is never dereferenced. The ABI already carries it through every edge. */ @@ -782,31 +803,31 @@ static void scr_mg_visit(void *child, void *ctx) { } static void scr_mark_gray(void *obj, uintptr_t depth) { ScrCycHdr *h = scr_cyc_hdr(obj); - if (h->color == SCR_CYC_GRAY) return; - h->color = SCR_CYC_GRAY; + if (scr_cyc_color(h) == SCR_CYC_GRAY) return; + scr_cyc_set_color(h, SCR_CYC_GRAY); if (depth == SCR_CYC_WALK_DEPTH) { scr_cyc_push(&scr_pending, obj); return; } - h->type->trace(obj, scr_mg_visit, (void *)(depth + 1)); + scr_cyc_type(h)->trace(obj, scr_mg_visit, (void *)(depth + 1)); } static void scr_scan_black(void *obj, uintptr_t depth); static void scr_sb_visit(void *child, void *ctx) { if (SCR_CYC_SKIP(child)) return; SCR_RC(child) += 1; - if (scr_cyc_hdr(child)->color != SCR_CYC_BLACK) + if (scr_cyc_color(scr_cyc_hdr(child)) != SCR_CYC_BLACK) scr_scan_black(child, (uintptr_t)ctx); } static void scr_scan_black(void *obj, uintptr_t depth) { ScrCycHdr *h = scr_cyc_hdr(obj); - h->color = SCR_CYC_BLACK; - if (h->gen < SCR_CYC_OLD) scr_cyc_push(&scr_promote, obj); + scr_cyc_set_color(h, SCR_CYC_BLACK); + if (scr_cyc_gen(h) < SCR_CYC_OLD) scr_cyc_push(&scr_promote, obj); if (depth == SCR_CYC_WALK_DEPTH) { scr_cyc_push(&scr_restore_pending, obj); return; } - h->type->trace(obj, scr_sb_visit, (void *)(depth + 1)); + scr_cyc_type(h)->trace(obj, scr_sb_visit, (void *)(depth + 1)); } static void scr_scan(void *obj, uintptr_t depth); @@ -816,16 +837,16 @@ static void scr_scan_visit(void *child, void *ctx) { } static void scr_scan(void *obj, uintptr_t depth) { ScrCycHdr *h = scr_cyc_hdr(obj); - if (h->color != SCR_CYC_GRAY) return; + if (scr_cyc_color(h) != SCR_CYC_GRAY) return; if (SCR_RC(obj) > 0) { scr_scan_black(obj, 0); while (scr_restore_pending.n) { void *next = scr_restore_pending.v[--scr_restore_pending.n]; - scr_cyc_hdr(next)->type->trace(next, scr_sb_visit, NULL); + scr_cyc_type(scr_cyc_hdr(next))->trace(next, scr_sb_visit, NULL); } return; } - h->color = SCR_CYC_WHITE; + scr_cyc_set_color(h, SCR_CYC_WHITE); /* Record each whitened object once. A later outside root can restore it; * filter the list only after every scan finishes. No gather trace needed. */ scr_cyc_push(&scr_white, obj); @@ -833,7 +854,7 @@ static void scr_scan(void *obj, uintptr_t depth) { scr_cyc_push(&scr_pending, obj); return; } - h->type->trace(obj, scr_scan_visit, (void *)(depth + 1)); + scr_cyc_type(h)->trace(obj, scr_scan_visit, (void *)(depth + 1)); } /* ── cross-generation edges ───────────────────────────────────────────── */ @@ -870,7 +891,7 @@ static void scr_scan(void *obj, uintptr_t depth) { static SCR_CYC_COMPACT void scr_xg_pin_visit(void *child, void *ctx) { (void)ctx; if (child == NULL || SCR_RC(child) == SIZE_MAX) return; - if (scr_cyc_hdr(child)->gen <= scr_gen_limit) return; /* markGray had it */ + if (scr_cyc_gen(scr_cyc_hdr(child)) <= scr_gen_limit) return; /* markGray had it */ SCR_RC(child) += 1; /* pin across the teardowns */ scr_cyc_push(&scr_xgen, child); } @@ -890,8 +911,8 @@ static void scr_xg_child_visit(void *child, void *ctx) { /* One genuine release, generically: the header carries everything needed. */ static void scr_xg_destroy(void *obj) { ScrCycHdr *h = scr_cyc_hdr(obj); - h->type->trace(obj, scr_xg_child_visit, NULL); - h->type->free_fn(obj); + scr_cyc_type(h)->trace(obj, scr_xg_child_visit, NULL); + scr_cyc_type(h)->free_fn(obj); scr_xg_freed++; } @@ -919,10 +940,10 @@ static SCR_CYC_COMPACT size_t scr_cyc_settle(unsigned gen_limit) { for (size_t i = 0; i < scr_white.n; i++) { void *obj = scr_white.v[i]; ScrCycHdr *h = scr_cyc_hdr(obj); - if (h->color != SCR_CYC_WHITE) continue; - h->color = SCR_CYC_DOOMED; - if (gen_limit < SCR_CYC_OLD) scr_cyc_freed(h->type, gen_limit); - else scr_cyc_mature_dead += h->gen == SCR_CYC_MATURE; + if (scr_cyc_color(h) != SCR_CYC_WHITE) continue; + scr_cyc_set_color(h, SCR_CYC_DOOMED); + if (gen_limit < SCR_CYC_OLD) scr_cyc_freed(scr_cyc_type(h), gen_limit); + else scr_cyc_mature_dead += scr_cyc_gen(h) == SCR_CYC_MATURE; scr_white.v[dead++] = obj; } scr_white.n = dead; @@ -934,7 +955,7 @@ static SCR_CYC_COMPACT size_t scr_cyc_settle(unsigned gen_limit) { if (gen_limit < SCR_CYC_OLD) { for (size_t i = 0; i < scr_white.n; i++) { void *obj = scr_white.v[i]; - scr_cyc_hdr(obj)->type->trace(obj, scr_xg_pin_visit, NULL); + scr_cyc_type(scr_cyc_hdr(obj))->trace(obj, scr_xg_pin_visit, NULL); } } @@ -943,7 +964,7 @@ static SCR_CYC_COMPACT size_t scr_cyc_settle(unsigned gen_limit) { * referenced, so none of them is in the white set about to be freed. */ for (size_t i = 0; i < scr_promote.n; i++) { ScrCycHdr *h = scr_cyc_hdr(scr_promote.v[i]); - if (h->gen < SCR_CYC_OLD) h->gen++; + if (scr_cyc_gen(h) < SCR_CYC_OLD) scr_cyc_set_gen(h, scr_cyc_gen(h) + 1); } scr_promote.n = 0; @@ -966,15 +987,14 @@ static SCR_CYC_COMPACT size_t scr_cyc_settle(unsigned gen_limit) { for (size_t i = 0; i < scr_cands.n; i++) { void *obj = scr_cands.v[i]; ScrCycHdr *h = scr_cyc_hdr(obj); - if (h->color == SCR_CYC_DOOMED) continue; /* being freed below */ - if (!h->buffered) { + if (scr_cyc_color(h) == SCR_CYC_DOOMED) continue; /* being freed below */ + if (!scr_cyc_is_buffered(h)) { /* A survivor promoted into a level its type bypasses moves past * it, as a release would buffer it. */ - if (h->gen == SCR_CYC_MATURE && scr_cyc_bypassed(h->type, SCR_CYC_MATURE)) - h->gen = SCR_CYC_OLD; - h->buffered = 1; - h->buf_index = (uint32_t)scr_roots[h->gen].n; - scr_cyc_push(&scr_roots[h->gen], obj); + if (scr_cyc_gen(h) == SCR_CYC_MATURE && scr_cyc_bypassed(scr_cyc_type(h), SCR_CYC_MATURE)) + scr_cyc_set_gen(h, SCR_CYC_OLD); + scr_cyc_set_buffered(h, (uint32_t)scr_roots[scr_cyc_gen(h)].n); + scr_cyc_push(&scr_roots[scr_cyc_gen(h)], obj); } } } @@ -985,7 +1005,7 @@ static SCR_CYC_COMPACT size_t scr_cyc_settle(unsigned gen_limit) { size_t freed = scr_white.n; for (size_t i = 0; i < scr_white.n; i++) { void *obj = scr_white.v[i]; - scr_cyc_hdr(obj)->type->free_fn(obj); + scr_cyc_type(scr_cyc_hdr(obj))->free_fn(obj); } scr_white.n = 0; @@ -1033,8 +1053,8 @@ static size_t scr_cyc_pass(unsigned gen_limit) { for (size_t i = 0; i < scr_roots[g].n; i++) { void *obj = scr_roots[g].v[i]; ScrCycHdr *h = scr_cyc_hdr(obj); - h->buffered = 0; - if (gen_limit < SCR_CYC_OLD) scr_cyc_walked(h->type, gen_limit); + scr_cyc_clear_buffered(h); + if (gen_limit < SCR_CYC_OLD) scr_cyc_walked(scr_cyc_type(h), gen_limit); scr_cyc_push(&scr_cands, obj); } scr_roots[g].n = 0; @@ -1042,7 +1062,7 @@ static size_t scr_cyc_pass(unsigned gen_limit) { for (size_t i = 0; i < scr_cands.n; i++) scr_mark_gray(scr_cands.v[i], 0); while (scr_pending.n) { void *obj = scr_pending.v[--scr_pending.n]; - scr_cyc_hdr(obj)->type->trace(obj, scr_mg_visit, NULL); + scr_cyc_type(scr_cyc_hdr(obj))->trace(obj, scr_mg_visit, NULL); } scr_white.n = 0; @@ -1051,8 +1071,8 @@ static size_t scr_cyc_pass(unsigned gen_limit) { void *obj = scr_pending.v[--scr_pending.n]; /* A later outside root may have restored this deferred white node and * its descendants already. Never scan a black node's edges again. */ - if (scr_cyc_hdr(obj)->color == SCR_CYC_WHITE) - scr_cyc_hdr(obj)->type->trace(obj, scr_scan_visit, NULL); + if (scr_cyc_color(scr_cyc_hdr(obj)) == SCR_CYC_WHITE) + scr_cyc_type(scr_cyc_hdr(obj))->trace(obj, scr_scan_visit, NULL); } return scr_cyc_settle(gen_limit); diff --git a/packages/runtime/src/scr_error.c b/packages/runtime/src/scr_error.c index 672eb3681..18b5a9573 100644 --- a/packages/runtime/src/scr_error.c +++ b/packages/runtime/src/scr_error.c @@ -204,9 +204,9 @@ static ScrError *scr_error_alloc(int kind) { size_t size = kind == SCR_ERR_DOMEX ? sizeof(ScrDomException) : kind == SCR_ERR_AGGREGATE ? sizeof(ScrAggregateError) : sizeof(ScrError); - static const ScrCycType error_type = { &scr_error_trace, &scr_error_gcfree, SCR_CYC_KIND_ERROR }; - static const ScrCycType domex_type = { &scr_error_trace, &scr_domex_gcfree, SCR_CYC_KIND_DOMEXCEPTION }; - static const ScrCycType aggregate_type = { &scr_error_trace, &scr_aggregate_gcfree, SCR_CYC_KIND_ERROR }; + static ScrCycType error_type = { &scr_error_trace, &scr_error_gcfree, SCR_CYC_KIND_ERROR }; + static ScrCycType domex_type = { &scr_error_trace, &scr_domex_gcfree, SCR_CYC_KIND_DOMEXCEPTION }; + static ScrCycType aggregate_type = { &scr_error_trace, &scr_aggregate_gcfree, SCR_CYC_KIND_ERROR }; ScrError *e; if (scr_error_traced) { e = scr_cyc_alloc(size, kind == SCR_ERR_DOMEX ? &domex_type diff --git a/packages/runtime/src/scr_events_emitter.c b/packages/runtime/src/scr_events_emitter.c index ae9cdb4dd..219c3d8ab 100644 --- a/packages/runtime/src/scr_events_emitter.c +++ b/packages/runtime/src/scr_events_emitter.c @@ -284,7 +284,7 @@ ScrEmitter *scr_emitter_new(void) { /* Always collector-headered: the registry holds closures, so the * emitter hierarchy is unconditionally cycle-capable (the canonical * capture cycle is `e.on('x', () => e.emit(...))`). */ - static const ScrCycType cyc_type = { &scr_emitter_trace, &scr_emitter_gcfree, SCR_CYC_KIND_EMITTER }; + static ScrCycType cyc_type = { &scr_emitter_trace, &scr_emitter_gcfree, SCR_CYC_KIND_EMITTER }; ScrEmitter *em = scr_cyc_alloc(sizeof *em, &cyc_type); em->rc = 1; em->vt = &scr_emitter_vt; diff --git a/packages/runtime/src/scr_exception.c b/packages/runtime/src/scr_exception.c index ffc88b53e..a55cc8e99 100644 --- a/packages/runtime/src/scr_exception.c +++ b/packages/runtime/src/scr_exception.c @@ -282,7 +282,7 @@ static void scr_caught_gcfree(void *object) { } ScrCaught *scr_exc_take(void) { - static const ScrCycType cyc_type = { &scr_caught_trace_v, &scr_caught_gcfree, SCR_CYC_KIND_CAUGHT }; + static ScrCycType cyc_type = { &scr_caught_trace_v, &scr_caught_gcfree, SCR_CYC_KIND_CAUGHT }; ScrCaught *c = scr_cyc_alloc(sizeof *c, &cyc_type); c->rc = 1; c->kind = scr_exc_kind; diff --git a/packages/runtime/src/scr_json.c b/packages/runtime/src/scr_json.c index 89cc33eb2..44c64740f 100644 --- a/packages/runtime/src/scr_json.c +++ b/packages/runtime/src/scr_json.c @@ -817,8 +817,8 @@ static ScrDyn *scr_dyn_alloc(ScrDynKind kind) { *list = (ScrDyn *)d->v.str; /* freelist link */ scr_dyn_free_count--; ScrCycHdr *header = scr_cyc_hdr(d); - header->color = SCR_CYC_BLACK; - header->gen = SCR_CYC_NURSERY; + scr_cyc_set_color(header, SCR_CYC_BLACK); + scr_cyc_set_gen(header, SCR_CYC_NURSERY); d->rc = 1; d->kind = kind; d->buffer = false; @@ -850,7 +850,7 @@ static ScrDyn *scr_dyn_alloc(ScrDynKind kind) { return d; } #endif - static const ScrCycType cyc_type = { &scr_dyn_trace_v, &scr_dyn_gcfree, SCR_CYC_KIND_DYN }; + static ScrCycType cyc_type = { &scr_dyn_trace_v, &scr_dyn_gcfree, SCR_CYC_KIND_DYN }; ScrDyn *fresh = scr_cyc_alloc_inline(sizeof *fresh, &cyc_type); if (!fresh) scr_json_oom(); fresh->rc = 1; @@ -9957,7 +9957,7 @@ static void scr_weak_remove_impl(ScrWeakEntry *entry, bool collected) { } static void scr_weak_remove(ScrWeakEntry *entry) { - scr_weak_remove_impl(entry, scr_cyc_hdr(entry->owner)->color == SCR_CYC_DOOMED); + scr_weak_remove_impl(entry, scr_cyc_color(scr_cyc_hdr(entry->owner)) == SCR_CYC_DOOMED); } static void scr_weak_map_trace(void *ptr, ScrTraceVisit visit, void *ctx) { @@ -10157,7 +10157,7 @@ static ScrDyn *scr_weak_collection_new(ScrDyn *seed, bool is_set) { ScrDynHandleTag tag = is_set ? SCR_DYNH_WEAK_SET : SCR_DYNH_WEAK_MAP; scr_weak_dispose_hook = &scr_weak_disposed; scr_dyn_handle_install(tag, is_set ? &set_ops : &map_ops); - static const ScrCycType cyc_type = { &scr_weak_map_trace, &scr_weak_map_gcfree, SCR_CYC_KIND_WEAK_MAP }; + static ScrCycType cyc_type = { &scr_weak_map_trace, &scr_weak_map_gcfree, SCR_CYC_KIND_WEAK_MAP }; ScrWeakMap *map = scr_cyc_alloc(sizeof *map, &cyc_type); if (!map) scr_json_oom(); map->rc = 1; diff --git a/packages/runtime/src/scr_json.test.c b/packages/runtime/src/scr_json.test.c index 3c99eaaba..b123cf97e 100644 --- a/packages/runtime/src/scr_json.test.c +++ b/packages/runtime/src/scr_json.test.c @@ -286,7 +286,7 @@ static void checked_leaf_cycles(void) { ScrStr *text = scr_str_new("survives either edge", 20); ScrDyn *leaf = scr_dyn_new_str(text); scr_str_release(text); - scr_cyc_hdr(leaf)->gen = gen; + scr_cyc_set_gen(scr_cyc_hdr(leaf), gen); ScrDyn *first = scr_dyn_new_obj(), *second = scr_dyn_new_obj(); ScrDyn *array = scr_dyn_new_arr(); ScrClosure *closure = scr_closure_new(NULL, 1); diff --git a/packages/runtime/src/scr_map.c b/packages/runtime/src/scr_map.c index 5b6e333e8..58dd506c8 100644 --- a/packages/runtime/src/scr_map.c +++ b/packages/runtime/src/scr_map.c @@ -720,7 +720,7 @@ ScrMap *scr_map_new_typed(ScrMapKeyKind key_kind, ScrMapValKind val_kind, ScrMap *m; if (key_trace || val_trace) { /* Either side can point back: collector header + trace. */ - static const ScrCycType cyc_type = { &scr_map_trace, &scr_map_gcfree, SCR_CYC_KIND_MAP }; + static ScrCycType cyc_type = { &scr_map_trace, &scr_map_gcfree, SCR_CYC_KIND_MAP }; m = scr_cyc_alloc(sizeof *m, &cyc_type); } else { m = scr_map_alloc_plain(); diff --git a/packages/runtime/src/scr_object.c b/packages/runtime/src/scr_object.c index 7455dbd4a..a533a409c 100644 --- a/packages/runtime/src/scr_object.c +++ b/packages/runtime/src/scr_object.c @@ -17,7 +17,7 @@ static void scr_classobj_gcfree(void *object) { ScrClassObj *scr_classobj_new(const ScrClassObj *template, size_t ncaps) { if (ncaps > (SIZE_MAX - sizeof(ScrClassObj)) / sizeof(ScrBox *)) scr_trap("scriptc: class capture allocation overflow\n"); - static const ScrCycType cyc_type = { &scr_classobj_trace_v, &scr_classobj_gcfree, SCR_CYC_KIND_CLASS_OBJECT }; + static ScrCycType cyc_type = { &scr_classobj_trace_v, &scr_classobj_gcfree, SCR_CYC_KIND_CLASS_OBJECT }; ScrClassObj *c = scr_cyc_alloc(sizeof(ScrClassObj) + ncaps * sizeof(ScrBox *), &cyc_type); c->rc = 1; c->pre = template->pre; diff --git a/packages/runtime/src/scr_runtime.h b/packages/runtime/src/scr_runtime.h index 0c55b507d..0a6683c70 100644 --- a/packages/runtime/src/scr_runtime.h +++ b/packages/runtime/src/scr_runtime.h @@ -74,7 +74,7 @@ void scr_init(void); /* Program objects emitted by the bundled LLVM helper reference this symbol. * Its versioned spelling makes a mismatched manual runtime link fail before * the program can start. */ -void scr_runtime_abi_v19(void); +void scr_runtime_abi_v20(void); void scr_runtime_workers_v8(void); /* ── the trap funnel (scr_console.c; scr_library.c under -DSCR_LIB) ────── @@ -393,10 +393,11 @@ char **scr_worker_argv(void); * Reference counting alone cannot free cycles, so every object that can * participate in one — capture boxes, heap closures, unions, promises, and * compiler-emitted class/record shapes with cycle-capable fields — is - * allocated with a hidden header directly BEFORE the object: a trace - * function (enumerates the object's cycle-capable children), a teardown - * function (frees the object WITHOUT releasing traced children), and the - * trial-deletion bookkeeping (color + candidate-root buffer state). Types + * allocated with a hidden 8-byte header directly BEFORE the object: the + * registry id of its type's descriptor (a trace function enumerating the + * object's cycle-capable children, and a teardown function freeing the + * object WITHOUT releasing traced children) and the trial-deletion + * bookkeeping (color, generation and candidate-root buffer state). Types * that can never be in a cycle — strings above all, plus arrays of scalar/ * string/bytes elements, dyn trees, and shapes whose fields are all acyclic * — keep the lean 1-word `rc` header and pay nothing. Arrays and maps are @@ -448,9 +449,11 @@ enum { SCR_CYC_BLACK = 0, SCR_CYC_GRAY = 1, SCR_CYC_WHITE = 2, SCR_CYC_DOOMED = enum { SCR_CYC_NURSERY = 0, SCR_CYC_MATURE = 1, SCR_CYC_OLD = 2, SCR_CYC_NGENS = 3 }; /* A headered type's collector entry points, shared by all its objects: the - * header names one constant descriptor instead of carrying both function - * pointers, which keeps the header at 16 bytes. Emitted shapes define one - * per traced type (llvm/shapes.ts); runtime families use SCR_CYC_TYPE. */ + * header names its descriptor by a registry id (`id`, assigned on the type's + * first allocation) instead of carrying a pointer, which keeps the header at + * 8 bytes. Emitted shapes define one per traced type (llvm/shapes.ts); + * runtime families define their own. Descriptors are writable (the id) and + * must outlive every object of the type. */ typedef struct ScrCycType { ScrTraceFn trace; ScrCycFreeFn free_fn; @@ -462,6 +465,9 @@ typedef struct ScrCycType { * share an index (separately emitted code in one process) are merely * judged together. */ uint32_t kind; + /* Registry id (scr_cyc_types), 0 until scr_cyc_register assigns it. + * Emitted allocation paths read it and take the slow path while it is 0. */ + uint32_t id; } ScrCycType; /* Runtime families' kinds. Emitted descriptors count from * SCR_CYC_KIND_EMITTED (llvm/alloc.ts FIRST_EMITTED_CYC_KIND). */ @@ -484,46 +490,71 @@ enum { SCR_CYC_KIND_WORKER_VIEW, SCR_CYC_KIND_EMITTED = 32 }; -#define SCR_CYC_TYPE(name, trace_fn, free) static const ScrCycType name = { (trace_fn), (free), 0 } - +/* Eight bytes in front of the object: the descriptor's registry id, the + * collector color (SCR_CYC_*) and generation (SCR_CYC_NURSERY..SCR_CYC_OLD, + * the walk filter), and `buf`: SCR_CYC_BUFFERED_BIT while the object sits in + * its generation's candidate buffer, with its position there below it (O(1) + * removal when rc hits 0). */ typedef struct ScrCycHdr { -#if UINTPTR_MAX == UINT32_MAX - /* calloc aligns the block for doubles; keep the payload aligned too. - * Padding BEFORE the descriptor keeps every object-relative offset of the - * fields below identical to 64-bit targets. */ - uint32_t payload_alignment; -#endif - const ScrCycType *type; /* trace and teardown */ - uint8_t color; /* SCR_CYC_* */ - uint8_t gen; /* SCR_CYC_NURSERY..SCR_CYC_OLD (the walk filter) */ - uint16_t buffered; /* 1 = sitting in its generation's candidate buffer */ - uint32_t buf_index; /* position there (O(1) removal when rc hits 0) */ + uint16_t id; + uint8_t color; + uint8_t gen; + uint32_t buf; } ScrCycHdr; +#define SCR_CYC_BUFFERED_BIT 0x80000000u -/* This layout is an ABI, not an implementation detail. Every emitted release - * of a headered object (llvm/shapes.ts cycleRootLines, used by the record, - * class and inline RC helpers) mirrors scr_cyc_on_release's early return: - * it reads `buffered` as an i16 at obj-6 (the backend's cycleBufferedOffset) - * and calls scr_cyc_on_release only when it is 0; the emitted inline free - * reads the generation as an i8 at obj-7. Hence the assertions. */ -_Static_assert(sizeof(ScrCycHdr) == 16, "LLVM backend expects a 16-byte cycle header"); -_Static_assert(sizeof(ScrCycHdr) - offsetof(ScrCycHdr, buffered) == 6 && - sizeof(((ScrCycHdr *)0)->buffered) == 2, - "the emitted release fast path reads `buffered` as an i16 at obj-6"); -_Static_assert(sizeof(ScrCycHdr) - offsetof(ScrCycHdr, gen) == 7, - "LLVM backend's inline free reads the generation as an i8 at obj-7"); -_Static_assert(sizeof(ScrCycHdr) - offsetof(ScrCycHdr, type) == 16 || - UINTPTR_MAX == UINT32_MAX, - "LLVM backend stores the descriptor at the block start on 64-bit targets"); +/* This layout is an ABI, not an implementation detail. The emitted + * allocation stores the id and zeroes the rest; every emitted release of a + * headered object (llvm/shapes.ts cycleRootLines, used by the record, class + * and inline RC helpers) mirrors scr_cyc_on_release's early return by + * testing the sign of `buf` as an i32 at obj-4, and the emitted inline free + * reads the generation as an i8 at obj-5. Hence the assertions. */ +_Static_assert(sizeof(ScrCycHdr) == 8, "LLVM backend expects an 8-byte cycle header"); +_Static_assert(offsetof(ScrCycHdr, id) == 0 && sizeof(((ScrCycHdr *)0)->id) == 2, + "the emitted allocation stores the registry id as an i16 at the block start"); +_Static_assert(sizeof(ScrCycHdr) - offsetof(ScrCycHdr, buf) == 4, + "the emitted release fast path tests `buf` as an i32 at obj-4"); +_Static_assert(sizeof(ScrCycHdr) - offsetof(ScrCycHdr, gen) == 5, + "LLVM backend's inline free reads the generation as an i8 at obj-5"); _Static_assert(sizeof(ScrCycHdr) % _Alignof(double) == 0, "cycle-headered payloads must preserve double alignment"); static inline ScrCycHdr *scr_cyc_hdr(void *obj) { return (ScrCycHdr *)obj - 1; } +/* Registry of descriptors by id (copies, so a header reaches its entry + * points in one step); ids are assigned on first allocation and shared by + * every thread. */ +#define SCR_CYC_MAX_TYPES ((uint32_t)1 << 15) +extern ScrCycType scr_cyc_types[SCR_CYC_MAX_TYPES]; +uint32_t scr_cyc_register(ScrCycType *type); +static inline uint32_t scr_cyc_type_id(ScrCycType *type) { + /* Acquire pairs with scr_cyc_register's publishing compare-exchange, so + * a thread that sees the id also sees its scr_cyc_types entry. */ + uint32_t id = __atomic_load_n(&type->id, __ATOMIC_ACQUIRE); + return id ? id : scr_cyc_register(type); +} + +static inline const ScrCycType *scr_cyc_type(const ScrCycHdr *h) { return &scr_cyc_types[h->id]; } +static inline unsigned scr_cyc_color(const ScrCycHdr *h) { return h->color; } +static inline void scr_cyc_set_color(ScrCycHdr *h, unsigned color) { h->color = (uint8_t)color; } +static inline unsigned scr_cyc_gen(const ScrCycHdr *h) { return h->gen; } +static inline void scr_cyc_set_gen(ScrCycHdr *h, unsigned gen) { h->gen = (uint8_t)gen; } +static inline bool scr_cyc_is_buffered(const ScrCycHdr *h) { + return (h->buf & SCR_CYC_BUFFERED_BIT) != 0; +} +static inline uint32_t scr_cyc_buf_index(const ScrCycHdr *h) { + return h->buf & ~SCR_CYC_BUFFERED_BIT; +} +/* Buffered at position `index` of its generation's buffer, or not at all. */ +static inline void scr_cyc_set_buffered(ScrCycHdr *h, uint32_t index) { + h->buf = SCR_CYC_BUFFERED_BIT | index; +} +static inline void scr_cyc_clear_buffered(ScrCycHdr *h) { h->buf = 0; } + /* Zeroed allocation with a cycle header in front; returns the OBJECT * pointer (header at scr_cyc_hdr). `type` must outlive the object (a * static descriptor). Aborts on OOM. */ -void *scr_cyc_alloc(size_t size, const ScrCycType *type); +void *scr_cyc_alloc(size_t size, ScrCycType *type); void scr_cyc_free(void *obj); /* frees the block, header included */ /* ── small-object allocator (scr_alloc.c) ─────────────────────────────── @@ -714,26 +745,27 @@ static inline void scr_mem_free(void *p) { free(p); } * sites (union boxes, arrays, boxes, closures, maps, dyn nodes). With a * constant size the size class and the zeroing fold at compile time. The * live count is exported for these and for the emitted inline paths - * (llvm/alloc.ts), which do the same three things: header words, live - * count, block. Every other header word starts zero (SCR_CYC_BLACK, - * SCR_CYC_NURSERY, unbuffered). */ + * (llvm/alloc.ts), which do the same three things: header (the registry + * id; the rest starts zero: SCR_CYC_BLACK, SCR_CYC_NURSERY, unbuffered), + * live count, block. */ #ifndef SCR_WORKERS extern SCR_TL size_t scr_cyc_live; extern SCR_TL size_t scr_cyc_old_freed; extern SCR_TL void (*scr_weak_dispose_hook)(void *); #endif -static inline void *scr_cyc_alloc_inline(size_t size, const ScrCycType *type) { +static inline void *scr_cyc_alloc_inline(size_t size, ScrCycType *type) { if (size > SIZE_MAX - sizeof(ScrCycHdr)) scr_trap("scriptc: out of memory\n"); + uint32_t id = scr_cyc_type_id(type); ScrCycHdr *h = (ScrCycHdr *)scr_mem_calloc(sizeof(ScrCycHdr) + size); if (!h) scr_trap("scriptc: out of memory\n"); - h->type = type; + h->id = (uint16_t)id; scr_cyc_live++; return h + 1; } static inline void scr_cyc_free_inline(void *obj) { if (scr_weak_dispose_hook) scr_weak_dispose_hook(obj); scr_cyc_live--; - if (scr_cyc_hdr(obj)->gen == SCR_CYC_OLD) scr_cyc_old_freed++; + if (scr_cyc_gen(scr_cyc_hdr(obj)) == SCR_CYC_OLD) scr_cyc_old_freed++; scr_mem_free(scr_cyc_hdr(obj)); } _Static_assert(SCR_CYC_NURSERY == 0, "zeroed cycle headers start in the nursery"); @@ -761,10 +793,10 @@ void scr_cyc_on_dead(void *obj); * object, and the death of an object that was never buffered, need no call. * Most releases of long-lived containers take the first early return. */ static inline void scr_cyc_released(void *obj) { - if (!scr_cyc_hdr(obj)->buffered) scr_cyc_on_release(obj); + if (!scr_cyc_is_buffered(scr_cyc_hdr(obj))) scr_cyc_on_release(obj); } static inline void scr_cyc_dead(void *obj) { - if (scr_cyc_hdr(obj)->buffered) scr_cyc_on_dead(obj); + if (scr_cyc_is_buffered(scr_cyc_hdr(obj))) scr_cyc_on_dead(obj); } /* Full sweep: trial-deletion passes over EVERY generation, to a fixpoint. diff --git a/packages/runtime/src/scr_stream.c b/packages/runtime/src/scr_stream.c index 86a9dfecb..7a502ec81 100644 --- a/packages/runtime/src/scr_stream.c +++ b/packages/runtime/src/scr_stream.c @@ -1230,7 +1230,7 @@ static ScrStreamState *scr_stream_state_new(bool has_r, bool has_w, static ScrStream *scr_stream_alloc(const ScrVt *vt, const char *cls, bool has_r, bool has_w, double rhwm, double whwm, bool auto_destroy, bool emit_close, bool allow_half_open) { - static const ScrCycType cyc_type = { &scr_stream_trace, &scr_stream_gcfree, SCR_CYC_KIND_STREAM }; + static ScrCycType cyc_type = { &scr_stream_trace, &scr_stream_gcfree, SCR_CYC_KIND_STREAM }; ScrStream *s = scr_cyc_alloc(sizeof *s, &cyc_type); s->rc = 1; s->vt = vt; diff --git a/packages/runtime/src/scr_union.c b/packages/runtime/src/scr_union.c index c4f3b41f6..9cdbe1b62 100644 --- a/packages/runtime/src/scr_union.c +++ b/packages/runtime/src/scr_union.c @@ -39,7 +39,7 @@ static void scr_union_gcfree(void *o) { } static ScrUnion *scr_union_alloc(uint32_t tag) { - static const ScrCycType cyc_type = { &scr_union_trace, &scr_union_gcfree, SCR_CYC_KIND_UNION }; + static ScrCycType cyc_type = { &scr_union_trace, &scr_union_gcfree, SCR_CYC_KIND_UNION }; ScrUnion *u = scr_cyc_alloc_inline(sizeof(ScrUnion), &cyc_type); u->rc = 1; u->tag = tag; diff --git a/packages/runtime/src/scr_worker_events.c b/packages/runtime/src/scr_worker_events.c index 5cd68b558..ab4b4878a 100644 --- a/packages/runtime/src/scr_worker_events.c +++ b/packages/runtime/src/scr_worker_events.c @@ -77,7 +77,7 @@ static void scr_worker_port_listeners(void *ptr, const ScrStr *name, bool presen } static ScrWorkerView *scr_worker_view_new(bool port) { - static const ScrCycType cyc_type = { &scr_worker_view_trace, &scr_worker_view_gcfree, SCR_CYC_KIND_WORKER_VIEW }; + static ScrCycType cyc_type = { &scr_worker_view_trace, &scr_worker_view_gcfree, SCR_CYC_KIND_WORKER_VIEW }; ScrWorkerView *view = scr_cyc_alloc(sizeof(*view), &cyc_type); view->rc = 1; view->port = port; diff --git a/packages/runtime/test/test_array.c b/packages/runtime/test/test_array.c index 150f4de6d..536aeaa52 100644 --- a/packages/runtime/test/test_array.c +++ b/packages/runtime/test/test_array.c @@ -388,7 +388,7 @@ static void mock_cyc_release(void *p) { } static MockRec *mock_cyc_new(int value) { - static const ScrCycType cyc_type = { &mock_trace, &mock_gcfree, 0 }; + static ScrCycType cyc_type = { &mock_trace, &mock_gcfree, 0 }; MockRec *r = scr_cyc_alloc(sizeof *r, &cyc_type); r->rc = 1; r->value = value; diff --git a/packages/runtime/test/test_closure.c b/packages/runtime/test/test_closure.c index 74f760452..792312db1 100644 --- a/packages/runtime/test/test_closure.c +++ b/packages/runtime/test/test_closure.c @@ -95,20 +95,20 @@ int main(void) { &scr_str_release_v, NULL); scr_union_retain(su); scr_union_release(su); - CHECK(!scr_cyc_hdr(su)->buffered); + CHECK(!scr_cyc_is_buffered(scr_cyc_hdr(su))); scr_union_release(su); CHECK(scr_str_live_count() == 0); ScrUnion *fu = scr_union_new_f64(1, 2.5); scr_union_retain(fu); scr_union_release(fu); - CHECK(!scr_cyc_hdr(fu)->buffered); + CHECK(!scr_cyc_is_buffered(scr_cyc_hdr(fu))); scr_union_release(fu); ScrUnion *cu = scr_union_new_ref(0, scr_closure_new((void *)&dummy_fn, 0), &scr_closure_retain_v, &scr_closure_release_v, &scr_closure_trace_v); scr_union_retain(cu); scr_union_release(cu); - CHECK(scr_cyc_hdr(cu)->buffered); + CHECK(scr_cyc_is_buffered(scr_cyc_hdr(cu))); scr_union_release(cu); /* dies while buffered: leaves the buffer, frees */ CHECK(scr_closure_live_count() == 0); diff --git a/packages/runtime/test/test_cycle.c b/packages/runtime/test/test_cycle.c index 2656d529a..7f7f249d4 100644 --- a/packages/runtime/test/test_cycle.c +++ b/packages/runtime/test/test_cycle.c @@ -63,7 +63,7 @@ static Node *make_ring(size_t count) { Node **nodes = calloc(count, sizeof(*nodes)); if (!nodes) scr_trap("cycle test: out of memory\n"); for (size_t i = 0; i < count; i++) { - static const ScrCycType cyc_type = { &node_trace, &node_free, 0 }; + static ScrCycType cyc_type = { &node_trace, &node_free, 0 }; nodes[i] = scr_cyc_alloc(sizeof(*nodes[i]), &cyc_type); nodes[i]->rc = 1; /* the ring edge that will point at this node */ } @@ -119,7 +119,7 @@ static void check_age_reset_when_last_mature_root_dies(void) { ring = make_ring(32); ring->rc++; release_live(ring); - static const ScrCycType cyc_type = { &node_trace, &node_free, 0 }; + static ScrCycType cyc_type = { &node_trace, &node_free, 0 }; leaf = scr_cyc_alloc(sizeof(*leaf), &cyc_type); leaf->rc = 1; leaf->rc++; /* temporary release makes it a nursery candidate */ @@ -155,7 +155,7 @@ static void check_dead_backlog_rearms_release_trigger(void) { Node *leaves[BACKLOG]; for (size_t i = 0; i < BACKLOG; i++) { - static const ScrCycType cyc_type = { &node_trace, &node_free, 0 }; + static ScrCycType cyc_type = { &node_trace, &node_free, 0 }; leaves[i] = scr_cyc_alloc(sizeof(*leaves[i]), &cyc_type); leaves[i]->rc = 2; /* external owner plus a temporary reference */ release_live(leaves[i]); @@ -183,7 +183,7 @@ static void check_dead_backlog_rearms_release_trigger(void) { } static Node *make_leaf(void) { - static const ScrCycType cyc_type = { &node_trace, &node_free, 0 }; + static ScrCycType cyc_type = { &node_trace, &node_free, 0 }; Node *leaf = scr_cyc_alloc(sizeof(*leaf), &cyc_type); leaf->rc = 1; return leaf; @@ -267,7 +267,7 @@ static void age_to_old(Node *node) { read_live(node); scr_collect_cycles(); } - check(scr_cyc_hdr(node)->gen == SCR_CYC_OLD, "survivor did not reach the old generation"); + check(scr_cyc_gen(scr_cyc_hdr(node)) == SCR_CYC_OLD, "survivor did not reach the old generation"); } /* Fresh live leaves, each released once so the release path keeps running @@ -328,7 +328,7 @@ static void check_older_rings_into_old(void) { } scr_cyc_collect_scheduled(); /* every ring survives at least one pass */ for (size_t i = 0; i < RINGS; i++) { - check(scr_cyc_hdr(rings[i])->gen != SCR_CYC_NURSERY, "ring was not promoted"); + check(scr_cyc_gen(scr_cyc_hdr(rings[i])) != SCR_CYC_NURSERY, "ring was not promoted"); release_live(rings[i]); } size_t grown = grow_until(keep, LIMIT, before + RINGS * RING); @@ -435,8 +435,8 @@ static size_t grow_dead_cycle_until_full_pass(size_t old_garbage, bool mixed) { age_to_old(tie); garbage = make_ring(old_garbage); read_live(garbage); /* a release-triggered pass may promote it at once */ - if (scr_cyc_hdr(garbage)->gen == SCR_CYC_NURSERY) scr_collect_cycles(); - check(scr_cyc_hdr(garbage)->gen == SCR_CYC_MATURE, "survivor did not reach the mature generation"); + if (scr_cyc_gen(scr_cyc_hdr(garbage)) == SCR_CYC_NURSERY) scr_collect_cycles(); + check(scr_cyc_gen(scr_cyc_hdr(garbage)) == SCR_CYC_MATURE, "survivor did not reach the mature generation"); tie->other = garbage; garbage->rc++; garbage->other = tie; @@ -648,7 +648,7 @@ static void check_deep_cross_generation_chain(void) { size_t before = freed; Node *chain = NULL; for (size_t i = 0; i < DEPTH; i++) { - static const ScrCycType cyc_type = { &node_trace, &node_free, 0 }; + static ScrCycType cyc_type = { &node_trace, &node_free, 0 }; Node *node = scr_cyc_alloc(sizeof(*node), &cyc_type); node->rc = 1; node->next = chain; @@ -719,7 +719,7 @@ static void bypass_free(void *obj) { bypass_freed++; node_free(obj); } -static const ScrCycType bypass_type = { &bypass_trace, &bypass_free, 7 }; +static ScrCycType bypass_type = { &bypass_trace, &bypass_free, 7 }; static Node *make_bypass_node(void) { Node *node = scr_cyc_alloc(sizeof(*node), &bypass_type); @@ -818,7 +818,7 @@ static void mature_kind_free(void *obj) { mature_kind_freed++; node_free(obj); } -static const ScrCycType mature_kind_type = { &node_trace, &mature_kind_free, 8 }; +static ScrCycType mature_kind_type = { &node_trace, &mature_kind_free, 8 }; static Node *make_mature_kind_node(void) { Node *node = scr_cyc_alloc(sizeof(*node), &mature_kind_type); @@ -834,7 +834,7 @@ static size_t probe_old_on_release(Node **owned, size_t probe) { for (size_t i = 0; i < probe; i++) { owned[i] = make_mature_kind_node(); read_live(owned[i]); - old += scr_cyc_hdr(owned[i])->gen == SCR_CYC_OLD; + old += scr_cyc_gen(scr_cyc_hdr(owned[i])) == SCR_CYC_OLD; } return old; } @@ -855,7 +855,7 @@ static void check_mature_bypass(void) { * at that level: none dies. */ scr_collect_cycles(); for (size_t i = 0; i < nlive; i++) { - scr_cyc_hdr(live[i])->gen = SCR_CYC_MATURE; + scr_cyc_set_gen(scr_cyc_hdr(live[i]), SCR_CYC_MATURE); read_live(live[i]); } scr_cyc_pass_at(SCR_CYC_MATURE); @@ -924,7 +924,7 @@ static void paired_free(void *obj) { paired_freed++; node_free(obj); } -static const ScrCycType paired_type = { &paired_trace, &paired_free, 10 }; +static ScrCycType paired_type = { &paired_trace, &paired_free, 10 }; static void check_probe_sees_candidate_rings(void) { /* Both members of a ring must be buffered before a pass walks either: with @@ -978,7 +978,7 @@ static void check_probe_sees_candidate_rings(void) { /* A nursery pass re-buffers its survivors past the mature level when * their type bypasses it, even if the type's nursery candidates are * walked (here half of them die young, in rings). */ -static const ScrCycType young_dying_type = { &node_trace, &node_free, 9 }; +static ScrCycType young_dying_type = { &node_trace, &node_free, 9 }; static void check_rebuffer_past_mature(void) { enum { BATCH = 1024, ROUNDS = 24, PROBE = 1024 }; @@ -1011,7 +1011,7 @@ static void check_rebuffer_past_mature(void) { } scr_cyc_pass_at(SCR_CYC_NURSERY); size_t old = 0; - for (size_t i = nlive - PROBE; i < nlive; i++) old += scr_cyc_hdr(live[i])->gen == SCR_CYC_OLD; + for (size_t i = nlive - PROBE; i < nlive; i++) old += scr_cyc_gen(scr_cyc_hdr(live[i])) == SCR_CYC_OLD; check(old >= PROBE / 2, "nursery survivors were re-buffered into a level their type bypasses"); for (size_t i = 0; i < nlive; i++) { live[i]->rc = 0; diff --git a/tests/corpus/5155-cycle-header-types.ts b/tests/corpus/5155-cycle-header-types.ts new file mode 100644 index 000000000..16eb83ed5 --- /dev/null +++ b/tests/corpus/5155-cycle-header-types.ts @@ -0,0 +1,146 @@ +// Cycle-capable objects of many types share one compact header that names +// each type by an id assigned on its first allocation. Several worker +// threads allocate their first objects of the same types at the same time; +// cycles of every shape are built, kept, broken and dropped across the +// collector's generations; and the survivors are checked on every thread. +import { Worker, isMainThread, parentPort, workerData } from "node:worker_threads"; + +class Ring { + next: Ring | null = null; + readonly id: number; + constructor(id: number) { + this.id = id; + } +} +class Pair { + other: Pair | null = null; + value: number; + constructor(value: number) { + this.value = value; + } +} +class Graph { + readonly nodes: GraphNode[] = []; +} +class GraphNode { + readonly edges: GraphEdge[] = []; + owner: Graph | null = null; + readonly label: string; + constructor(label: string) { + this.label = label; + } +} +class GraphEdge { + readonly to: GraphNode; + readonly weight: number; + constructor(to: GraphNode, weight: number) { + this.to = to; + this.weight = weight; + } +} +interface Cell { + value: number; + next: Cell | null; + link: (() => number) | null; +} + +function ring(size: number, base: number): Ring { + const first = new Ring(base); + let last = first; + for (let i = 1; i < size; i++) { + const node = new Ring(base + i); + last.next = node; + last = node; + } + last.next = first; + return first; +} + +function ringSum(start: Ring): number { + let sum = start.id; + for (let at = start.next; at !== null && at !== start; at = at.next) sum += at.id; + return sum; +} + +function graph(n: number, seed: number): Graph { + const g = new Graph(); + for (let i = 0; i < n; i++) { + const node = new GraphNode(`n${i}`); + node.owner = g; + g.nodes.push(node); + } + let x = seed; + for (const node of g.nodes) { + for (let k = 0; k < 3; k++) { + x = (Math.imul(x, 1103515245) + 12345) >>> 0; + node.edges.push(new GraphEdge(g.nodes[x % n]!, x % 17)); + } + } + return g; +} + +function graphWeight(g: Graph): number { + let total = 0; + for (const node of g.nodes) for (const edge of node.edges) total += edge.weight + edge.to.label.length; + return total; +} + +function cells(n: number): Cell { + const head: Cell = { value: 0, next: null, link: null }; + let at = head; + for (let i = 1; i < n; i++) { + const cell: Cell = { value: i, next: null, link: null }; + cell.link = () => cell.value + head.value; + at.next = cell; + at = cell; + } + at.next = head; + return head; +} + +function work(tag: number): string { + const kept: Ring[] = []; + const pairs: Pair[] = []; + let dropped = 0; + for (let round = 0; round < 40; round++) { + // Garbage rings, pairs and graphs; every few rounds one survives. + const r = ring(5 + (round % 7), round * 100 + tag); + dropped += ringSum(r); + if (round % 9 === 0) kept.push(r); + const a = new Pair(round); + const b = new Pair(-round); + a.other = b; + b.other = a; + if (round % 13 === 0) pairs.push(a); + dropped += graphWeight(graph(30 + round, round + tag)); + const c = cells(8); + let links = 0; + for (let at = c.next; at !== c && at !== null; at = at.next) links += at.link!(); + dropped += links; + } + // Break one kept ring and drop it; the rest must survive the passes. + kept[0]!.next = null; + kept.shift(); + let survivors = 0; + for (const r of kept) survivors += ringSum(r); + let pairSum = 0; + for (const p of pairs) pairSum += p.value + p.other!.value + p.other!.other!.value; + return `tag ${tag} dropped ${dropped} kept ${kept.length} survivors ${survivors} pairs ${pairSum}`; +} + +if (isMainThread) { + const results: Promise[] = []; + for (let t = 1; t <= 3; t++) { + results.push( + new Promise((resolve) => { + const worker = new Worker(new URL(import.meta.url), { workerData: t }); + worker.on("message", (text: string) => resolve(text)); + }), + ); + } + console.log("main", work(0)); + for (const text of await Promise.all(results)) console.log("worker", text); + console.log("main again", work(9)); +} else { + parentPort!.postMessage(work(workerData as number)); +} diff --git a/tests/fixtures/self-hosting/llvm-layouts.ts b/tests/fixtures/self-hosting/llvm-layouts.ts index d767defab..2c0f1f440 100644 --- a/tests/fixtures/self-hosting/llvm-layouts.ts +++ b/tests/fixtures/self-hosting/llvm-layouts.ts @@ -52,7 +52,7 @@ try { return `@native_literal_${index}`; }, sizeType: request.bits === 32 ? "i32" : "i64", - cycleBufferedOffset: 6, + cycleBufOffset: 4, // Collector statistics indices, numbered as LlEmitter numbers them. nextCycKind: () => FIRST_EMITTED_CYC_KIND + cycKinds++, tracedShapes: traced.shapes, diff --git a/tests/harness/self-hosting-native-toolchain.test.ts b/tests/harness/self-hosting-native-toolchain.test.ts index e1334197a..eb7b5ef16 100644 --- a/tests/harness/self-hosting-native-toolchain.test.ts +++ b/tests/harness/self-hosting-native-toolchain.test.ts @@ -96,7 +96,7 @@ for (const backend of ["llvm"] as const) { object_format: target.objectFormat, minimum_os: target.minimumOs, }, - runtime_abi: { version: 19, marker: "scr_runtime_abi_v19" }, + runtime_abi: { version: 20, marker: "scr_runtime_abi_v20" }, compiler: { command: "fixture", identity: "fixture", target: target.llvmTriple }, macros: { executable: [], excluded: [], sanitizer: "external-toolchain-required" }, flavors: {