From c6fb8b728251974c5eb6a01c0df32b09d16fcff9 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Tue, 13 Oct 2026 18:37:26 -0500 Subject: [PATCH] Add counted lifetime regions to the runtime --- .../src/backend/native/runtime-inputs.ts | 1 + .../runtime-pack-matrix.mjs | 1 + .../runtime-pack-matrix.mjs | 1 + packages/runtime/src/scr_object.c | 7 +- packages/runtime/src/scr_region.c | 848 ++++++++++++++++++ packages/runtime/src/scr_region.test.c | 460 ++++++++++ packages/runtime/src/scr_region.test.ts | 157 ++++ packages/runtime/src/scr_runtime.h | 95 ++ 8 files changed, 1567 insertions(+), 3 deletions(-) create mode 100644 packages/runtime/src/scr_region.c create mode 100644 packages/runtime/src/scr_region.test.c create mode 100644 packages/runtime/src/scr_region.test.ts diff --git a/packages/compiler/src/backend/native/runtime-inputs.ts b/packages/compiler/src/backend/native/runtime-inputs.ts index 771161dc5..403088321 100644 --- a/packages/compiler/src/backend/native/runtime-inputs.ts +++ b/packages/compiler/src/backend/native/runtime-inputs.ts @@ -65,6 +65,7 @@ export const EXECUTABLE_RUNTIME_SOURCES = [ "scr_child.c", "scr_cycle.c", "scr_publish.c", + "scr_region.c", "scr_alloc.c", ] as const; diff --git a/packages/runtime-darwin-arm64/runtime-pack-matrix.mjs b/packages/runtime-darwin-arm64/runtime-pack-matrix.mjs index 79f4f8a6d..b57cdb8f0 100644 --- a/packages/runtime-darwin-arm64/runtime-pack-matrix.mjs +++ b/packages/runtime-darwin-arm64/runtime-pack-matrix.mjs @@ -40,6 +40,7 @@ const BASE_RUNTIME_SOURCES = [ "scr_child.c", "scr_cycle.c", "scr_publish.c", + "scr_region.c", "scr_alloc.c", ]; diff --git a/packages/runtime-pack-common/runtime-pack-matrix.mjs b/packages/runtime-pack-common/runtime-pack-matrix.mjs index b7ca15bb7..70072e085 100644 --- a/packages/runtime-pack-common/runtime-pack-matrix.mjs +++ b/packages/runtime-pack-common/runtime-pack-matrix.mjs @@ -38,6 +38,7 @@ const BASE_RUNTIME_SOURCES = [ "scr_child.c", "scr_cycle.c", "scr_publish.c", + "scr_region.c", "scr_alloc.c", ]; diff --git a/packages/runtime/src/scr_object.c b/packages/runtime/src/scr_object.c index 5e5e29216..7455dbd4a 100644 --- a/packages/runtime/src/scr_object.c +++ b/packages/runtime/src/scr_object.c @@ -87,9 +87,9 @@ void scr_record_key_miss(ScrStr *k) { #ifdef SCR_RC_AUDIT static SCR_TL long scr_live_objects = 0; -/* ProgramLifetime instances (@scriptc/types) this thread created: never - * freed by design, and neither is anything they hold (see the exit audit in - * scr_console.c). */ +/* ProgramLifetime instances (@scriptc/types) this thread created and no + * released lifetime region freed (scr_region.c): never freed by design, and + * neither is anything they hold (see the exit audit in scr_console.c). */ static SCR_TL long scr_lifetime_objects = 0; long scr_obj_live_count(void) { return scr_live_objects; } long scr_obj_lifetime_count(void) { return scr_lifetime_objects; } @@ -101,6 +101,7 @@ void scr_obj_lifetime_note(const void *obj) { scr_lifetime_objects++; scr_lsan_immortal(obj); } +void scr_obj_lifetime_forget(long n) { scr_lifetime_objects -= n; } #else #undef scr_obj_alloc_note #undef scr_obj_free_note diff --git a/packages/runtime/src/scr_region.c b/packages/runtime/src/scr_region.c new file mode 100644 index 000000000..bb6b6b46a --- /dev/null +++ b/packages/runtime/src/scr_region.c @@ -0,0 +1,848 @@ +/* Counted lifetime regions (@scriptc/experimental LifetimeRegion): the + * allocator, the region table and the release check for ProgramLifetime + * instances. The emitted half (llvm/regions.ts) inlines the allocation fast + * path and the count updates, and supplies the per-class visits and the + * walkers of ordinary classes and records; the contract is in + * scr_runtime.h. + * + * Threads. Every script thread owns its regions, its cursors and its + * default region; region ids are process-wide and never reused, so a region + * handle that reaches another thread (or outlives its region) selects that + * thread's default region and releases nothing. Chunks record their owner: + * worker executables count only references to their own chunks, so a + * published instance that other threads read is never counted by them. + * Publication makes an instance immortal (rc == SIZE_MAX); the release walk + * sees that and pins the region, which is then never freed. + * + * The release check. A walk visits every instance of the region (each + * class's visit reports its reference fields) and, from there, every + * ordinary object it can explain: classes and records through the emitted + * walkers, and the runtime's own containers (arrays, maps and sets, unions, + * closures and their capture boxes, dyn values) through the release entry + * points they store for their elements. Each ordinary object reached is an + * entry: the references the walk found to it, the references it holds into + * the region, and the entries it references. An entry whose count exceeds + * the references found is referenced from outside, and so is everything it + * reaches. The region is freed when the references found from its own + * objects and from the entries nothing outside references add up to its + * counted references. Anything the walk cannot see only leaves references + * unexplained: mistakes keep memory, they never free it. + * + * Sanitized builds (the RC audit lane, AddressSanitizer) never reuse a + * freed chunk: it stays mapped without access, so any use after free + * faults. SCRIPTC_REGION_STATS=1 reports every release attempt on stderr. */ + +#include "scr_runtime.h" + +#include +#include +#include +#include + +#if defined(_WIN32) +#include +#define SCR_RGN_MMAP 0 +#elif defined(__wasi__) || defined(__wasm__) +#define SCR_RGN_MMAP 0 +#else +#include +#define SCR_RGN_MMAP 1 +#endif + +#if defined(SCR_RC_AUDIT) || defined(SCR_LSAN_IMMORTAL) +#define SCR_RGN_POISON 1 +#endif +#ifdef SCR_LSAN_IMMORTAL +/* Chunks are not heap blocks: register them as leak-scan roots so what + * their instances hold is reachable, like the published-object roots. */ +void __lsan_register_root_region(const void *p, size_t size) __attribute__((weak)); +void __lsan_unregister_root_region(const void *p, size_t size) __attribute__((weak)); +#endif + +#define SCR_RGN_HDR ((size_t)64) +#define SCR_RGN_SEG ((size_t)16384) +_Static_assert(sizeof(ScrRgnChunk) <= SCR_RGN_HDR, "the chunk header fits before the first segment"); + +/* Regions start with small pages and switch to huge pages once they span + * this many chunks (a default region at once): small regions then do not + * pay a 2 MiB fault, large ones do not pay 512 small ones. */ +#define SCR_RGN_HUGE_AFTER 4 + +/* Pressure retries: a thread that has condemned regions retries them at a + * region entry once it has allocated this many bytes (or the condemned + * size, if larger) since the last attempt, doubling after each attempt + * that freed nothing. */ +#define SCR_RGN_RETRY_BYTES ((size_t)8 << 20) +#define SCR_RGN_MAX_BACKOFF 6 + +enum { SCR_RGN_LIVE = 1, SCR_RGN_CONDEMNED, SCR_RGN_PINNED, SCR_RGN_LEAKED }; + +typedef struct ScrRgnSeg { + char *start, *end, *limit; /* instances of one class at its stride */ + const ScrRgnClass *cls; + struct ScrRgnSeg *next; +} ScrRgnSeg; + +typedef struct ScrRegion { + uint64_t id; /* 0: the thread's default region */ + int state; + uint32_t depth; /* run() frames active on this region */ + ScrRgnChunk *chunks; + size_t nchunks; + ScrRgnSeg *segs; + ScrRgnSeg **cur; /* per class slot: the segment allocations continue in */ + size_t ncur; + size_t bytes; +} ScrRegion; + +typedef struct { + ScrRegion *prev, *entered; +} ScrRgnFrame; + +/* The release walk's per-thread state (buffers are reused across walks). */ +typedef struct { + void *obj; + int64_t found; /* references to it the walk saw */ + int64_t rrefs; /* references it holds into the region */ + bool black; /* referenced from outside, or reached from such an entry */ +} ScrRgnEntry; + +typedef struct { + size_t parent, child; +} ScrRgnEdge; + +enum { + SCR_RGN_W_EMITTED, + SCR_RGN_W_ARR, + SCR_RGN_W_MAP, + SCR_RGN_W_UNION, + SCR_RGN_W_CLOSURE, + SCR_RGN_W_BOX, + SCR_RGN_W_DYN, +}; + +typedef struct { + size_t entry; + int kind; + void (*walk)(void *, ScrRgnWalk *); +} ScrRgnWork; + +struct ScrRgnWalk { + ScrRegion *region; + int64_t internal; /* references found from the region's own objects */ + size_t src; /* entry index + 1 being walked; 0 = a region object */ + ScrRgnEntry *ents; + size_t nents, cap_ents; + size_t *hash; /* open addressing: entry index + 1 */ + size_t cap_hash; + ScrRgnEdge *edges; + size_t nedges, cap_edges; + ScrRgnWork *work; + size_t nwork, cap_work; +}; + +typedef struct { + ScrRegion *current, *dflt; + ScrRgnCursor **cursors; /* per class slot; NULL until this thread allocates it */ + size_t ncursors; + ScrRegion **regions; /* every non-default region this thread owns and has not freed */ + size_t nregions, cap_regions; + ScrRgnFrame *stack; + size_t depth, cap_stack; + size_t since_attempt; + unsigned backoff; + ScrRgnWalk walk; + bool exit_registered; + double cycle_ms; /* the cycle pass before the next attempt (stats) */ + struct { + size_t created, released, freed, pinned, leaked, attempts; + } stats; +} ScrRgnThread; + +static SCR_TL ScrRgnThread scr_rgn; + +static _Atomic(uint64_t) scr_rgn_next_id = 1; +static _Atomic(int32_t) scr_rgn_next_slot = 0; +static _Atomic(const ScrRgnType *) scr_rgn_types = NULL; +static _Atomic(int) scr_rgn_stats_mode = -1; + +/* The walker index of the emitted type table, by release entry point. + * Built once, by the first walk; the table itself is constant. */ +static _Atomic(int) scr_rgn_index_state = 0; /* 0 none, 1 building, 2 ready */ +static const ScrRgnType **scr_rgn_index; +static size_t scr_rgn_index_cap; + +static bool scr_rgn_stats(void) { + int mode = atomic_load_explicit(&scr_rgn_stats_mode, memory_order_relaxed); + if (mode < 0) { + const char *env = scr_getenv("SCRIPTC_REGION_STATS"); + mode = env != NULL && env[0] != 0 && strcmp(env, "0") != 0; + atomic_store_explicit(&scr_rgn_stats_mode, mode, memory_order_relaxed); + } + return mode != 0; +} + +static void *scr_rgn_grow(void *p, size_t *cap, size_t size, size_t first) { + size_t n = *cap ? *cap * 2 : first; + void *q = realloc(p, n * size); + if (!q) scr_trap("scriptc: out of memory\n"); + *cap = n; + return q; +} + +static void *scr_rgn_self(void) { +#ifdef SCR_WORKERS + return &scr_thread_hot; +#else + return NULL; +#endif +} + +/* ── chunks and segments ─────────────────────────────────────────────── */ + +static ScrRgnChunk *scr_rgn_chunk_map(void) { +#if SCR_RGN_MMAP + char *raw = mmap(NULL, 2 * SCR_RGN_CHUNK, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (raw == MAP_FAILED) scr_trap("scriptc: out of memory\n"); + uintptr_t base = ((uintptr_t)raw + SCR_RGN_CHUNK - 1) & ~(uintptr_t)(SCR_RGN_CHUNK - 1); + if (base > (uintptr_t)raw) munmap(raw, base - (uintptr_t)raw); + uintptr_t end = (uintptr_t)raw + 2 * SCR_RGN_CHUNK; + if (end > base + SCR_RGN_CHUNK) munmap((void *)(base + SCR_RGN_CHUNK), end - (base + SCR_RGN_CHUNK)); + return (ScrRgnChunk *)base; +#elif defined(_WIN32) + void *p = _aligned_malloc(SCR_RGN_CHUNK, SCR_RGN_CHUNK); + if (!p) scr_trap("scriptc: out of memory\n"); + memset(p, 0, SCR_RGN_CHUNK); + return p; +#else + void *p = aligned_alloc(SCR_RGN_CHUNK, SCR_RGN_CHUNK); + if (!p) scr_trap("scriptc: out of memory\n"); + memset(p, 0, SCR_RGN_CHUNK); + return p; +#endif +} + +/* A freed region's chunk. Sanitized builds keep it mapped without access + * and never hand it out again, so a reference that survived faults. */ +static void scr_rgn_chunk_unmap(ScrRgnChunk *c) { +#ifdef SCR_LSAN_IMMORTAL + if (__lsan_unregister_root_region) __lsan_unregister_root_region(c, SCR_RGN_CHUNK); +#endif +#if SCR_RGN_MMAP +#ifdef SCR_RGN_POISON + if (mprotect(c, SCR_RGN_CHUNK, PROT_NONE) != 0) scr_trap("scriptc: mprotect failed\n"); +#else + munmap(c, SCR_RGN_CHUNK); +#endif +#elif defined(SCR_RGN_POISON) + memset(c, 0xdb, SCR_RGN_CHUNK); /* no page protection here: poison, keep */ +#elif defined(_WIN32) + _aligned_free(c); +#else + free(c); +#endif +} + +static ScrRgnChunk *scr_rgn_chunk_new(ScrRegion *r) { + ScrRgnChunk *c = scr_rgn_chunk_map(); +#if SCR_RGN_MMAP && defined(MADV_HUGEPAGE) + if (r->id == 0 || r->nchunks + 1 >= SCR_RGN_HUGE_AFTER) (void)madvise(c, SCR_RGN_CHUNK, MADV_HUGEPAGE); +#endif +#ifdef SCR_LSAN_IMMORTAL + if (__lsan_register_root_region) __lsan_register_root_region(c, SCR_RGN_CHUNK); +#endif + c->count = 0; + c->owner = scr_rgn_self(); + c->region = r; + c->next = r->chunks; + c->bump = (char *)c + SCR_RGN_HDR; + c->found = 0; + r->chunks = c; + r->nchunks++; + return c; +} + +/* The instance stride: the emitted fast path rounds the size the same way. */ +static inline size_t scr_rgn_stride(const ScrRgnClass *cls) { return (cls->size + 7) & ~(size_t)7; } + +static ScrRgnSeg *scr_rgn_seg_new(ScrRegion *r, const ScrRgnClass *cls) { + size_t stride = scr_rgn_stride(cls); + size_t need = stride > SCR_RGN_SEG ? stride : SCR_RGN_SEG; + if (need > SCR_RGN_CHUNK - SCR_RGN_HDR) scr_trap("scriptc: ProgramLifetime instance too large for a region\n"); + ScrRgnChunk *c = r->chunks; + if (!c || (size_t)((char *)c + SCR_RGN_CHUNK - c->bump) < need) c = scr_rgn_chunk_new(r); + ScrRgnSeg *s = malloc(sizeof *s); + if (!s) scr_trap("scriptc: out of memory\n"); + s->start = s->end = c->bump; + s->limit = c->bump + need; + c->bump += need; + s->cls = cls; + s->next = r->segs; + r->segs = s; + r->bytes += need; + scr_rgn.since_attempt += need; + return s; +} + +/* ── the thread's region table ───────────────────────────────────────── */ + +static ScrRegion *scr_rgn_region_new(uint64_t id) { + ScrRegion *r = calloc(1, sizeof *r); + if (!r) scr_trap("scriptc: out of memory\n"); + r->id = id; + r->state = SCR_RGN_LIVE; + return r; +} + +static void scr_rgn_thread_exit(void); + +static ScrRgnThread *scr_rgn_thread(void) { + ScrRgnThread *t = &scr_rgn; + if (!t->dflt) { + t->dflt = scr_rgn_region_new(0); + t->current = t->dflt; + if (!t->exit_registered) { + t->exit_registered = true; + scr_atexit(scr_rgn_thread_exit); + } + } + return t; +} + +static ScrRegion *scr_rgn_lookup(ScrRgnThread *t, double id) { + if (!(id >= 1)) return NULL; + for (size_t i = 0; i < t->nregions; i++) + if ((double)t->regions[i]->id == id) return t->regions[i]; + return NULL; +} + +static void scr_rgn_forget(ScrRgnThread *t, ScrRegion *r) { + for (size_t i = 0; i < t->nregions; i++) + if (t->regions[i] == r) { + t->regions[i] = t->regions[--t->nregions]; + return; + } +} + +/* The current region's cursors write their bump pointers back into its + * segments; then `r`'s segments load into the cursors. */ +static void scr_rgn_switch(ScrRgnThread *t, ScrRegion *r) { + ScrRegion *cur = t->current; + if (cur == r) return; + for (size_t i = 0; i < t->ncursors; i++) { + ScrRgnCursor *c = t->cursors[i]; + if (!c) continue; + if (c->bump) cur->cur[i]->end = c->bump; + ScrRgnSeg *s = i < r->ncur ? r->cur[i] : NULL; + c->bump = s ? s->end : NULL; + c->limit = s ? s->limit : NULL; + } + t->current = r; +} + +static int32_t scr_rgn_slot(ScrRgnClass *cls) { + int32_t slot = atomic_load_explicit((_Atomic(int32_t) *)&cls->slot, memory_order_acquire); + if (slot >= 0) return slot; + int32_t fresh = atomic_fetch_add_explicit(&scr_rgn_next_slot, 1, memory_order_relaxed); + int32_t expected = -1; + if (atomic_compare_exchange_strong_explicit((_Atomic(int32_t) *)&cls->slot, &expected, fresh, + memory_order_acq_rel, memory_order_acquire)) + return fresh; + return expected; /* another thread numbered it first; `fresh` stays unused */ +} + +void *scr_region_alloc(ScrRgnCursor *cursor, ScrRgnClass *cls) { + ScrRgnThread *t = scr_rgn_thread(); + size_t slot = (size_t)scr_rgn_slot(cls); + if (slot >= t->ncursors) { + size_t n = slot + 16; + t->cursors = realloc(t->cursors, n * sizeof *t->cursors); + if (!t->cursors) scr_trap("scriptc: out of memory\n"); + for (size_t i = t->ncursors; i < n; i++) t->cursors[i] = NULL; + t->ncursors = n; + } + t->cursors[slot] = cursor; + ScrRegion *r = t->current; + if (slot >= r->ncur) { + size_t n = slot + 16; + r->cur = realloc(r->cur, n * sizeof *r->cur); + if (!r->cur) scr_trap("scriptc: out of memory\n"); + for (size_t i = r->ncur; i < n; i++) r->cur[i] = NULL; + r->ncur = n; + } + ScrRgnSeg *s = r->cur[slot]; + if (s && cursor->bump) s->end = cursor->bump; + size_t stride = scr_rgn_stride(cls); + if (!s || (size_t)(s->limit - s->end) < stride) { + s = scr_rgn_seg_new(r, cls); + r->cur[slot] = s; + } + char *o = s->end; + s->end += stride; + cursor->bump = s->end; + cursor->limit = s->limit; + scr_rgn_chunk_of(o)->count++; /* the new instance's first reference */ + return o; +} + +/* ── the release walk ────────────────────────────────────────────────── */ + +static size_t scr_rgn_hash_of(const void *p, size_t cap) { + return (size_t)(((uintptr_t)p >> 3) * 0x9E3779B97F4A7C15ull) & (cap - 1); +} + +static size_t scr_rgn_find(ScrRgnWalk *w, const void *p) { + size_t j = scr_rgn_hash_of(p, w->cap_hash); + while (w->hash[j] && w->ents[w->hash[j] - 1].obj != p) j = (j + 1) & (w->cap_hash - 1); + return j; +} + +static void scr_rgn_rehash(ScrRgnWalk *w) { + size_t cap = w->cap_hash ? w->cap_hash * 2 : (size_t)1 << 12; + free(w->hash); + w->hash = calloc(cap, sizeof *w->hash); + if (!w->hash) scr_trap("scriptc: out of memory\n"); + w->cap_hash = cap; + for (size_t i = 0; i < w->nents; i++) w->hash[scr_rgn_find(w, w->ents[i].obj)] = i + 1; +} + +/* One more reference to the ordinary object `p`, from the object being + * walked; queues `p`'s own walk when it is first reached. */ +static void scr_rgn_enter(ScrRgnWalk *w, void *p, int kind, void (*walk)(void *, ScrRgnWalk *)) { + if (!p || *(size_t *)p == SIZE_MAX) return; /* immortal: never freed, not walked */ + if ((w->nents + 1) * 2 > w->cap_hash) scr_rgn_rehash(w); + size_t j = scr_rgn_find(w, p); + size_t idx = w->hash[j]; + if (!idx) { + if (w->nents == w->cap_ents) w->ents = scr_rgn_grow(w->ents, &w->cap_ents, sizeof *w->ents, 4096); + w->ents[w->nents] = (ScrRgnEntry){p, 0, 0, false}; + idx = w->hash[j] = ++w->nents; + if (w->nwork == w->cap_work) w->work = scr_rgn_grow(w->work, &w->cap_work, sizeof *w->work, 4096); + w->work[w->nwork++] = (ScrRgnWork){idx - 1, kind, walk}; + } + w->ents[idx - 1].found++; + if (w->src) { + if (w->nedges == w->cap_edges) w->edges = scr_rgn_grow(w->edges, &w->cap_edges, sizeof *w->edges, 4096); + w->edges[w->nedges++] = (ScrRgnEdge){w->src - 1, idx - 1}; + } +} + +void scr_region_ref(ScrRgnWalk *w, void *obj) { + if (!obj) return; + ScrRgnChunk *c = scr_rgn_chunk_of(obj); + if (c->region != w->region) return; /* another region's: counted there */ + c->found++; + if (w->src) w->ents[w->src - 1].rrefs++; + else w->internal++; +} + +static const ScrRgnType *scr_rgn_type_of(void (*release)(void *)) { + if (atomic_load_explicit(&scr_rgn_index_state, memory_order_acquire) != 2 || !scr_rgn_index_cap) return NULL; + size_t j = scr_rgn_hash_of((const void *)(uintptr_t)release, scr_rgn_index_cap); + while (scr_rgn_index[j]) { + if (scr_rgn_index[j]->release == release) return scr_rgn_index[j]; + j = (j + 1) & (scr_rgn_index_cap - 1); + } + return NULL; +} + +static void scr_rgn_index_build(void) { + int state = 0; + if (!atomic_compare_exchange_strong(&scr_rgn_index_state, &state, 1)) { + while (atomic_load_explicit(&scr_rgn_index_state, memory_order_acquire) != 2) { + } + return; + } + const ScrRgnType *types = atomic_load_explicit(&scr_rgn_types, memory_order_acquire); + size_t n = 0; + for (const ScrRgnType *ty = types; ty && ty->release; ty++) n++; + size_t cap = 16; + while (cap < n * 2) cap *= 2; + scr_rgn_index = calloc(cap, sizeof *scr_rgn_index); + if (!scr_rgn_index) scr_trap("scriptc: out of memory\n"); + scr_rgn_index_cap = cap; + for (const ScrRgnType *ty = types; ty && ty->release; ty++) { + size_t j = scr_rgn_hash_of((const void *)(uintptr_t)ty->release, cap); + while (scr_rgn_index[j] && scr_rgn_index[j]->release != ty->release) j = (j + 1) & (cap - 1); + scr_rgn_index[j] = ty; + } + atomic_store_explicit(&scr_rgn_index_state, 2, memory_order_release); +} + +void scr_region_slot(ScrRgnWalk *w, void *obj, void (*release)(void *)) { + if (!obj || !release) return; + if (release == scr_arr_release_v) scr_rgn_enter(w, obj, SCR_RGN_W_ARR, NULL); + else if (release == scr_map_release_v) scr_rgn_enter(w, obj, SCR_RGN_W_MAP, NULL); + else if (release == scr_union_release_v) scr_rgn_enter(w, obj, SCR_RGN_W_UNION, NULL); + else if (release == scr_closure_release_v) scr_rgn_enter(w, obj, SCR_RGN_W_CLOSURE, NULL); + else if (release == scr_dyn_release_v) scr_rgn_enter(w, obj, SCR_RGN_W_DYN, NULL); + else { + const ScrRgnType *ty = scr_rgn_type_of(release); + if (!ty) return; /* not walkable: its references stay unexplained */ + if (!ty->walk) scr_region_ref(w, obj); + else scr_rgn_enter(w, obj, SCR_RGN_W_EMITTED, ty->walk); + } +} + +static void *scr_rgn_slot_ptr(uint64_t slot) { + void *p; + memcpy(&p, &slot, sizeof p); + return p; +} + +static void scr_rgn_walk_elem(ScrRgnWalk *w, const ScrArr *a, uint64_t slot) { + if (a->elem == SCR_ELEM_REF) scr_region_slot(w, scr_rgn_slot_ptr(slot), a->ops ? a->ops->release : NULL); + else if (a->elem == SCR_ELEM_ARR) scr_rgn_enter(w, scr_rgn_slot_ptr(slot), SCR_RGN_W_ARR, NULL); +} + +static void scr_rgn_walk_arr(ScrRgnWalk *w, ScrArr *a) { + if (a->elem != SCR_ELEM_REF && a->elem != SCR_ELEM_ARR) return; + size_t dense = a->len < a->cap ? a->len : a->cap; + for (size_t i = 0; i < dense; i++) + if (!a->present || a->present[i] == SCR_ARR_VALUE) scr_rgn_walk_elem(w, a, a->data[i]); + if (!a->ext) return; + for (size_t i = 0; i < a->ext->sparse_len; i++) + if (a->ext->sparse[i].state == SCR_ARR_VALUE) scr_rgn_walk_elem(w, a, a->ext->sparse[i].slot); + for (size_t i = 0; i < a->ext->prop_len; i++) + if (a->ext->props[i].state == SCR_ARR_VALUE) scr_rgn_walk_elem(w, a, a->ext->props[i].slot); + scr_rgn_enter(w, a->ext->metadata, SCR_RGN_W_DYN, NULL); +} + +static void scr_rgn_walk_map(ScrRgnWalk *w, ScrMap *m) { + bool keyref = m->key_kind == SCR_MAP_KEY_REF || m->key_kind == SCR_MAP_KEY_UNION_REF || + m->key_kind == SCR_MAP_KEY_UNION_VALUE; + bool keydyn = m->key_kind == SCR_MAP_KEY_DYN; + bool valref = m->val_kind == SCR_MAP_VAL_REF; + if (!keyref && !keydyn && !valref) return; + for (size_t i = 0; i < m->nentries; i++) { + const ScrMapEntry *e = &m->entries[i]; + if (!e->hash) continue; /* tombstone */ + if (keyref) scr_region_slot(w, scr_rgn_slot_ptr(e->key), m->key_release); + else if (keydyn) scr_rgn_enter(w, scr_rgn_slot_ptr(e->key), SCR_RGN_W_DYN, NULL); + if (valref) scr_region_slot(w, scr_rgn_slot_ptr(e->val), m->val_release); + } +} + +static void scr_rgn_walk_box(ScrRgnWalk *w, ScrBox *b) { + void *p = scr_rgn_slot_ptr(b->slot); + switch (b->kind) { + case SCR_BOX_OBJ: scr_region_slot(w, p, b->obj_release); break; + case SCR_BOX_ARR: scr_rgn_enter(w, p, SCR_RGN_W_ARR, NULL); break; + case SCR_BOX_FUNC: scr_rgn_enter(w, p, SCR_RGN_W_CLOSURE, NULL); break; + default: break; /* strings and scalars hold no references */ + } +} + +static void scr_rgn_walk_closure(ScrRgnWalk *w, ScrClosure *c) { + for (size_t i = 0; i < c->ncaps; i++) scr_rgn_enter(w, c->caps[i], SCR_RGN_W_BOX, NULL); + scr_rgn_enter(w, c->props, SCR_RGN_W_BOX, NULL); +} + +static void scr_rgn_walk_dyn(ScrRgnWalk *w, ScrDyn *d) { + switch (d->kind) { + case SCR_DYN_ARR: + for (size_t i = 0; i < d->v.arr.len; i++) + if (!d->v.arr.presence || d->v.arr.presence[i]) scr_rgn_enter(w, d->v.arr.items[i], SCR_RGN_W_DYN, NULL); + scr_rgn_enter(w, d->v.arr.properties, SCR_RGN_W_DYN, NULL); + break; + case SCR_DYN_OBJ: + /* source_identity is owned through its access callback: not walked. */ + for (size_t i = 0; i < d->v.obj.len; i++) { + const ScrDynEntry *e = &d->v.obj.entries[i]; + if (e->accessor) { + scr_rgn_enter(w, e->getter, SCR_RGN_W_DYN, NULL); + scr_rgn_enter(w, e->setter, SCR_RGN_W_DYN, NULL); + } else scr_rgn_enter(w, e->value, SCR_RGN_W_DYN, NULL); + } + break; + case SCR_DYN_FUNC: scr_rgn_enter(w, d->v.fn.clo, SCR_RGN_W_CLOSURE, NULL); break; + case SCR_DYN_TYPED_REF: + scr_region_slot(w, d->v.typed_ref.ptr, d->v.typed_ref.release); + scr_rgn_enter(w, d->v.typed_ref.materialized, SCR_RGN_W_DYN, NULL); + for (ScrDynTypedCast *cast = d->v.typed_ref.casts; cast; cast = cast->next) + scr_region_slot(w, cast->ptr, cast->release); + break; + case SCR_DYN_PROXY: + scr_rgn_enter(w, d->v.proxy.target, SCR_RGN_W_DYN, NULL); + scr_rgn_enter(w, d->v.proxy.handler, SCR_RGN_W_DYN, NULL); + break; + default: break; /* scalars, strings, bytes; handles and promises are not walked */ + } + scr_rgn_enter(w, d->prototype, SCR_RGN_W_DYN, NULL); + scr_rgn_enter(w, d->symbol_properties, SCR_RGN_W_DYN, NULL); + scr_rgn_enter(w, d->symbol_keys, SCR_RGN_W_DYN, NULL); +} + +static void scr_rgn_drain(ScrRgnWalk *w) { + while (w->nwork) { + ScrRgnWork job = w->work[--w->nwork]; + void *p = w->ents[job.entry].obj; + w->src = job.entry + 1; + switch (job.kind) { + case SCR_RGN_W_EMITTED: job.walk(p, w); break; + case SCR_RGN_W_ARR: scr_rgn_walk_arr(w, p); break; + case SCR_RGN_W_MAP: scr_rgn_walk_map(w, p); break; + case SCR_RGN_W_UNION: { + ScrUnion *u = p; + if (u->arm_release) scr_region_slot(w, scr_rgn_slot_ptr(u->slot), u->arm_release); + break; + } + case SCR_RGN_W_CLOSURE: scr_rgn_walk_closure(w, p); break; + case SCR_RGN_W_BOX: scr_rgn_walk_box(w, p); break; + case SCR_RGN_W_DYN: scr_rgn_walk_dyn(w, p); break; + } + } + w->src = 0; +} + +/* Trial deletion over the entries: marks the ones referenced from outside + * and everything they reach, and returns the references into the region + * held by the rest. */ +static int64_t scr_rgn_settle(ScrRgnWalk *w, size_t *black_out) { + size_t n = w->nents; + size_t *start = calloc(n + 1, sizeof *start); + size_t *child = malloc((w->nedges + 1) * sizeof *child); + size_t *stack = malloc((n + 1) * sizeof *stack); + if (!start || !child || !stack) scr_trap("scriptc: out of memory\n"); + for (size_t e = 0; e < w->nedges; e++) start[w->edges[e].parent + 1]++; + for (size_t i = 0; i < n; i++) start[i + 1] += start[i]; + for (size_t e = 0; e < w->nedges; e++) child[start[w->edges[e].parent]++] = w->edges[e].child; + for (size_t i = n; i > 0; i--) start[i] = start[i - 1]; /* the fill moved each start to its end */ + start[0] = 0; + size_t top = 0; + for (size_t i = 0; i < n; i++) { + ScrRgnEntry *x = &w->ents[i]; + if (*(size_t *)x->obj != (size_t)x->found) { + x->black = true; + stack[top++] = i; + } + } + while (top) { + size_t i = stack[--top]; + for (size_t k = start[i]; k < start[i + 1]; k++) { + ScrRgnEntry *c = &w->ents[child[k]]; + if (c->black) continue; + c->black = true; + stack[top++] = child[k]; + } + } + int64_t sum = 0; + size_t black = 0; + for (size_t i = 0; i < n; i++) { + if (w->ents[i].black) black++; + else sum += w->ents[i].rrefs; + } + free(start); + free(child); + free(stack); + *black_out = black; + return sum; +} + +static int64_t scr_rgn_counted(const ScrRegion *r) { + int64_t n = 0; + for (const ScrRgnChunk *c = r->chunks; c; c = c->next) n += c->count; + return n; +} + +static double scr_rgn_ms(uint64_t from, uint64_t to) { return (double)(to - from) / 1e6; } + +/* The region's segments hold instances up to their `end`; a region that + * is not current has every cursor written back (scr_rgn_switch). */ +#define SCR_RGN_EACH(r, s, o) \ + for (ScrRgnSeg *s = (r)->segs; s; s = s->next) \ + for (char *o = s->start; o < s->end; o += scr_rgn_stride(s->cls)) + +static void scr_rgn_free_region(ScrRgnThread *t, ScrRegion *r) { + for (ScrRgnChunk *c = r->chunks, *next; c; c = next) { + next = c->next; + scr_rgn_chunk_unmap(c); + } + for (ScrRgnSeg *s = r->segs, *next; s; s = next) { + next = s->next; + free(s); + } + free(r->cur); + scr_rgn_forget(t, r); + free(r); +} + +/* One release attempt; true when `r` was freed. */ +static bool scr_rgn_try_free(ScrRgnThread *t, ScrRegion *r) { + bool stats = scr_rgn_stats(); + uint64_t t0 = stats ? scr_hrtime_ns() : 0; + double cycles = t->cycle_ms; + t->cycle_ms = 0; + t->stats.attempts++; + ScrRgnWalk *w = &t->walk; + w->region = r; + w->internal = 0; + w->src = 0; + w->nents = w->nedges = w->nwork = 0; + if (w->hash) memset(w->hash, 0, w->cap_hash * sizeof *w->hash); + for (ScrRgnChunk *c = r->chunks; c; c = c->next) c->found = 0; + size_t objects = 0; + SCR_RGN_EACH(r, s, o) { + if (*(size_t *)o == SIZE_MAX) { + /* Published: other threads may hold it, uncounted. */ + r->state = SCR_RGN_PINNED; + t->stats.pinned++; + if (stats) fprintf(stderr, "scriptc region %llu: pinned: it holds a published object\n", (unsigned long long)r->id); + return false; + } + s->cls->visit(o, w); + objects++; + } + scr_rgn_drain(w); + int64_t counted = scr_rgn_counted(r); + size_t black = 0; + int64_t explained = w->internal + scr_rgn_settle(w, &black); + uint64_t t1 = stats ? scr_hrtime_ns() : 0; + if (explained != counted) { + if (stats) + fprintf(stderr, + "scriptc region %llu: kept: %lld references counted, %lld explained; %zu of %zu owned objects referenced from outside; cycles %.1f ms, walk %.1f ms\n", + (unsigned long long)r->id, (long long)counted, (long long)explained, black, w->nents, cycles, scr_rgn_ms(t0, t1)); + return false; + } + /* Free: drop weak entries keyed by the instances, then release every + * reference field. The owned objects tear down through their own + * releases; the cycle collector frees the owned cycles. */ + if (scr_weak_dispose_hook) SCR_RGN_EACH(r, s, o) scr_weak_dispose(o); + SCR_RGN_EACH(r, s, o) s->cls->release_fields(o); + int64_t left = scr_rgn_counted(r); + if (left != 0) { + scr_collect_cycles(); + left = scr_rgn_counted(r); + } + if (left != 0) { + /* The walk explained references that do not exist: the instances are + * already released, so keep their memory forever. */ + r->state = SCR_RGN_LEAKED; + t->stats.leaked++; + fprintf(stderr, "scriptc region %llu: %lld references left after release; memory kept\n", + (unsigned long long)r->id, (long long)left); +#ifdef SCR_RC_AUDIT + abort(); +#endif + return false; + } +#ifdef SCR_RC_AUDIT + scr_obj_lifetime_forget((long)objects); +#endif + size_t bytes = r->nchunks * SCR_RGN_CHUNK, nchunks = r->nchunks; + uint64_t id = r->id; + scr_rgn_free_region(t, r); + t->stats.freed++; + if (stats) + fprintf(stderr, + "scriptc region %llu: freed %zu objects, %zu owned objects, %zu chunks (%.1f MiB); cycles %.1f ms, walk %.1f ms, free %.1f ms\n", + (unsigned long long)id, objects, w->nents, nchunks, (double)bytes / 1048576.0, cycles, scr_rgn_ms(t0, t1), + scr_rgn_ms(t1, scr_hrtime_ns())); + return true; +} + +static bool scr_rgn_due(const ScrRegion *r, const ScrRgnThread *t) { + return r->state == SCR_RGN_CONDEMNED && r->depth == 0 && r != t->current; +} + +/* Tries every condemned region that is not in use. */ +static void scr_rgn_retry(ScrRgnThread *t) { + bool any = false; + for (size_t i = 0; i < t->nregions && !any; i++) any = scr_rgn_due(t->regions[i], t); + t->since_attempt = 0; + if (!any) return; + if (atomic_load_explicit(&scr_rgn_index_state, memory_order_acquire) != 2) scr_rgn_index_build(); + /* Dead cycles of ordinary objects can hold counted references into a + * condemned region until the collector frees them. */ + uint64_t t0 = scr_rgn_stats() ? scr_hrtime_ns() : 0; + scr_collect_cycles(); + if (t0) t->cycle_ms = scr_rgn_ms(t0, scr_hrtime_ns()); + bool freed = false; + for (size_t i = 0; i < t->nregions;) { + ScrRegion *r = t->regions[i]; + if (scr_rgn_due(r, t) && scr_rgn_try_free(t, r)) { + freed = true; + i = 0; /* the table shrank; freeing may have unblocked another region */ + continue; + } + i++; + } + if (freed) t->backoff = 0; + else if (t->backoff < SCR_RGN_MAX_BACKOFF) t->backoff++; +} + +/* ── program-facing entry points ─────────────────────────────────────── */ + +double scr_region_create(const ScrRgnType *types) { + const ScrRgnType *expected = NULL; + atomic_compare_exchange_strong(&scr_rgn_types, &expected, types); + ScrRgnThread *t = scr_rgn_thread(); + scr_rgn_retry(t); + uint64_t id = atomic_fetch_add_explicit(&scr_rgn_next_id, 1, memory_order_relaxed); + if (t->nregions == t->cap_regions) t->regions = scr_rgn_grow(t->regions, &t->cap_regions, sizeof *t->regions, 16); + t->regions[t->nregions++] = scr_rgn_region_new(id); + t->stats.created++; + return (double)id; +} + +void scr_region_enter(double id) { + ScrRgnThread *t = scr_rgn_thread(); + ScrRegion *r = scr_rgn_lookup(t, id); + if (!r || r->state != SCR_RGN_LIVE) r = t->dflt; /* released regions take no new objects */ + if (t->nregions && t->since_attempt >= (SCR_RGN_RETRY_BYTES << t->backoff)) scr_rgn_retry(t); + if (t->depth == t->cap_stack) t->stack = scr_rgn_grow(t->stack, &t->cap_stack, sizeof *t->stack, 16); + t->stack[t->depth++] = (ScrRgnFrame){t->current, r}; + r->depth++; + scr_rgn_switch(t, r); +} + +void scr_region_leave(void) { + ScrRgnThread *t = &scr_rgn; + if (t->depth == 0) scr_trap("scriptc: internal error: unbalanced region leave\n"); + ScrRgnFrame f = t->stack[--t->depth]; + f.entered->depth--; + scr_rgn_switch(t, f.prev); +} + +void scr_region_release(double id) { + ScrRgnThread *t = scr_rgn_thread(); + ScrRegion *r = scr_rgn_lookup(t, id); + if (!r || r->state != SCR_RGN_LIVE) return; + r->state = SCR_RGN_CONDEMNED; + t->stats.released++; + scr_rgn_retry(t); +} + +/* Thread (or process) exit: the regions can no longer be released, but + * their instances may be published, so the chunks stay; the bookkeeping + * goes. */ +static void scr_rgn_thread_exit(void) { + ScrRgnThread *t = &scr_rgn; + if (scr_rgn_stats()) { + size_t condemned = 0; + for (size_t i = 0; i < t->nregions; i++) condemned += t->regions[i]->state == SCR_RGN_CONDEMNED; + fprintf(stderr, "scriptc regions: %zu created, %zu released, %zu freed, %zu pinned, %zu kept\n", + t->stats.created, t->stats.released, t->stats.freed, t->stats.pinned, condemned); + } + ScrRegion *all[1] = {t->dflt}; + for (size_t i = 0; i <= t->nregions; i++) { + ScrRegion *r = i < t->nregions ? t->regions[i] : all[0]; + if (!r) continue; + for (ScrRgnSeg *s = r->segs, *next; s; s = next) { + next = s->next; + free(s); + } + free(r->cur); + free(r); + } + free(t->regions); + free(t->cursors); + free(t->stack); + free(t->walk.ents); + free(t->walk.hash); + free(t->walk.edges); + free(t->walk.work); + memset(t, 0, sizeof *t); + t->exit_registered = true; /* the cleanup ran; a late allocation keeps its memory */ +} diff --git a/packages/runtime/src/scr_region.test.c b/packages/runtime/src/scr_region.test.c new file mode 100644 index 000000000..43bebb387 --- /dev/null +++ b/packages/runtime/src/scr_region.test.c @@ -0,0 +1,460 @@ +/* Counted lifetime regions (scr_region.c), driven the way emitted code + * drives them: a ProgramLifetime class with the inline allocation fast + * path, chunk-count retain/release and a visit, an ordinary class with an + * emitted walker, and the runtime's own containers around them. Each mode + * prints its checks to stdout; SCRIPTC_REGION_STATS=1 reports every release + * attempt on stderr, which the driver (scr_region.test.ts) compares. */ +#undef NDEBUG +#include "scr_runtime.h" +#include +#include +#include +#include +#include +#if !defined(_WIN32) +#include +#include +#endif + +#ifdef SCR_WORKERS +#define TEST_TL _Thread_local +#else +#define TEST_TL +#endif + +/* ── the "emitted" ProgramLifetime class ───────────────────────────── */ + +typedef struct Plain Plain; +typedef struct Node { + size_t rc; + struct Node *parent; + ScrArr *kids; + ScrMap *names; + ScrUnion *tag; + ScrClosure *fn; + ScrDyn *data; + Plain *plain; + struct Node *other; + double value; +} Node; + +static void *node_retain(void *o) { + if (o) { + ScrRgnChunk *c = scr_rgn_chunk_of(o); +#ifdef SCR_WORKERS + if (c->owner == (void *)&scr_thread_hot) +#endif + c->count++; + } + return o; +} + +static void node_release(void *o) { + if (o) { + ScrRgnChunk *c = scr_rgn_chunk_of(o); +#ifdef SCR_WORKERS + if (c->owner == (void *)&scr_thread_hot) +#endif + c->count--; + } +} + +static const ScrArrOps node_ops = {node_retain, node_release, NULL}; + +/* ── the "emitted" ordinary class (acyclic, plain heap block) ────────── */ + +struct Plain { + size_t rc; + Node *node; + Plain *next; +}; + +static void plain_release(void *p) { + Plain *o = p; + if (!o || o->rc == SIZE_MAX) return; + if (--o->rc) return; + node_release(o->node); + plain_release(o->next); + free(o); +} + +static Plain *plain_new(Node *owned, Plain *next) { + Plain *o = calloc(1, sizeof *o); + o->rc = 1; + o->node = owned; + o->next = next; + return o; +} + +static void plain_walk(void *p, ScrRgnWalk *w) { + Plain *o = p; + scr_region_ref(w, o->node); + scr_region_slot(w, o->next, plain_release); +} + +/* A container entry point the type table does not list. */ +static void opaque_release(void *p) { node_release(p); } + +static const ScrRgnType types[] = {{node_release, NULL}, {plain_release, plain_walk}, {NULL, NULL}}; + +static void node_visit(void *p, ScrRgnWalk *w) { + Node *n = p; + scr_region_ref(w, n->parent); + scr_region_slot(w, n->kids, scr_arr_release_v); + scr_region_slot(w, n->names, scr_map_release_v); + scr_region_slot(w, n->tag, scr_union_release_v); + scr_region_slot(w, n->fn, scr_closure_release_v); + scr_region_slot(w, n->data, scr_dyn_release_v); + scr_region_slot(w, n->plain, plain_release); + scr_region_ref(w, n->other); +} + +static void node_release_fields(void *p) { + Node *n = p; + node_release(n->parent); + scr_arr_release(n->kids); + scr_map_release(n->names); + scr_union_release(n->tag); + scr_closure_release(n->fn); + scr_dyn_release(n->data); + plain_release(n->plain); + node_release(n->other); +} + +static ScrRgnClass node_cls = {sizeof(Node), node_visit, node_release_fields, "Node", -1}; +static TEST_TL ScrRgnCursor node_cursor; + +/* The emitted constructor: the inline fast path, else the runtime. */ +static Node *node_new(double value) { + Node *n; + char *p = node_cursor.bump; + if (p && p + sizeof(Node) <= node_cursor.limit) { + node_cursor.bump = p + sizeof(Node); + scr_rgn_chunk_of(p)->count++; + n = (Node *)p; + } else n = scr_region_alloc(&node_cursor, &node_cls); + n->rc = (size_t)1 << (sizeof(size_t) * 8 - 1); /* programLifetimeRc */ + n->value = value; +#ifdef SCR_RC_AUDIT + scr_obj_lifetime_note(n); +#endif + return n; +} + +static double region(void) { return scr_region_create(types); } + +static ScrArr *node_array(void) { return scr_arr_new_ops(&node_ops, 0); } + +/* A closure whose first capture holds the closure itself (a cycle the + * collector frees) and whose second holds `held` through `release`. */ +static ScrClosure *cyclic_closure(Node *held, void (*release)(void *)) { + ScrClosure *c = scr_closure_new(NULL, 2); + c->caps[0] = scr_box_new(SCR_BOX_FUNC); + scr_box_set_ref(c->caps[0], scr_closure_retain(c)); + c->caps[1] = scr_box_new_obj(node_retain, release, NULL); + scr_box_set_ref(c->caps[1], held); + return c; +} + +static void *disposed[4096]; +static size_t ndisposed; +static void note_disposed(void *obj) { + if (ndisposed < sizeof disposed / sizeof *disposed) disposed[ndisposed++] = obj; +} + +static bool was_disposed(const void *obj) { + for (size_t i = 0; i < ndisposed; i++) + if (disposed[i] == obj) return true; + return false; +} + +/* A graph exercising every walked shape, rooted at the returned node. */ +static Node *build_graph(int n) { + Node *root = node_new(0); + root->kids = node_array(); + root->names = scr_map_new(SCR_MAP_KEY_STR, SCR_MAP_VAL_REF, node_retain, node_release, NULL); + ScrMap *set = scr_map_new_typed(SCR_MAP_KEY_REF, SCR_MAP_VAL_F64, node_retain, node_release, NULL, NULL, NULL, NULL); + ScrDyn *list = scr_dyn_new_arr(); + for (int i = 0; i < n; i++) { + Node *child = node_new(i + 1); + child->parent = node_retain(root); + char key[16]; + int len = snprintf(key, sizeof key, "k%d", i); + ScrStr *k = scr_str_new(key, (size_t)len); + scr_map_set_str_ref(root->names, k, node_retain(child)); + scr_str_release(k); + scr_map_set_ref_f64(set, child, i); + child->tag = scr_union_new_ref(1, node_retain(child), node_retain, node_release, NULL); + child->plain = plain_new(node_retain(root), i % 2 ? plain_new(node_retain(child), NULL) : NULL); + ScrDyn *ref = scr_dyn_new_typed_ref(child, node_retain, node_release, "Node", 4, NULL, NULL); + scr_dyn_arr_push(list, ref); + ScrArr *nested = scr_arr_new(SCR_ELEM_ARR, 0); + ScrArr *inner = node_array(); + scr_arr_push_ref(inner, node_retain(child)); + scr_arr_push_ref(nested, inner); + ScrBox *box = scr_box_new(SCR_BOX_ARR); + scr_box_set_ref(box, nested); + ScrClosure *fn = scr_closure_new(NULL, 1); + fn->caps[0] = box; + child->fn = fn; + scr_arr_push_ref(root->kids, child); /* moves the constructor's reference */ + } + ScrDyn *obj = scr_dyn_new_obj(); + scr_dyn_obj_set(obj, "list", 4, list); + scr_dyn_obj_set(obj, "set", 3, + scr_dyn_new_typed_ref(set, scr_map_retain_v, scr_map_release_v, "Set", 9, NULL, NULL)); + scr_map_release(set); + root->data = obj; + return root; +} + +static void check_owned_graph(void) { + double r = region(); + scr_region_enter(r); + Node *root = build_graph(100); + scr_region_leave(); + Node *nodes[101] = {root}; + for (size_t i = 0; i < 100; i++) nodes[i + 1] = (Node *)(uintptr_t)root->kids->data[i]; + node_release(root); + void (*hook)(void *) = scr_weak_dispose_hook; + scr_weak_dispose_hook = note_disposed; + scr_region_release(r); + scr_weak_dispose_hook = hook; + for (size_t i = 0; i < 101; i++) assert(was_disposed(nodes[i])); /* weak entries keyed by them go */ + puts("an owned graph of every walked shape is freed with its region"); +} + +static void check_outside_local(void) { + double r = region(); + scr_region_enter(r); + Node *keep = build_graph(10); + scr_region_leave(); + scr_region_release(r); /* `keep` is a counted outside reference */ + assert(keep->value == 0 && keep->kids->len == 10); + node_release(keep); + scr_region_release(region()); /* creation retries the condemned region */ + puts("an outside reference keeps a region until it is dropped"); +} + +static void check_outside_container(void) { + double r = region(); + scr_region_enter(r); + Node *root = build_graph(3); + Node *shared = node_new(7); + scr_region_leave(); + ScrArr *outside = node_array(); + scr_arr_push_ref(outside, shared); + ScrArr *kids = scr_arr_retain(root->kids); /* owned by the region and held outside */ + node_release(root); + scr_region_release(r); + scr_arr_release(kids); + scr_region_release(region()); + assert(((Node *)(uintptr_t)outside->data[0])->value == 7); + scr_arr_release(outside); + scr_region_release(region()); + puts("ordinary objects referenced from outside keep what they reach"); +} + +static void check_cycles(void) { + double r = region(); + scr_region_enter(r); + Node *root = build_graph(2); + Node *held = node_new(1); + root->fn = cyclic_closure(held, node_release); /* an owned cycle */ + Node *dead = node_new(2); + scr_region_leave(); + ScrClosure *garbage = cyclic_closure(dead, node_release); + scr_closure_release(garbage); /* a dead cycle outside the region holds `dead` */ + node_release(root); + scr_region_release(r); + puts("owned cycles and dead outside cycles do not keep a region"); +} + +static void check_unwalked(void) { + double r = region(); + scr_region_enter(r); + Node *root = build_graph(2); + Node *hidden = node_new(3); + ScrClosure *fn = cyclic_closure(hidden, opaque_release); + scr_closure_release(root->fn); + root->fn = fn; + scr_region_leave(); + node_release(root); + scr_region_release(r); /* the walk cannot explain `hidden`'s reference */ + /* What the program does once it no longer needs the shape: */ + ScrClosure *drop = root->fn; + root->fn = NULL; + scr_closure_release(drop); + scr_region_release(region()); + puts("references the walk cannot explain keep a region"); +} + +static void check_in_use(void) { + double r = region(); + double other = region(); + scr_region_enter(r); + Node *a = node_new(1); + scr_region_enter(other); + Node *b = node_new(2); + b->other = node_retain(a); /* other → r */ + scr_region_leave(); + Node *c = node_new(3); + scr_region_release(r); /* current: deferred */ + scr_region_leave(); + assert(a->value == 1 && c->value == 3); + node_release(a); + node_release(c); + node_release(b); + scr_region_release(region()); /* `other` still refers into r */ + scr_region_release(other); + puts("regions in use or referenced by other regions are kept"); +} + +static void check_cursors(void) { + double r = region(); + double s = region(); + Node *keep[3000]; + int n = 0; + for (int round = 0; round < 3; round++) { + scr_region_enter(r); + for (int i = 0; i < 500; i++, n++) keep[n] = node_new(n + 1); + scr_region_enter(s); + for (int i = 0; i < 500; i++, n++) keep[n] = node_new(n + 1); + scr_region_leave(); + scr_region_leave(); + } + for (int i = 0; i < n; i++) assert(keep[i]->value == i + 1); + for (int i = 0; i < n; i++) node_release(keep[i]); + scr_region_release(r); + scr_region_release(s); + puts("allocation continues per class across region switches"); +} + +static void check_poison(void) { +#if defined(SCR_RC_AUDIT) && !defined(_WIN32) + double r = region(); + scr_region_enter(r); + Node *stale = node_new(5); + scr_region_leave(); + node_release(stale); + scr_region_release(r); + fflush(NULL); + pid_t pid = fork(); + if (pid == 0) { + freopen("/dev/null", "w", stderr); /* the sanitizer's report */ + volatile double v = stale->value; + (void)v; + _exit(0); + } + int status = 0; + assert(waitpid(pid, &status, 0) == pid); + assert(!(WIFEXITED(status) && WEXITSTATUS(status) == 0)); + puts("sanitized builds fault on any access to a freed region"); +#endif +} + +/* ── what is never freed: the default region, published objects ────── */ + +static void check_default_region(void) { + double r = region(); + scr_region_enter(r); + Node *a = node_new(1); + scr_region_leave(); + scr_region_release(r); + node_release(a); + scr_region_enter(r); /* a released region takes no new objects */ + Node *d = node_new(4); + scr_region_enter(1e15); /* never created */ + Node *e = node_new(5); + scr_region_leave(); + scr_region_leave(); + assert(scr_rgn_chunk_of(d) == scr_rgn_chunk_of(e) && scr_rgn_chunk_of(d) != scr_rgn_chunk_of(a)); + scr_region_release(-1); + scr_region_release(region()); + node_release(d); + node_release(e); + puts("objects outside every live region go to the default region"); +} + + +static void check_published(void) { + double r = region(); + scr_region_enter(r); + Node *root = build_graph(2); + scr_region_leave(); + root->rc = SIZE_MAX; /* what publication does to every object it reaches */ + scr_region_release(r); + scr_region_release(region()); + assert(root->value == 0); + puts("a published object pins its region"); +} + +#ifdef SCR_WORKERS +enum { THREADS = 4 }; +static Node *published_node; +static atomic_uint finished; + +static void *thread_main(void *arg) { + uint64_t id = (uint64_t)(uintptr_t)arg; + scr_context_enter(id); + scr_init(); + scr_lib_init(0, NULL); + for (int round = 0; round < 20; round++) { + double r = region(); + scr_region_enter(r); + Node *root = build_graph(50); + for (int i = 0; i < 50; i++) { + /* A foreign, published instance is read and referenced uncounted. */ + Node *seen = node_retain(published_node); + assert(seen->value == 42); + node_release(seen); + } + scr_region_leave(); + node_release(root); + scr_region_release(r); + } + scr_region_release(1); /* another thread's region: not ours to release */ + atomic_fetch_add(&finished, 1); + scr_context_cleanup(); + return NULL; +} + +static void check_threads(void) { + double r = region(); + scr_region_enter(r); + published_node = node_new(42); + scr_region_leave(); + published_node->rc = SIZE_MAX; + pthread_t threads[THREADS]; + for (unsigned i = 0; i < THREADS; i++) + assert(!pthread_create(&threads[i], NULL, thread_main, (void *)(uintptr_t)(i + 1))); + for (unsigned i = 0; i < THREADS; i++) assert(!pthread_join(threads[i], NULL)); + assert(atomic_load(&finished) == THREADS); + scr_region_release(r); + puts("threads free their own regions; a shared published object pins its region"); +} +#endif + +int main(int argc, char **argv) { + scr_init(); + scr_lib_init(argc, argv); + const char *mode = argc > 1 ? argv[1] : "regions"; + if (!strcmp(mode, "regions")) { + check_owned_graph(); + check_outside_local(); + check_outside_container(); + check_cycles(); + check_unwalked(); + check_in_use(); + check_cursors(); + check_poison(); + } else if (!strcmp(mode, "retained")) { + check_default_region(); + check_published(); +#ifdef SCR_WORKERS + } else if (!strcmp(mode, "threads")) { + check_threads(); +#endif + } else return 2; + return 0; +} diff --git a/packages/runtime/src/scr_region.test.ts b/packages/runtime/src/scr_region.test.ts new file mode 100644 index 000000000..d436c8d9b --- /dev/null +++ b/packages/runtime/src/scr_region.test.ts @@ -0,0 +1,157 @@ +import { execFile } from "node:child_process"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { promisify } from "node:util"; +import { describe, expect, test } from "vitest"; +import { EXECUTABLE_RUNTIME_SOURCES } from "../../compiler/src/backend/native-toolchain.js"; + +const exec = promisify(execFile); + +/* Every release attempt of a mode, in order, as " " + * (SCRIPTC_REGION_STATS=1), and the per-thread summary lines. */ +const OWNED_ATTEMPTS = [ + "1 freed", + "2 kept", + "2 freed", + "3 freed", + "4 kept", + "4 kept", + "4 kept", + "5 freed", + "4 kept", + "4 freed", + "6 freed", + "7 freed", + "8 kept", + "8 freed", + "9 freed", + "10 kept", + "10 kept", + "12 freed", + "10 kept", + "10 kept", + "11 freed", + "10 freed", + "13 freed", + "14 freed", +]; +const OWNED_STDOUT = [ + "an owned graph of every walked shape is freed with its region", + "an outside reference keeps a region until it is dropped", + "ordinary objects referenced from outside keep what they reach", + "owned cycles and dead outside cycles do not keep a region", + "references the walk cannot explain keep a region", + "regions in use or referenced by other regions are kept", + "allocation continues per class across region switches", +]; +const RETAINED_ATTEMPTS = ["1 kept", "1 freed", "2 freed", "3 pinned", "4 freed"]; +const RETAINED_STDOUT = [ + "objects outside every live region go to the default region", + "a published object pins its region", +]; + +function attempts(stderr: string): { attempts: string[]; summaries: string[]; other: string[] } { + const out = { attempts: [] as string[], summaries: [] as string[], other: [] as string[] }; + for (const line of stderr.split("\n")) { + if (line === "") continue; + const m = /^scriptc region (\d+): (freed|kept|pinned)\b/.exec(line); + if (m) out.attempts.push(`${m[1]} ${m[2]}`); + else if (line.startsWith("scriptc regions: ")) out.summaries.push(line); + else out.other.push(line); + } + return out; +} + +// The size-class allocator build, the sanitized lane's build (ASan and the +// RC audit, whose exit check then proves that freeing a region freed every +// ordinary object it owned, and where freed chunks fault), and worker +// executables under ThreadSanitizer. +describe.skipIf(process.platform === "win32")("counted lifetime regions", () => { + const build = async (dir: string, name: string, flags: string[]): Promise => { + const bin = join(dir, name); + await exec("clang", [ + "-std=c11", + "-O1", + "-pthread", + "-Wno-deprecated-declarations", + ...(process.platform === "linux" ? ["-D_GNU_SOURCE"] : []), + ...flags, + ...["scr_region.test.c", ...EXECUTABLE_RUNTIME_SOURCES].map((source) => + join(import.meta.dirname, source), + ), + ...(process.platform === "linux" ? ["-lm"] : []), + "-o", + bin, + ]); + return bin; + }; + const run = async (bin: string, mode: string) => + exec(bin, [mode], { + timeout: 60000, + env: { + ...process.env, + SCRIPTC_REGION_STATS: "1", + ASAN_OPTIONS: "halt_on_error=1", + TSAN_OPTIONS: "halt_on_error=1", + UBSAN_OPTIONS: "halt_on_error=1", + }, + }); + + test.each([ + ["size-class", ["-fsanitize=undefined"]], + ["sanitized", ["-fsanitize=address,undefined", "-DSCR_RC_AUDIT"]], + ])("release walks, keeps and frees regions (%s)", async (lane, flags) => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-region-")); + try { + const bin = await build(dir, "region", flags); + const owned = await run(bin, "regions"); + const sanitized = lane === "sanitized"; + expect(owned.stdout.trim().split("\n")).toEqual([ + ...OWNED_STDOUT, + ...(sanitized ? ["sanitized builds fault on any access to a freed region"] : []), + ]); + const ownedStats = attempts(owned.stderr); + expect(ownedStats.attempts).toEqual([...OWNED_ATTEMPTS, ...(sanitized ? ["15 freed"] : [])]); + const n = sanitized ? 15 : 14; + expect(ownedStats.summaries).toEqual([ + `scriptc regions: ${n} created, ${n} released, ${n} freed, 0 pinned, 0 kept`, + ]); + // No RC audit line: every region was freed, so the strict exit audit ran. + expect(ownedStats.other).toEqual([]); + + const retained = await run(bin, "retained"); + expect(retained.stdout.trim().split("\n")).toEqual(RETAINED_STDOUT); + const retainedStats = attempts(retained.stderr); + expect(retainedStats.attempts).toEqual(RETAINED_ATTEMPTS); + expect(retainedStats.summaries).toEqual([ + "scriptc regions: 4 created, 4 released, 3 freed, 1 pinned, 0 kept", + ]); + expect(retainedStats.other).toEqual( + sanitized ? ["scriptc RC audit skipped: 5 program-lifetime object(s) never freed"] : [], + ); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); + + test("worker threads free their own regions under ThreadSanitizer", async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-region-")); + try { + const bin = await build(dir, "region-threads", ["-fsanitize=thread", "-DSCR_WORKERS"]); + const result = await run(bin, "threads"); + expect(result.stdout).toBe( + "threads free their own regions; a shared published object pins its region\n", + ); + const stats = attempts(result.stderr); + expect(stats.attempts.filter((a) => a.endsWith(" freed"))).toHaveLength(80); + expect(stats.attempts.filter((a) => !a.endsWith(" freed"))).toEqual(["1 pinned"]); + expect(stats.summaries).toEqual( + Array(4).fill("scriptc regions: 20 created, 20 released, 20 freed, 0 pinned, 0 kept"), + ); + expect(stats.other).toEqual([]); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); +}); diff --git a/packages/runtime/src/scr_runtime.h b/packages/runtime/src/scr_runtime.h index 4456f8e8f..dc9b3a33a 100644 --- a/packages/runtime/src/scr_runtime.h +++ b/packages/runtime/src/scr_runtime.h @@ -5453,6 +5453,101 @@ long scr_obj_lifetime_count(void); #define scr_obj_alloc_note() ((void)0) #define scr_obj_free_note() ((void)0) #endif +#ifdef SCR_RC_AUDIT +/* A released lifetime region freed `n` program-lifetime instances. */ +void scr_obj_lifetime_forget(long n); +#endif + +/* ── counted lifetime regions (scr_region.c) ─────────────────────────── + * Programs that construct a LifetimeRegion (@scriptc/experimental) allocate + * every ProgramLifetime instance in the thread's CURRENT region instead of + * the small-object allocator. A region owns SCR_RGN_CHUNK-aligned chunks; + * each class packs its instances into segments of a chunk, so a region's + * objects are enumerable without a header. Instead of a per-object count, + * the class's retain and release adjust the chunk's first word (`count`), + * which therefore holds the number of references to the chunk's objects. + * + * release() condemns a region. A condemned region is freed only when a walk + * of its objects, and of the ordinary objects they own, explains every + * counted reference: trial deletion restricted to the region. References + * the walk cannot see (locals, globals, objects outside the walked graph, + * shapes it does not know) stay unexplained and keep the region. Freeing + * releases every reference field of every object, which tears the owned + * ordinary objects down through their normal paths, runs the cycle + * collector for the owned cycles, and then requires every count to be 0. + * Region 0 of each thread (the default) is never freed: programs that do + * not use regions keep ProgramLifetime's behavior. + * + * The layouts below are an ABI with the emitted code (llvm/regions.ts): + * the inline allocation fast path bumps the class's ScrRgnCursor and + * increments the chunk count, and the retain/release bodies address the + * count (and, in worker executables, the owner) at the chunk base. */ +#define SCR_RGN_CHUNK_SHIFT 21 +#define SCR_RGN_CHUNK ((size_t)1 << SCR_RGN_CHUNK_SHIFT) + +typedef struct ScrRgnWalk ScrRgnWalk; + +/* One per ProgramLifetime class, emitted as a mutable global: `slot` is + * assigned process-wide on the first allocation (-1 until then). */ +typedef struct ScrRgnClass { + size_t size; /* instance size (the stride rounds it to 8) */ + void (*visit)(void *obj, ScrRgnWalk *w); /* reports each reference field */ + void (*release_fields)(void *obj); /* releases each reference field */ + const char *name; + int32_t slot; +} ScrRgnClass; + +/* The calling thread's allocation cursor for one class in its current + * region: emitted per class (thread-local in worker executables). NULL + * fields send the next allocation to scr_region_alloc. */ +typedef struct ScrRgnCursor { + char *bump, *limit; +} ScrRgnCursor; + +/* The emitted type table (terminated by a NULL release): the release entry + * point a container stores for a type (shapes.ts vAdapters), and the walker + * of its objects. A NULL walker marks a ProgramLifetime class. */ +typedef struct ScrRgnType { + void (*release)(void *); + void (*walk)(void *obj, ScrRgnWalk *w); +} ScrRgnType; + +typedef struct ScrRgnChunk { + int64_t count; /* word 0: references to this chunk's objects */ + void *owner; /* word 1: the owning thread (worker executables) */ + struct ScrRegion *region; + struct ScrRgnChunk *next; + char *bump; /* the next free segment byte */ + int64_t found; /* references the last walk saw into this chunk */ +} ScrRgnChunk; +_Static_assert(offsetof(ScrRgnChunk, count) == 0 && offsetof(ScrRgnChunk, owner) == sizeof(int64_t), + "llvm/regions.ts addresses the chunk count at 0 and the owner at 8"); + +static inline ScrRgnChunk *scr_rgn_chunk_of(const void *obj) { + return (ScrRgnChunk *)((uintptr_t)obj & ~(uintptr_t)(SCR_RGN_CHUNK - 1)); +} + +/* The allocation slow path: a zeroed instance of `cls` in the current + * region, its first reference counted. Refills `cursor`. */ +void *scr_region_alloc(ScrRgnCursor *cursor, ScrRgnClass *cls); +/* new LifetimeRegion(): a fresh region id (ids are never reused). `types` + * is the program's constant type table. Retries condemned regions. */ +double scr_region_create(const ScrRgnType *types); +/* run(fn): makes region `id` current until the matching leave. An id this + * thread does not own (another thread's, or a freed region's) selects the + * thread's default region. Calls nest. */ +void scr_region_enter(double id); +void scr_region_leave(void); +/* release(): condemns region `id` and tries to free it. */ +void scr_region_release(double id); + +/* Walk callbacks for emitted visits and walkers. scr_region_ref reports a + * reference to a ProgramLifetime instance; scr_region_slot reports a + * reference of the type whose container release entry point is `release` + * (unknown entry points are not walked, which keeps their references + * unexplained). Both tolerate NULL. */ +void scr_region_ref(ScrRgnWalk *w, void *obj); +void scr_region_slot(ScrRgnWalk *w, void *obj, void (*release)(void *)); /* ── async: promises, fibers, event loop ──────────────────────────── * Async function bodies run on stackful fibers; `await` parks the fiber on