From c903daefcf2babb9d4e9aff53be909ce10b63d84 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 8 Oct 2026 11:54:45 -0500 Subject: [PATCH] Back off cycle scans for long-lived object graphs (#741) --- .../test/ts7/baselines/order-parity.json | 6 + packages/runtime/src/scr_cycle.c | 288 +++++++++++------- packages/runtime/src/scr_json.test.c | 6 +- packages/runtime/src/scr_runtime.h | 20 +- packages/runtime/test/test_cycle.c | 104 +++++++ tests/corpus/cycle-generations-long-lived.ts | 115 +++++++ 6 files changed, 425 insertions(+), 114 deletions(-) create mode 100644 tests/corpus/cycle-generations-long-lived.ts diff --git a/packages/compiler/test/ts7/baselines/order-parity.json b/packages/compiler/test/ts7/baselines/order-parity.json index c4ef6c01f..832626afc 100644 --- a/packages/compiler/test/ts7/baselines/order-parity.json +++ b/packages/compiler/test/ts7/baselines/order-parity.json @@ -14901,6 +14901,12 @@ "/tests/diagnostics/cycle-var-early/main.ts" ], "diags": [] + }, + "/tests/corpus/cycle-generations-long-lived.ts": { + "order": [ + "/tests/corpus/cycle-generations-long-lived.ts" + ], + "diags": [] } } } diff --git a/packages/runtime/src/scr_cycle.c b/packages/runtime/src/scr_cycle.c index 3faaed5c5..7bf5d1a71 100644 --- a/packages/runtime/src/scr_cycle.c +++ b/packages/runtime/src/scr_cycle.c @@ -39,12 +39,17 @@ * from it drags in every node below, and none of it is ever garbage. * * So the collector is generational, in the shape CPython uses for this - * same algorithm. Each header carries a `gen`; a pass names the oldest - * generation it will walk and SKIPS every object above it, exactly as it - * skips NULL and immortals. Objects that survive a pass are promoted, so - * the next nursery pass never walks them again: the cost of a nursery pass - * is proportional to what has been allocated since the last one, not to - * the live heap. + * same algorithm: nursery, mature and old. Each header carries a `gen`; a + * pass names the oldest generation it will walk and SKIPS every object above + * it, exactly as it skips NULL and immortals. Objects that survive a pass are + * promoted one generation, so the next pass at a lower level never walks + * them again: the cost of a nursery pass is proportional to what has been + * allocated since the last one, and the cost of a mature pass to what has + * survived a nursery pass since the last mature one — not to the live heap. + * The third generation is what keeps a long-lived structure out of the walk + * that reclaims medium-lived garbage: objects that die after surviving one + * pass (scratch structures built for a phase of work, say) are reclaimed by + * mature passes that never touch the data that has been live for a while. * * Restricting the walk stays sound because trial deletion is already * conservative in the right direction. An edge from a skipped (older) @@ -53,41 +58,57 @@ * answer the collector gives for a reference held by the stack. Edges the * other way are simply not followed; the older object was not a candidate * for freeing in this pass. What this gives up is cycles that SPAN - * generations: those are invisible to a nursery pass and are found by the - * full pass, which walks every generation and so is exactly the old - * algorithm. + * generations: those are invisible to a restricted pass and are found by + * the full pass, which walks every generation and so is exactly the + * single-generation algorithm. * * Scheduling. One counter drives the release path: candidates buffered - * since the last pass. When it trips, the pass runs at the nursery level - * unless either full-pass condition holds: - * heap growth the live cycle-headered heap is a fraction past its size - * after the last full pass — the standard mature-generation - * rule, which catches garbage made from fresh allocation; - * mature backlog the mature candidate buffer has reached a fraction of the + * since the last pass. When it trips, the pass runs at the highest level + * whose condition holds: + * full, growth the live cycle-headered heap is a fraction past its size + * after the last full pass. The fraction starts at 1/4 and + * ADAPTS to what full passes find: a growth-triggered full + * pass that reclaims few OLD objects relative to the growth + * that triggered it, twice in a row, doubles the fraction + * (up to 1/2), and one that does reclaim them restores 1/4. + * Garbage younger than the old generation does not count: + * the mature level reclaims it without walking the old heap. + * This is what stops a large retained structure from being + * re-walked at every quarter of growth while it is building + * or while the garbage is all young. + * full, backlog the old candidate buffer has reached a fraction of the * live heap. This one is not redundant: live data that is * unlinked INTO a dead cycle grows no counter at all (it * was tallied when it was allocated) and is invisible to a - * nursery pass, so without it a program that churns its + * restricted pass, so without it a program that churns its * long-lived structures would never collect anything. But - * a mature object is also buffered whenever a temporary + * an old object is also buffered whenever a temporary * reference to it is dropped, so merely READING a live * structure (walking a tree, say) refills the buffer with - * objects that are not garbage. A full pass that frees - * almost nothing therefore doubles this fraction, and a - * productive one restores it. - * scheduled age a mature candidate has waited through a nursery's worth - * of event-loop checkpoints. Candidate COUNT cannot bound - * the garbage behind one root, and an idle heap does not - * trip growth, so this keeps a sparse mature backlog from + * objects that are not garbage. A pass that frees almost + * nothing therefore doubles this fraction, and a productive + * one restores it. + * mature the same two rules one level down, with a fixed 1/4 + * growth fraction measured from the last mature-or-full + * pass and a backlog rule over the mature buffer. + * scheduled age a mature or old candidate has waited through a nursery's + * worth of event-loop checkpoints. Candidate COUNT cannot + * bound the garbage behind one root, and an idle heap does + * not trip growth, so this keeps a sparse backlog from * floating forever without putting a full walk on every * turn. - * The first two bound collection work to a constant factor of mutator work — - * one heap-sized walk per live/N candidates — while the scheduled age is the - * liveness backstop, limiting its forced full walks to one per nursery's worth - * of checkpoints while mature roots wait. That is the trade: a fixed root - * threshold bounds floating garbage tightly and pays unbounded time for it; - * this bounds ordinary mutator-triggered work and lets garbage float in - * proportion, but not forever. + * The growth and backlog rules bound collection work to a constant factor of + * mutator work — one heap-sized walk per live/N candidates or per growth of + * live/N — while the scheduled age is the liveness backstop, limiting its + * forced full walks to one per nursery's worth of checkpoints while older + * roots wait. That is the trade: a fixed root threshold bounds floating + * garbage tightly and pays unbounded time for it; this bounds ordinary + * mutator-triggered work and lets garbage float in proportion, but not + * forever. Peak memory stays bounded the same way: young garbage floats by + * at most a quarter of the heap before a mature pass, and old garbage by at + * most half the heap's size after the last full pass, and by more than a + * quarter only once two full passes in a row have shown that little of the + * growth was old garbage. */ #include "scr_runtime.h" @@ -154,8 +175,11 @@ void scr_rc_destroy(void *obj, void (*destroy)(void *)) { } } -/* Live cycle-headered objects — what the full-pass trigger watches. */ +/* Live cycle-headered objects — what the growth triggers watch — and the + * running count of OLD objects freed, which is how a full pass measures what + * only it could have reclaimed. */ static SCR_TL size_t scr_cyc_live = 0; +static SCR_TL size_t scr_cyc_old_freed = 0; void *scr_cyc_alloc(size_t size, ScrTraceFn trace, ScrCycFreeFn free_fn) { if (size > SIZE_MAX - sizeof(ScrCycHdr)) scr_cyc_oom(); @@ -172,6 +196,7 @@ void *scr_cyc_alloc(size_t size, ScrTraceFn trace, ScrCycFreeFn free_fn) { void scr_cyc_free(void *obj) { scr_weak_dispose(obj); scr_cyc_live--; + if (scr_cyc_hdr(obj)->gen == SCR_CYC_OLD) scr_cyc_old_freed++; free(scr_cyc_hdr(obj)); } @@ -223,23 +248,36 @@ static SCR_TL ScrVec scr_white; static SCR_TL ScrVec scr_xgen; /* Objects above this generation are invisible to the pass in flight. */ -static SCR_TL unsigned scr_gen_limit = SCR_CYC_MATURE; +static SCR_TL unsigned scr_gen_limit = SCR_CYC_OLD; static size_t scr_cyc_pass(unsigned gen_limit); /* ── when to collect ──────────────────────────────────────────────────── */ #define SCR_CYC_NURSERY_CANDIDATES 256 /* nursery trigger */ -#define SCR_CYC_FULL_GROWTH_DIV 4 /* full pass per +1/4 of live heap */ -#define SCR_CYC_FULL_FLOOR 4096 /* ...but not below this many objects */ +#define SCR_CYC_GROWTH_DIV 4 /* growth pass per +1/4 of live heap */ +#define SCR_CYC_GROWTH_FLOOR 4096 /* ...but not below this many objects */ -/* Live count as of the end of the last full pass. */ +/* Live count as of the end of the last full pass, and of the last pass that + * walked the mature generation (mature or full). */ static SCR_TL size_t scr_cyc_live_after_full = 0; +static SCR_TL size_t scr_cyc_live_after_mature = 0; -/* Doublings of the mature-backlog threshold earned by consecutive full passes - * that found almost nothing to free (see scr_cyc_mature_threshold). */ +/* Doublings of the full-pass growth fraction earned by consecutive + * growth-triggered full passes that reclaimed little of the old generation + * (see scr_cyc_full_pass). Capped at a fraction of 1/2: the heap may grow + * at most by half past its size after the last full pass before the next. + * The streak counts those unproductive passes since the last productive one; + * the fraction only grows once two have run in a row. */ +#define SCR_CYC_GROWTH_MAX_SHIFT 1 +static SCR_TL unsigned scr_cyc_growth_shift = 0; +static SCR_TL unsigned scr_cyc_growth_streak = 0; + +/* Doublings of each generation's backlog threshold earned by consecutive + * passes that found almost nothing to free (see scr_cyc_backlog_threshold). + * Only the mature and old entries are used. */ #define SCR_CYC_BACKLOG_MAX_SHIFT 8 -static SCR_TL unsigned scr_cyc_backlog_shift = 0; +static SCR_TL unsigned scr_cyc_backlog_shift[SCR_CYC_NGENS]; static size_t scr_cyc_nursery_threshold(void) { static SCR_TL size_t cached = 0; @@ -251,79 +289,122 @@ static size_t scr_cyc_nursery_threshold(void) { return cached; } -/* Buffered candidates across both generations, and the count at which the +/* Buffered candidates across every generation, and the count at which the * release path next collects. Keeping a running total (rather than summing * the buffers) is what holds the hot path to one load and one compare, as * it was when there was a single buffer. The trigger is re-armed at the end * of every pass to "whatever survived, plus a nursery's worth": without that - * hysteresis a large mature buffer — which a nursery pass cannot drain — + * hysteresis a large older buffer — which a nursery pass cannot drain — * would re-arm on every single release. Both start at zero, so the first * release runs one trivial pass that arms them properly. */ static SCR_TL size_t scr_cyc_nbuffered = 0; static SCR_TL size_t scr_cyc_trigger = 0; -/* Scheduled nursery passes for which at least one mature root was waiting. - * A full pass resets the age; with no mature backlog there is nothing to age. */ +/* Scheduled nursery passes for which at least one mature or old root was + * waiting. A full pass resets the age; with no such backlog there is nothing + * to age. */ static SCR_TL size_t scr_cyc_scheduled_mature_age = 0; static size_t scr_cyc_buffered(void) { - return scr_roots[SCR_CYC_NURSERY].n + scr_roots[SCR_CYC_MATURE].n; + return scr_roots[SCR_CYC_NURSERY].n + scr_roots[SCR_CYC_MATURE].n + + scr_roots[SCR_CYC_OLD].n; } -/* A full pass is due once the live heap has grown a fraction past what it - * was when the last one finished. Since a pass resets the baseline to the - * live count it leaves behind, an unproductive full pass cannot re-trigger - * itself — the next one waits for another fraction of growth. */ -static bool scr_cyc_full_due(void) { - size_t base = scr_cyc_live_after_full; - if (base < SCR_CYC_FULL_FLOOR) base = SCR_CYC_FULL_FLOOR; - return scr_cyc_live > base + base / SCR_CYC_FULL_GROWTH_DIV; +static bool scr_cyc_older_backlog(void) { + return scr_roots[SCR_CYC_MATURE].n != 0 || scr_roots[SCR_CYC_OLD].n != 0; } -/* Cold half of the release path: pick the generation and collect. +static size_t scr_cyc_growth_base(size_t live_after) { + return live_after < SCR_CYC_GROWTH_FLOOR ? SCR_CYC_GROWTH_FLOOR : live_after; +} + +/* A pass is due once the live heap has grown a fraction past what it was + * when the last pass at that level finished. Since a pass resets its + * baseline to the live count it leaves behind, an unproductive pass cannot + * re-trigger itself — the next one waits for another fraction of growth. */ +static bool scr_cyc_grown(size_t live_after, unsigned shift) { + size_t base = scr_cyc_growth_base(live_after); + return scr_cyc_live > base + ((base / SCR_CYC_GROWTH_DIV) << shift); +} + +/* The backlog rule for one generation's candidate buffer. * - * Heap growth is not the only thing that owes a full pass. Live data that - * TURNS INTO garbage — a mature structure unlinked into a dead cycle — - * grows no counter at all: those objects were already tallied in - * scr_cyc_live when they were allocated, so scr_cyc_full_due stays false - * forever, and a nursery pass cannot see them because they are mature. Left - * at that, a program that churns its long-lived structures without - * allocating would never collect anything. So the mature buffer's own size - * is the second full-pass trigger, at a fraction of the live heap: that - * bounds uncollected mature candidates proportionally and keeps the - * amortized cost linear (one heap-sized walk per live/N candidates). + * Heap growth is not the only thing that owes a pass. Live data that TURNS + * INTO garbage — an older structure unlinked into a dead cycle — grows no + * counter at all: those objects were already tallied in scr_cyc_live when + * they were allocated, so the growth rules stay false forever, and a + * restricted pass cannot see them because they sit above its level. Left at + * that, a program that churns its long-lived structures without allocating + * would never collect anything. So a buffer's own size is the second trigger + * for its level, at a fraction of the live heap: that bounds uncollected + * candidates proportionally and keeps the amortized cost linear (one walk + * per live/N candidates). * * Re-buffered live objects cannot be told apart from new garbage until a * pass walks them, and a program that keeps traversing one live structure - * buffers it again after every pass. Each full pass that frees almost none - * of its candidates doubles the threshold, so repeated unproductive walks - * thin out geometrically; once it exceeds the mature population this - * trigger rests, leaving heap growth, the scheduled age and exit to collect. - * That only delays garbage made from existing data: replacing it requires - * allocation, which grows the heap and trips the growth trigger. The first - * productive full pass restores the original fraction. */ -static size_t scr_cyc_mature_threshold(void) { - size_t t = scr_cyc_live / SCR_CYC_FULL_GROWTH_DIV; + * buffers it again after every pass. Each backlog-level pass that frees + * almost none of its candidates doubles the threshold, so repeated + * unproductive walks thin out geometrically; once it exceeds the population + * this trigger rests, leaving heap growth, the scheduled age and exit to + * collect. That only delays garbage made from existing data: replacing it + * requires allocation, which grows the heap and trips a growth trigger. The + * first productive pass at that level restores the original fraction. */ +static size_t scr_cyc_backlog_threshold(unsigned gen) { + size_t t = scr_cyc_live / SCR_CYC_GROWTH_DIV; if (t < SCR_CYC_NURSERY_CANDIDATES) t = SCR_CYC_NURSERY_CANDIDATES; - return t > (SIZE_MAX >> scr_cyc_backlog_shift) ? SIZE_MAX : t << scr_cyc_backlog_shift; + unsigned shift = scr_cyc_backlog_shift[gen]; + return t > (SIZE_MAX >> shift) ? SIZE_MAX : t << shift; } -/* A scheduled full pass, adapting the backlog threshold to its yield. The - * explicit sweep (scr_collect_cycles) is not a scheduling decision and - * leaves the threshold alone. Productive means at least one object freed - * per FULL_GROWTH_DIV candidates; a pass with no candidates says nothing. */ -static void scr_cyc_full_pass(void) { - size_t freed = scr_cyc_pass(SCR_CYC_MATURE); +/* A scheduled pass at the mature or old level, adapting that level's backlog + * threshold to its yield. The explicit sweep (scr_collect_cycles) is not a + * scheduling decision and leaves the thresholds alone. Productive means at + * least one object freed per GROWTH_DIV candidates; a pass with no + * candidates says nothing. */ +static void scr_cyc_level_pass(unsigned gen) { + size_t freed = scr_cyc_pass(gen); if (scr_cands.n == 0) return; - if (freed * SCR_CYC_FULL_GROWTH_DIV >= scr_cands.n) - scr_cyc_backlog_shift = 0; - else if (scr_cyc_backlog_shift < SCR_CYC_BACKLOG_MAX_SHIFT) - scr_cyc_backlog_shift++; + if (freed * SCR_CYC_GROWTH_DIV >= scr_cands.n) + scr_cyc_backlog_shift[gen] = 0; + else if (scr_cyc_backlog_shift[gen] < SCR_CYC_BACKLOG_MAX_SHIFT) + scr_cyc_backlog_shift[gen]++; +} + +/* A scheduled full pass. When heap growth triggered it, also adapt the + * growth fraction to what it reclaimed from the OLD generation — the one + * part of its work no restricted pass could have done. Weighing that against + * the growth since the last full pass keeps full passes frequent while old + * structures are being discarded (their garbage is what floats until the + * next full pass) and spaces them out while the heap is growing with live or + * young data, where each walk of the retained heap buys almost nothing. + * Backing off only after two unproductive passes in a row keeps a program + * that discards old structures at a steady rate on the base schedule: its + * passes alternate between finding the garbage just aged into the old + * generation and finding it still younger, and a single miss says little. */ +static void scr_cyc_full_pass(bool by_growth) { + size_t base = scr_cyc_growth_base(scr_cyc_live_after_full); + size_t growth = scr_cyc_live > base ? scr_cyc_live - base : 0; + size_t old_freed_before = scr_cyc_old_freed; + scr_cyc_level_pass(SCR_CYC_OLD); + if (!by_growth) return; + size_t reclaimed = scr_cyc_old_freed - old_freed_before; + if (reclaimed * SCR_CYC_GROWTH_DIV >= growth) { + scr_cyc_growth_shift = 0; + scr_cyc_growth_streak = 0; + } else if (++scr_cyc_growth_streak >= 2 + && scr_cyc_growth_shift < SCR_CYC_GROWTH_MAX_SHIFT) { + scr_cyc_growth_shift++; + } } static void scr_cyc_collect_due(void) { - if (scr_cyc_full_due() || scr_roots[SCR_CYC_MATURE].n >= scr_cyc_mature_threshold()) - scr_cyc_full_pass(); + if (scr_cyc_grown(scr_cyc_live_after_full, scr_cyc_growth_shift)) + scr_cyc_full_pass(true); + else if (scr_roots[SCR_CYC_OLD].n >= scr_cyc_backlog_threshold(SCR_CYC_OLD)) + scr_cyc_full_pass(false); + else if (scr_cyc_grown(scr_cyc_live_after_mature, 0) + || scr_roots[SCR_CYC_MATURE].n >= scr_cyc_backlog_threshold(SCR_CYC_MATURE)) + scr_cyc_level_pass(SCR_CYC_MATURE); else scr_cyc_pass(SCR_CYC_NURSERY); } @@ -338,11 +419,11 @@ void scr_cyc_collect_scheduled(void) { scr_cyc_scheduled_mature_age = 0; return; } - if (scr_roots[SCR_CYC_MATURE].n == 0) { + if (!scr_cyc_older_backlog()) { scr_cyc_scheduled_mature_age = 0; } else if (++scr_cyc_scheduled_mature_age >= scr_cyc_nursery_threshold()) { - scr_cyc_full_pass(); + scr_cyc_full_pass(false); return; } scr_cyc_collect_due(); @@ -363,7 +444,7 @@ void scr_cyc_on_dead(void *obj) { scr_cyc_nbuffered--; /* A death must not consume room reserved for a future candidate. */ scr_cyc_trigger--; - if (h->gen == SCR_CYC_MATURE && b->n == 0) + if (h->gen != SCR_CYC_NURSERY && !scr_cyc_older_backlog()) scr_cyc_scheduled_mature_age = 0; } @@ -427,7 +508,7 @@ static void scr_sb_visit(void *child, void *ctx) { static void scr_scan_black(void *obj, uintptr_t depth) { ScrCycHdr *h = scr_cyc_hdr(obj); h->color = SCR_CYC_BLACK; - if (h->gen < SCR_CYC_MATURE) scr_cyc_push(&scr_promote, obj); + if (h->gen < SCR_CYC_OLD) scr_cyc_push(&scr_promote, obj); if (depth == SCR_CYC_WALK_DEPTH) { scr_cyc_push(&scr_restore_pending, obj); return; @@ -541,7 +622,7 @@ static size_t scr_cyc_pass(unsigned gen_limit) { * object re-retained since buffering is black. Drain the buffers before * walking: tracing only adjusts counts and worklists, so nothing can * re-buffer underneath us. Candidates ABOVE the limit keep their slots - * and wait for a full pass. */ + * and wait for a pass at their own level. */ scr_cands.n = 0; for (unsigned g = 0; g <= gen_limit; g++) { for (size_t i = 0; i < scr_roots[g].n; i++) { @@ -583,7 +664,7 @@ static size_t scr_cyc_pass(unsigned gen_limit) { * `gen` still holds the values the walk filtered on (promotion below is * what changes them). A full pass skips no edge, so it pins nothing. */ scr_xgen.n = 0; - if (gen_limit < SCR_CYC_MATURE) { + if (gen_limit < SCR_CYC_OLD) { for (size_t i = 0; i < scr_white.n; i++) { void *obj = scr_white.v[i]; scr_cyc_hdr(obj)->trace(obj, scr_xg_pin_visit, NULL); @@ -591,10 +672,12 @@ static size_t scr_cyc_pass(unsigned gen_limit) { } /* Every phase that consults SCR_CYC_SKIP has run, so the recorded - * survivors can graduate now. They are externally referenced, so none of - * them is in the white set about to be freed. */ - for (size_t i = 0; i < scr_promote.n; i++) - scr_cyc_hdr(scr_promote.v[i])->gen = SCR_CYC_MATURE; + * survivors can graduate now, one generation each. They are externally + * referenced, so none of them is in the white set about to be freed. */ + for (size_t i = 0; i < scr_promote.n; i++) { + ScrCycHdr *h = scr_cyc_hdr(scr_promote.v[i]); + if (h->gen < SCR_CYC_OLD) h->gen++; + } scr_promote.n = 0; /* A restricted pass may have spared a candidate only because the edge @@ -605,13 +688,14 @@ static size_t scr_cyc_pass(unsigned gen_limit) { * root, and a dead cycle whose members all got spared that way would * never be walked again by any pass, full ones included. So a restricted * pass hands its survivors back as candidates in the generation they were - * just promoted into, where a full pass — which skips nothing, and so - * judges them on real reference counts — will settle them. A full pass + * just promoted into, where a later pass at that level — and in the end a + * full pass, which skips nothing and so judges them on real reference + * counts — will settle them. A full pass * needs none of this: it walked every edge, so rc > 0 there means a * genuine outside reference and Bacon-Rajan's own reasoning retires the * candidate. Re-buffering costs a walk, never correctness: an object that * is truly live gets re-blackened by the next retain and dropped then. */ - if (gen_limit < SCR_CYC_MATURE) { + if (gen_limit < SCR_CYC_OLD) { for (size_t i = 0; i < scr_cands.n; i++) { void *obj = scr_cands.v[i]; ScrCycHdr *h = scr_cyc_hdr(obj); @@ -650,7 +734,8 @@ static size_t scr_cyc_pass(unsigned gen_limit) { scr_xgen.n = 0; freed += scr_xg_freed; - if (gen_limit >= SCR_CYC_MATURE) { + if (gen_limit >= SCR_CYC_MATURE) scr_cyc_live_after_mature = scr_cyc_live; + if (gen_limit >= SCR_CYC_OLD) { scr_cyc_live_after_full = scr_cyc_live; scr_cyc_scheduled_mature_age = 0; } @@ -671,8 +756,7 @@ void scr_collect_cycles(void) { * can have buffered anything new, so a pass that comes up empty ends * this; and since each repeat needs a strictly smaller live heap to * continue, it terminates. */ - while (scr_cyc_pass(SCR_CYC_MATURE) - && (scr_roots[SCR_CYC_NURSERY].n || scr_roots[SCR_CYC_MATURE].n)) { + while (scr_cyc_pass(SCR_CYC_OLD) && scr_cyc_buffered()) { } } @@ -681,7 +765,7 @@ void scr_collect_cycles(void) { void scr_cyc_context_cleanup(void) { scr_collect_cycles(); ScrVec *vectors[] = {&scr_roots[SCR_CYC_NURSERY], &scr_roots[SCR_CYC_MATURE], - &scr_cands, &scr_promote, &scr_pending, &scr_restore_pending, &scr_white, &scr_xgen}; + &scr_roots[SCR_CYC_OLD], &scr_cands, &scr_promote, &scr_pending, &scr_restore_pending, &scr_white, &scr_xgen}; for (size_t i = 0; i < sizeof vectors / sizeof vectors[0]; i++) { free(vectors[i]->v); *vectors[i] = (ScrVec){0}; diff --git a/packages/runtime/src/scr_json.test.c b/packages/runtime/src/scr_json.test.c index 397eaa0fe..977506dd2 100644 --- a/packages/runtime/src/scr_json.test.c +++ b/packages/runtime/src/scr_json.test.c @@ -247,8 +247,8 @@ static void shared_property_keys(void) { static void checked_leaf_cycles(void) { /* Object properties omit primitives from trial deletion, while arrays and * captured unknown values still trace them. Exercise shared leaves through - * both edge kinds, with and without an external owner, in either generation. */ - for (int old = 0; old < 2; old++) { + * both edge kinds, with and without an external owner, in every generation. */ + for (uint16_t gen = SCR_CYC_NURSERY; gen < SCR_CYC_NGENS; gen++) { for (int external = 0; external < 2; external++) { #ifdef SCR_RC_AUDIT long before_dyns = scr_dyn_live_count(); @@ -257,7 +257,7 @@ static void checked_leaf_cycles(void) { ScrStr *text = scr_str_new("survives either edge", 20); ScrDyn *leaf = scr_dyn_new_str(text); scr_str_release(text); - if (old) scr_cyc_hdr(leaf)->gen = SCR_CYC_MATURE; + scr_cyc_hdr(leaf)->gen = gen; ScrDyn *first = scr_dyn_new_obj(), *second = scr_dyn_new_obj(); ScrDyn *array = scr_dyn_new_arr(); ScrClosure *closure = scr_closure_new(NULL, 1); diff --git a/packages/runtime/src/scr_runtime.h b/packages/runtime/src/scr_runtime.h index 8b02a12c6..4d65dd2f8 100644 --- a/packages/runtime/src/scr_runtime.h +++ b/packages/runtime/src/scr_runtime.h @@ -336,12 +336,13 @@ char **scr_worker_argv(void); * scan: restore externally-referenced subgraphs; gather: free the * dead cycle members, releasing only edges that LEAVE the white set). * It is GENERATIONAL: each header carries a generation, a pass names the - * oldest one it will walk, and objects that survive a pass are promoted out - * of the nursery so later nursery passes never re-walk them. That is what - * keeps a pass proportional to recent allocation rather than to the whole - * live heap — see the generation note in scr_cycle.c for the soundness - * argument and the schedule. Collection points: program exit (before the RC - * audit), event-loop quiescence, and the per-generation triggers. + * oldest one it will walk, and objects that survive a pass are promoted one + * generation (nursery, mature, old) so later restricted passes never re-walk + * them. That is what keeps a pass proportional to recent allocation rather + * than to the whole live heap — see the generation note in scr_cycle.c for + * the soundness argument and the schedule. Collection points: program + * exit (before the RC audit), event-loop quiescence, and the per-generation + * triggers. * SCR_CYCLE_THRESHOLD pins the nursery trigger to a fixed candidate count. * There is no concurrent or incremental collection. * @@ -367,8 +368,9 @@ enum { }; /* Generations. A candidate sits in the buffer named by its own `gen`, and a - * pass walks only objects at or below the generation it collects. */ -enum { SCR_CYC_NURSERY = 0, SCR_CYC_MATURE = 1, SCR_CYC_NGENS = 2 }; + * pass walks only objects at or below the generation it collects; a pass at + * SCR_CYC_OLD is a full pass. */ +enum { SCR_CYC_NURSERY = 0, SCR_CYC_MATURE = 1, SCR_CYC_OLD = 2, SCR_CYC_NGENS = 3 }; typedef struct ScrCycHdr { ScrTraceFn trace; @@ -380,7 +382,7 @@ typedef struct ScrCycHdr { #endif uint32_t color; /* SCR_CYC_* */ uint16_t buffered; /* 1 = sitting in its generation's candidate buffer */ - uint16_t gen; /* SCR_CYC_NURSERY..SCR_CYC_MATURE (the walk filter) */ + uint16_t gen; /* SCR_CYC_NURSERY..SCR_CYC_OLD (the walk filter) */ size_t buf_index; /* position there (O(1) removal when rc hits 0) */ } ScrCycHdr; diff --git a/packages/runtime/test/test_cycle.c b/packages/runtime/test/test_cycle.c index aaa116924..190d79e11 100644 --- a/packages/runtime/test/test_cycle.c +++ b/packages/runtime/test/test_cycle.c @@ -244,6 +244,107 @@ static void check_unproductive_backlog_backs_off(void) { } } +/* Survive two passes: nursery to mature, then mature to old. */ +static void age_to_old(Node *node) { + for (int i = 0; i < 2; i++) { + read_live(node); + scr_collect_cycles(); + } + check(scr_cyc_hdr(node)->gen == SCR_CYC_OLD, "survivor did not reach the old generation"); +} + +/* Fresh live leaves, each released once so the release path keeps running + * its passes, until `target` objects have been freed or `limit` leaves have + * been made. Returns how many it made. */ +static size_t grow_until(Node **keep, size_t limit, size_t target) { + size_t n = 0; + while (freed < target && n < limit) { + keep[n] = make_leaf(); + read_live(keep[n]); + n++; + } + return n; +} + +static void drop_leaves(Node **keep, size_t n) { + for (size_t i = 0; i < n; i++) { + keep[i]->rc--; + scr_cyc_on_dead(keep[i]); + node_free(keep[i]); + } +} + +/* A dead cycle that has aged into the old generation is reachable only by a + * full pass. While the heap grows with live data, full passes find nothing + * and back off, but the dead cycle must still be reclaimed before the heap + * has grown by half past its size after the last full pass. Runs first, so + * the live heap is below the growth floor and that bound is known. */ +static void check_old_garbage_reclaimed_while_growing(void) { + enum { RING = 64, FLOOR = 4096, LIMIT = 4 * FLOOR }; + static Node *keep[LIMIT]; + size_t before = freed; + Node *ring = make_ring(RING); + age_to_old(ring); + release_live(ring); /* last outside owner gone: a dead old cycle */ + size_t grown = grow_until(keep, LIMIT, before + RING); + check(freed == before + RING, "dead old cycle was never reclaimed while the heap grew"); + check(grown <= FLOOR + FLOOR / 2, "dead old cycle outlived the growth bound"); + drop_leaves(keep, grown); + scr_collect_cycles(); +} + +/* Dead rings that survived at least one pass, each holding a traced edge + * into an old ring: whichever level reclaims them must pay those edges off, + * and must do so without an explicit sweep. */ +static void check_older_rings_into_old(void) { + enum { OLD_RING = 2048, RING = 8, RINGS = 1024, LIMIT = 4 * (OLD_RING + RING * RINGS) }; + static Node *keep[LIMIT]; + Node *old = make_ring(OLD_RING); + age_to_old(old); + size_t before = freed; + Node *rings[RINGS]; + for (size_t i = 0; i < RINGS; i++) { + rings[i] = make_ring(RING); + rings[i]->other = old; + old->rc++; + read_live(rings[i]); + } + scr_cyc_collect_scheduled(); /* every ring survives at least one pass */ + for (size_t i = 0; i < RINGS; i++) { + check(scr_cyc_hdr(rings[i])->gen != SCR_CYC_NURSERY, "ring was not promoted"); + release_live(rings[i]); + } + size_t grown = grow_until(keep, LIMIT, before + RINGS * RING); + check(freed == before + RINGS * RING, "dead older rings were never reclaimed"); + check(old->rc == 2, "edges into the old generation were not paid off"); + drop_leaves(keep, grown); + release_live(old); + scr_collect_cycles(); +} + +/* A dead cycle spanning the mature and old generations is spared by mature + * passes; promotion and re-buffering must still hand it to a full pass. */ +static void check_cycle_spanning_mature_and_old(void) { + enum { LIMIT = 4 * 4096 }; + static Node *keep[LIMIT]; + Node *old = make_ring(4); + age_to_old(old); + Node *young = make_ring(4); + young->other = old; + old->rc++; + old->other = young; + young->rc++; + read_live(young); + scr_cyc_collect_scheduled(); /* young survives a pass and is promoted */ + size_t before = freed; + release_live(old); + release_live(young); /* only the cycle's own edges remain */ + size_t grown = grow_until(keep, LIMIT, before + 8); + check(freed == before + 8, "cycle spanning generations was never reclaimed"); + drop_leaves(keep, grown); + scr_collect_cycles(); +} + static void check_deep_ring(void) { enum { DEPTH = 100000 }; size_t before = freed; @@ -415,6 +516,9 @@ static void check_deferred_white_restoration(void) { } int main(void) { + check_old_garbage_reclaimed_while_growing(); + check_older_rings_into_old(); + check_cycle_spanning_mature_and_old(); check_unproductive_backlog_backs_off(); check_sparse_mature_backlog(1); check_sparse_mature_backlog(2); diff --git a/tests/corpus/cycle-generations-long-lived.ts b/tests/corpus/cycle-generations-long-lived.ts new file mode 100644 index 000000000..08afe7af7 --- /dev/null +++ b/tests/corpus/cycle-generations-long-lived.ts @@ -0,0 +1,115 @@ +// Long-lived cyclic structures next to garbage of every age. +// +// Warehouses keep bins with parent back-pointers and a name index for the +// whole run. Each restock builds scratch pick lists (doubly linked, pointing +// into the bins) that live through many collector passes before they are +// dropped, and every few rounds a whole warehouse is retired after it has +// been live for a long time. Every dead structure must be reclaimed without +// touching the live ones: the sanitized lane's RC audit checks that nothing +// is left over at exit, and the checksums check that nothing live was freed. +class Bin { + label: string; + parent: Bin | null = null; + children: Bin[] = []; + stock: number; + constructor(label: string, stock: number) { + this.label = label; + this.stock = stock; + } +} + +class Warehouse { + name: string; + root: Bin; + index: Map = new Map(); + constructor(name: string) { + this.name = name; + this.root = new Bin(`${name}/root`, 0); + this.index.set(this.root.label, this.root); + } + add(parent: Bin, label: string, stock: number): Bin { + const bin = new Bin(`${this.name}/${label}`, stock); + bin.parent = parent; + parent.children.push(bin); + this.index.set(bin.label, bin); + return bin; + } +} + +class Pick { + bin: Bin; + qty: number; + prev: Pick | null = null; + next: Pick | null = null; + constructor(bin: Bin, qty: number) { + this.bin = bin; + this.qty = qty; + } +} + +function stockWarehouse(name: string, bins: number): Warehouse { + const w = new Warehouse(name); + const all: Bin[] = [w.root]; + for (let i = 1; i < bins; i++) all.push(w.add(all[(i - 1) >> 2], `b${i}`, (i * 7) % 13)); + return w; +} + +function pickList(w: Warehouse, round: number, length: number): Pick | null { + let head: Pick | null = null; + let tail: Pick | null = null; + for (let i = 0; i < length; i++) { + const bin = w.index.get(`${w.name}/b${1 + ((i * 31 + round) % (w.index.size - 1))}`); + if (bin === undefined) continue; + const p = new Pick(bin, 1 + (i % 3)); + if (tail === null) head = p; + else { + tail.next = p; + p.prev = tail; + } + tail = p; + } + return head; +} + +function depth(bin: Bin): number { + let d = 0; + for (let b: Bin | null = bin.parent; b !== null; b = b.parent) d++; + return d; +} + +const warehouses: Warehouse[] = []; +for (let i = 0; i < 4; i++) warehouses.push(stockWarehouse(`w${i}`, 1500)); + +let pending: (Pick | null)[] = []; +let picked = 0; +let retired = 0; +for (let round = 0; round < 24; round++) { + const w = warehouses[round % warehouses.length]; + pending.push(pickList(w, round, 400)); + // Lists wait a few rounds before they are fulfilled and dropped. + if (pending.length > 3) { + for (const head of pending) { + for (let p = head; p !== null; p = p.next) picked += p.qty * (1 + depth(p.bin)); + } + pending = []; + } + // Retire the oldest warehouse every eight rounds and open a fresh one. + if (round % 8 === 7) { + const gone = warehouses.shift(); + if (gone !== undefined) retired += gone.index.size; + warehouses.push(stockWarehouse(`w${4 + round}`, 1500)); + } +} + +let bins = 0; +let stock = 0; +for (const w of warehouses) { + bins += w.index.size; + for (const bin of w.index.values()) { + stock += bin.stock; + if (bin.parent !== null && !bin.parent.children.includes(bin)) console.log("broken link", bin.label); + } +} +console.log(`warehouses ${warehouses.map((w) => w.name).join(",")}`); +console.log(`bins ${bins} stock ${stock} retired ${retired}`); +console.log(`picked ${picked} pending ${pending.length}`);