diff --git a/packages/runtime/src/scr_cycle.c b/packages/runtime/src/scr_cycle.c index d08f54a21..0d9df7106 100644 --- a/packages/runtime/src/scr_cycle.c +++ b/packages/runtime/src/scr_cycle.c @@ -603,7 +603,8 @@ static SCR_TL unsigned scr_cyc_backlog_shift[SCR_CYC_NGENS]; static SCR_TL bool scr_cyc_backlog_judged[SCR_CYC_NGENS]; #endif -/* Mature objects in the last full pass's dead set (scr_cyc_adapt_growth). */ +/* Mature objects in the last pass's dead set (scr_cyc_adapt_growth, + * scr_cyc_adapt_nursery). */ #ifndef SCR_WORKERS static SCR_TL size_t scr_cyc_mature_dead = 0; #endif @@ -639,23 +640,74 @@ static SCR_TL size_t scr_cyc_death_due_live = 0; /* The nursery trigger a program sets for itself * (scr_cyc_set_nursery_threshold), 0 for the default; SCR_CYCLE_THRESHOLD - * takes precedence over it. */ + * takes precedence over it. Either pins the trigger. */ static long scr_cyc_nursery_threshold_set = 0; static SCR_TL size_t scr_cyc_nursery_threshold_cached = 0; +#define SCR_CYC_NURSERY_ADAPTIVE SIZE_MAX -static size_t scr_cyc_nursery_threshold(void) { +/* An unpinned trigger follows where cyclic garbage dies. Garbage that + * outlives the nursery window survives its nursery pass and is promoted, + * and then only a mature or full pass finds it, after walking that level's + * live candidates. So after each scheduled mature or full pass whose dead + * set held more mature and old objects than 1/SCR_CYC_NURSERY_RATIO of what + * the nursery passes since the previous one freed, the window doubles, up + * to SCR_CYC_NURSERY_MAX_SHIFT doublings (4096 candidates); after one that + * held less than a quarter of that share, it halves, down to + * SCR_CYC_NURSERY_CANDIDATES. + * `nursery_freed`: what the nursery passes freed since the last scheduled + * mature or full pass. */ +#define SCR_CYC_NURSERY_RATIO 8 +#define SCR_CYC_NURSERY_MAX_SHIFT 4 +static SCR_TL unsigned char scr_cyc_nursery_shift = 0; +static SCR_TL size_t scr_cyc_nursery_freed = 0; + +static size_t scr_cyc_nursery_pinned(void) { if (scr_cyc_nursery_threshold_cached == 0) { const char *env = scr_getenv("SCR_CYCLE_THRESHOLD"); long v = env ? strtol(env, NULL, 10) : 0; if (v <= 0) v = scr_cyc_nursery_threshold_set; - scr_cyc_nursery_threshold_cached = v > 0 ? (size_t)v : SCR_CYC_NURSERY_CANDIDATES; + scr_cyc_nursery_threshold_cached = v > 0 ? (size_t)v : SCR_CYC_NURSERY_ADAPTIVE; } return scr_cyc_nursery_threshold_cached; } +/* The pinned trigger, or the default one. The scheduled-age rule counts + * collection points in it, so the adaptation does not lengthen how long an + * older backlog waits at them. */ +static size_t scr_cyc_nursery_base(void) { + size_t pinned = scr_cyc_nursery_pinned(); + return pinned != SCR_CYC_NURSERY_ADAPTIVE ? pinned : SCR_CYC_NURSERY_CANDIDATES; +} + +static size_t scr_cyc_nursery_threshold(void) { + size_t pinned = scr_cyc_nursery_pinned(); + if (pinned != SCR_CYC_NURSERY_ADAPTIVE) return pinned; + return (size_t)SCR_CYC_NURSERY_CANDIDATES << scr_cyc_nursery_shift; +} + +/* Adapts an unpinned trigger to the mature and old objects a scheduled + * mature or full pass reclaimed; returns whether the trigger changed. */ +static SCR_CYC_SCHEDULING bool scr_cyc_adapt_nursery(size_t older_freed) { + size_t nursery_freed = scr_cyc_nursery_freed; + scr_cyc_nursery_freed = 0; + if (scr_cyc_nursery_pinned() != SCR_CYC_NURSERY_ADAPTIVE) return false; + if (older_freed * SCR_CYC_NURSERY_RATIO > nursery_freed) { + if (scr_cyc_nursery_shift == SCR_CYC_NURSERY_MAX_SHIFT) return false; + scr_cyc_nursery_shift++; + return true; + } + if (older_freed * SCR_CYC_NURSERY_RATIO * 4 < nursery_freed && scr_cyc_nursery_shift > 0) { + scr_cyc_nursery_shift--; + return true; + } + return false; +} + void scr_cyc_set_nursery_threshold(long threshold) { scr_cyc_nursery_threshold_set = threshold; scr_cyc_nursery_threshold_cached = 0; + scr_cyc_nursery_shift = 0; + scr_cyc_nursery_freed = 0; } /* The number of buffered candidates (across every generation) at which the @@ -881,7 +933,14 @@ static SCR_CYC_SCHEDULING void scr_cyc_scheduled_pass(unsigned gen, bool by_grow size_t freed = scr_cyc_pass(gen); scr_cyc_keep_base = false; size_t old_reclaimed = scr_cyc_old_freed - old_freed_before + scr_cyc_mature_dead; - if (gen == SCR_CYC_NURSERY) return; + if (gen == SCR_CYC_NURSERY) { + scr_cyc_nursery_freed += freed; + return; + } + if (scr_cyc_adapt_nursery(old_reclaimed)) { + scr_cyc_trigger = scr_cyc_buffered() + scr_cyc_nursery_threshold(); + scr_cyc_window_update(); + } if (scr_cands.n != 0) { scr_cyc_backlog_judged[gen] = true; if (freed * SCR_CYC_GROWTH_DIV >= scr_cands.n) @@ -940,7 +999,7 @@ void scr_cyc_collect_scheduled(void) { if (!scr_cyc_older_backlog()) { scr_cyc_scheduled_mature_age = 0; } else if (++scr_cyc_scheduled_mature_age - >= scr_cyc_nursery_threshold()) { + >= scr_cyc_nursery_base()) { scr_cyc_scheduled_pass(SCR_CYC_OLD, false, false); return; } @@ -1981,7 +2040,7 @@ static SCR_CYC_COMPACT size_t scr_cyc_settle(unsigned gen_limit, bool doomed) { ScrCycHdr *h = scr_cyc_hdr(obj); const ScrCycType *type = scr_cyc_type(h); if (gen_limit < SCR_CYC_OLD) scr_cyc_freed(type, gen_limit); - else scr_cyc_mature_dead += scr_cyc_gen(h) == SCR_CYC_MATURE; + scr_cyc_mature_dead += scr_cyc_gen(h) == SCR_CYC_MATURE; #ifdef SCR_YIN if (scr_cyc_gen(h) == SCR_CYC_YOUNG) scr_yin_leave(h, obj); #endif diff --git a/packages/runtime/src/scr_rc.h b/packages/runtime/src/scr_rc.h index de9f21f44..171e420c2 100644 --- a/packages/runtime/src/scr_rc.h +++ b/packages/runtime/src/scr_rc.h @@ -703,8 +703,9 @@ void scr_cyc_set_growth_cap(long cap); /* Sets the nursery trigger SCR_CYCLE_THRESHOLD sets (scr_cycle.c): the * number of new candidates at which the release path runs a pass, for a * program that knows its own collection profile. The environment variable - * takes precedence; 0 restores the default. The trigger re-arms with it at - * the end of the next pass. Call it before the program allocates much. */ + * takes precedence; 0 restores the default, which starts at 256 and adapts + * to the program. The trigger re-arms with it at the end of the next pass. + * Call it before the program allocates much. */ void scr_cyc_set_nursery_threshold(long threshold); void scr_cyc_context_cleanup(void); diff --git a/packages/runtime/test/test_cycle.c b/packages/runtime/test/test_cycle.c index 0ab827e47..dc48abd26 100644 --- a/packages/runtime/test/test_cycle.c +++ b/packages/runtime/test/test_cycle.c @@ -575,6 +575,60 @@ static void check_program_nursery_threshold(void) { scr_collect_cycles(); } +/* How many dead rings the release path takes to run its next pass, from + * the end of the previous one: the nursery trigger in effect. */ +static size_t nursery_window(void) { + size_t before = freed, n = 0; + while (freed == before && n < 65536) { + release_live(make_ring(1)); + n++; + } + check(freed - before == n, "a nursery pass left dead rings behind"); + return n; +} + +/* A scheduled full pass that reclaims nothing: its only candidate is an + * older live leaf. */ +static void scheduled_pass_finding_nothing(Node *leaf) { + size_t before = freed; + read_live(leaf); + for (size_t i = 0; i < configured_nursery_threshold(); i++) + scr_cyc_collect_scheduled(); + check(freed == before, "a scheduled pass over a live leaf freed something"); +} + +/* Unpinned, the nursery trigger starts at 256 candidates. Each scheduled + * full pass that reclaims promoted rings doubles it, up to 4096; each one + * that reclaims nothing after the nursery passes freed rings halves it, + * down to 256. */ +static void check_adaptive_nursery_threshold(void) { + if (getenv("SCR_CYCLE_THRESHOLD") != NULL) return; /* it pins the trigger */ + scr_cyc_set_nursery_threshold(0); + scr_collect_cycles(); + check(nursery_window() == 256, "the unpinned nursery trigger did not start at 256"); + static const size_t widened[] = { 512, 1024, 2048, 4096, 4096 }; + for (size_t i = 0; i < sizeof(widened) / sizeof(widened[0]); i++) { + collect_dead_mature_rings(); + check(nursery_window() == widened[i], + "a full pass reclaiming promoted rings did not double the nursery trigger"); + } + + static const size_t narrowed[] = { 2048, 1024, 512, 256, 256 }; + Node *leaf = make_leaf(); + read_live(leaf); + scr_collect_cycles(); /* promote */ + for (size_t i = 0; i < sizeof(narrowed) / sizeof(narrowed[0]); i++) { + scheduled_pass_finding_nothing(leaf); + check(nursery_window() == narrowed[i], + "a full pass reclaiming nothing did not halve the nursery trigger"); + } + leaf->rc--; + scr_cyc_on_dead(leaf); + node_free(leaf); + scr_cyc_set_nursery_threshold(0); + scr_collect_cycles(); +} + /* The phase sweep (scr_collect_cycles_phase) is for a caller that knows a * phase's data has just died: it sweeps, and the growth schedule starts * over. After passes that find nothing have widened the old level's @@ -1506,6 +1560,9 @@ static void check_acyclic_objects(void) { } int main(void) { + /* The scheduling checks count candidates against a fixed nursery trigger; + * check_adaptive_nursery_threshold covers the unpinned one. */ + scr_cyc_set_nursery_threshold((long)configured_nursery_threshold()); check_inline_buffering(); check_old_garbage_reclaimed_while_growing(); check_older_rings_into_old(); @@ -1541,6 +1598,7 @@ int main(void) { check_acyclic_objects(); check_program_growth_cap(); check_program_nursery_threshold(); + check_adaptive_nursery_threshold(); printf("cycle collection checks passed: threshold=%zu growth_cap=%zu\n", configured_nursery_threshold(), configured_growth_cap()); return 0;