diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5714-cycle-large-candidate-buffers.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5714-cycle-large-candidate-buffers.ts.json new file mode 100644 index 000000000..e926bd654 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5714-cycle-large-candidate-buffers.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/5714-cycle-large-candidate-buffers.ts" + ], + "diags": [] +} diff --git a/packages/runtime/src/scr_cycle.c b/packages/runtime/src/scr_cycle.c index 655c5979f..d2508cbdb 100644 --- a/packages/runtime/src/scr_cycle.c +++ b/packages/runtime/src/scr_cycle.c @@ -154,6 +154,10 @@ static void scr_cyc_oom(void) { typedef struct { void **v; size_t n, cap; + /* The most entries a pass held, for the work vectors a pass fills + * (scr_cyc_push_work); scr_cyc_pass_done frees a large vector that the + * pass used little of. */ + size_t used; } ScrVec; typedef struct { @@ -321,11 +325,18 @@ static void scr_cyc_grow(ScrVec *vec) { vec->v = grown; } -static void scr_cyc_push(ScrVec *vec, void *obj) { +static __attribute__((noinline)) void scr_cyc_push(ScrVec *vec, void *obj) { if (vec->n == vec->cap) scr_cyc_grow(vec); vec->v[vec->n++] = obj; } +/* A push onto a vector that a pass fills and leaves empty, recording how + * many entries the pass held at most. */ +static inline void scr_cyc_push_work(ScrVec *vec, void *obj) { + scr_cyc_push(vec, obj); + if (vec->n > vec->used) vec->used = vec->n; +} + /* ── the candidate-root buffers (one per generation) ──────────────────── */ #ifndef SCR_WORKERS @@ -896,8 +907,8 @@ static void scr_mark_gray(void *obj, uintptr_t depth); static void scr_mg_visit(void *child, void *ctx) { if (child == NULL || SCR_RC(child) == SIZE_MAX) return; if (scr_cyc_gen(scr_cyc_hdr(child)) > scr_gen_limit) { - scr_cyc_push(&scr_xgen_edges, scr_gray_holder); - scr_cyc_push(&scr_xgen_edges, child); + scr_cyc_push_work(&scr_xgen_edges, scr_gray_holder); + scr_cyc_push_work(&scr_xgen_edges, child); return; } size_t rc = --SCR_RC(child); @@ -911,10 +922,10 @@ static void scr_mark_gray(void *obj, uintptr_t depth) { ScrCycHdr *h = scr_cyc_hdr(obj); if (scr_cyc_color(h) == SCR_CYC_GRAY) return; scr_cyc_set_color(h, SCR_CYC_GRAY); - if (scr_white.n < SCR_CYC_GRAY_LIST_MAX) scr_cyc_push(&scr_white, obj); + if (scr_white.n < SCR_CYC_GRAY_LIST_MAX) scr_cyc_push_work(&scr_white, obj); if (SCR_RC(obj) > 0) scr_gray_referenced++; if (depth == SCR_CYC_WALK_DEPTH) { - scr_cyc_push(&scr_pending, obj); + scr_cyc_push_work(&scr_pending, obj); return; } void *holder = scr_gray_holder; @@ -933,9 +944,9 @@ static void scr_sb_visit(void *child, void *ctx) { static void scr_scan_black(void *obj, uintptr_t depth) { ScrCycHdr *h = scr_cyc_hdr(obj); scr_cyc_set_color(h, SCR_CYC_BLACK); - if (scr_cyc_gen(h) < SCR_CYC_OLD) scr_cyc_push(&scr_promote, obj); + if (scr_cyc_gen(h) < SCR_CYC_OLD) scr_cyc_push_work(&scr_promote, obj); if (depth == SCR_CYC_WALK_DEPTH) { - scr_cyc_push(&scr_restore_pending, obj); + scr_cyc_push_work(&scr_restore_pending, obj); return; } scr_cyc_type(h)->trace(obj, scr_sb_visit, (void *)(depth + 1)); @@ -960,9 +971,9 @@ static void scr_scan(void *obj, uintptr_t depth) { scr_cyc_set_color(h, SCR_CYC_WHITE); /* Record each whitened object once. A later outside root can restore it; * filter the list only after every scan finishes. No gather trace needed. */ - scr_cyc_push(&scr_white, obj); + scr_cyc_push_work(&scr_white, obj); if (depth == SCR_CYC_WALK_DEPTH) { - scr_cyc_push(&scr_pending, obj); + scr_cyc_push_work(&scr_pending, obj); return; } scr_cyc_type(h)->trace(obj, scr_scan_visit, (void *)(depth + 1)); @@ -1061,7 +1072,7 @@ static SCR_CYC_COMPACT size_t scr_cyc_settle(unsigned gen_limit) { if (scr_cyc_color(scr_cyc_hdr(scr_xgen_edges.v[i])) != SCR_CYC_DOOMED) continue; void *target = scr_xgen_edges.v[i + 1]; SCR_RC(target) += 1; /* pin across the teardowns */ - scr_cyc_push(&scr_xgen, target); + scr_cyc_push_work(&scr_xgen, target); } scr_xgen_edges.n = 0; @@ -1144,6 +1155,39 @@ static SCR_CYC_COMPACT size_t scr_cyc_settle(unsigned gen_limit) { return freed; } +/* A work vector that holds more than this many entries is freed at the end + * of a pass that used less than a quarter of it, so a spike in one pass + * does not keep its buffers for the rest of the program, while passes of a + * steady size keep reusing theirs. */ +#define SCR_CYC_KEEP_VECTOR 65536 + +static SCR_CYC_COMPACT void scr_cyc_trim(ScrVec *vec) { + if (vec->n == 0 && vec->cap > SCR_CYC_KEEP_VECTOR && vec->used < vec->cap / 4) { + free(vec->v); + vec->v = NULL; + vec->cap = 0; + } + vec->used = 0; +} + +/* The end of every pass. The candidate list keeps its count, which the + * scheduler reads; its buffer goes when this pass's candidates filled + * less than a quarter of it. */ +static size_t scr_cyc_pass_done(size_t freed) { + if (scr_cands.cap > SCR_CYC_KEEP_VECTOR && scr_cands.n < scr_cands.cap / 4) { + free(scr_cands.v); + scr_cands.v = NULL; + scr_cands.cap = 0; + } + scr_cyc_trim(&scr_pending); + scr_cyc_trim(&scr_white); + scr_cyc_trim(&scr_promote); + scr_cyc_trim(&scr_restore_pending); + scr_cyc_trim(&scr_xgen); + scr_cyc_trim(&scr_xgen_edges); + return freed; +} + static size_t scr_cyc_pass(unsigned gen_limit) { if (scr_collecting) return 0; scr_collecting = true; @@ -1154,8 +1198,24 @@ static size_t scr_cyc_pass(unsigned gen_limit) { * almost none, and it cost a header store per retain). Drain the buffers * before walking: tracing only adjusts counts and worklists, so nothing * can re-buffer underneath us. Candidates ABOVE the limit keep their - * slots and wait for a pass at their own level. */ - scr_cands.n = 0; + * slots and wait for a pass at their own level. + * + * The fullest root buffer becomes the candidate list instead of being + * copied into it, so a pass never holds two buffers of every candidate; + * the generation gets the previous candidate buffer, empty. The other + * buffers are appended after it. */ + unsigned largest = 0; + for (unsigned g = 1; g <= gen_limit; g++) + if (scr_roots[g].n > scr_roots[largest].n) largest = g; + ScrVec previous = scr_cands; + scr_cands = scr_roots[largest]; + scr_roots[largest] = previous; + scr_roots[largest].n = 0; + for (size_t i = 0; i < scr_cands.n; i++) { + ScrCycHdr *h = scr_cyc_hdr(scr_cands.v[i]); + scr_cyc_clear_buffered(h); + if (gen_limit < SCR_CYC_OLD) scr_cyc_walked(scr_cyc_type(h), gen_limit); + } for (unsigned g = 0; g <= gen_limit; g++) { for (size_t i = 0; i < scr_roots[g].n; i++) { void *obj = scr_roots[g].v[i]; @@ -1177,7 +1237,7 @@ static size_t scr_cyc_pass(unsigned gen_limit) { } if (scr_gray_referenced == 0 && scr_white.n < SCR_CYC_GRAY_LIST_MAX) { for (size_t i = 0; i < scr_white.n; i++) scr_cyc_set_color(scr_cyc_hdr(scr_white.v[i]), SCR_CYC_WHITE); - return scr_cyc_settle(gen_limit); + return scr_cyc_pass_done(scr_cyc_settle(gen_limit)); } scr_white.n = 0; @@ -1190,7 +1250,7 @@ static size_t scr_cyc_pass(unsigned gen_limit) { scr_cyc_type(scr_cyc_hdr(obj))->trace(obj, scr_scan_visit, NULL); } - return scr_cyc_settle(gen_limit); + return scr_cyc_pass_done(scr_cyc_settle(gen_limit)); } #ifdef SCR_CYC_TESTING diff --git a/packages/runtime/test/test_cycle.c b/packages/runtime/test/test_cycle.c index 59504281f..2c17e671f 100644 --- a/packages/runtime/test/test_cycle.c +++ b/packages/runtime/test/test_cycle.c @@ -1099,6 +1099,46 @@ static void check_publication_takes_candidates(void) { check(immortal_traced == 0, "a pass walked a published ring"); } +/* A full pass whose fullest candidate buffer is the old generation's (more + * entries than a pass keeps between passes), with nursery candidates + * beside it: every dead ring in either buffer is reclaimed, the live rings + * keep their counts, and a second round collects through the buffers the + * first one left behind. */ +static void check_large_old_buffer_with_nursery_candidates(void) { + enum { OLD = 70000, YOUNG = 300, RING = 2 }; + static Node *old[OLD]; + Node *young[YOUNG]; + for (int round = 0; round < 2; round++) { + for (size_t i = 0; i < OLD; i++) { + old[i] = make_ring(RING); + read_live(old[i]); + } + scr_collect_cycles(); + for (size_t i = 0; i < OLD; i++) read_live(old[i]); + scr_collect_cycles(); + check(scr_cyc_gen(scr_cyc_hdr(old[OLD - 1])) == SCR_CYC_OLD, "rings did not reach the old generation"); + size_t before = freed; + for (size_t i = 0; i < OLD; i++) + if (i % 2) release_live(old[i]); /* dead: only the ring edge remains */ + else read_live(old[i]); /* live candidate */ + for (size_t i = 0; i < YOUNG; i++) { + young[i] = make_ring(RING); + if (i % 3) release_live(young[i]); + else read_live(young[i]); + } + scr_collect_cycles(); + size_t young_dead = YOUNG - (YOUNG + 2) / 3; + check(freed - before == (OLD / 2 + young_dead) * RING, "a dead ring was not reclaimed"); + for (size_t i = 0; i < OLD; i += 2) check(old[i]->rc == 2, "a live old ring lost a count"); + for (size_t i = 0; i < YOUNG; i += 3) check(young[i]->rc == 2, "a live young ring lost a count"); + before = freed; + for (size_t i = 0; i < OLD; i += 2) release_live(old[i]); + for (size_t i = 0; i < YOUNG; i += 3) release_live(young[i]); + scr_collect_cycles(); + check(freed - before == (OLD / 2 + (YOUNG + 2) / 3) * RING, "the remaining rings were not reclaimed"); + } +} + int main(void) { check_old_garbage_reclaimed_while_growing(); check_older_rings_into_old(); @@ -1125,6 +1165,7 @@ int main(void) { check_rebuffer_past_mature(); check_allocation_growth_runs_passes(); check_publication_takes_candidates(); + check_large_old_buffer_with_nursery_candidates(); printf("cycle collection checks passed: threshold=%zu growth_cap=%zu\n", configured_nursery_threshold(), configured_growth_cap()); return 0; diff --git a/tests/corpus/5714-cycle-large-candidate-buffers.ts b/tests/corpus/5714-cycle-large-candidate-buffers.ts new file mode 100644 index 000000000..2f5093285 --- /dev/null +++ b/tests/corpus/5714-cycle-large-candidate-buffers.ts @@ -0,0 +1,99 @@ +// Cyclic structures in numbers large enough that the cycle collector's +// candidate buffers grow past the size kept between passes: trees with +// parent links that survive several passes (and age into older +// generations) alongside short-lived ones, dropped in waves, then rebuilt +// on the buffers the earlier passes left behind. + +class TreeNode { + parent: TreeNode | undefined = undefined; + children: TreeNode[] = []; + readonly id: number; + constructor(id: number) { + this.id = id; + } +} + +function buildTree(id: number, width: number, depth: number): TreeNode { + const root = new TreeNode(id); + let frontier = [root]; + let next = id + 1; + for (let d = 0; d < depth; d++) { + const level: TreeNode[] = []; + for (const node of frontier) + for (let w = 0; w < width; w++) { + const child = new TreeNode(next++); + child.parent = node; + node.children.push(child); + level.push(child); + } + frontier = level; + } + return root; +} + +function size(node: TreeNode): number { + let n = 1; + for (const child of node.children) n += size(child); + return n; +} + +function checksum(node: TreeNode): number { + let sum = node.id % 1000; + for (const child of node.children) sum = (sum * 31 + checksum(child)) % 1000003; + return sum; +} + +interface Ring { + label: string; + next: Ring | undefined; +} + +function ring(label: string, length: number): Ring { + const first: Ring = { label: `${label}0`, next: undefined }; + let last = first; + for (let i = 1; i < length; i++) { + const node: Ring = { label: `${label}${i}`, next: undefined }; + last.next = node; + last = node; + } + last.next = first; + return first; +} + +function ringLength(start: Ring): number { + let n = 1; + for (let node = start.next; node !== start && node !== undefined; node = node.next) n++; + return n; +} + +for (let round = 0; round < 3; round++) { + // Long-lived trees: kept across many passes. + const kept: TreeNode[] = []; + for (let t = 0; t < 40; t++) kept.push(buildTree(t * 10000, 3, 6)); + let total = 0; + // Waves of short-lived cyclic garbage while the kept trees age. + for (let wave = 0; wave < 6; wave++) { + const rings: Ring[] = []; + for (let i = 0; i < 6000; i++) rings.push(ring(`w${wave}r${i}-`, 3)); + for (const r of rings) total += ringLength(r); + const scratch = buildTree(wave, 4, 5); + total += size(scratch); + } + let sum = 0; + for (const tree of kept) sum = (sum + checksum(tree) + size(tree)) % 1000003; + // Drop half of the long-lived trees, keep using the rest. + const survivors = kept.filter((_, i) => i % 2 === 0); + kept.length = 0; + for (let wave = 0; wave < 3; wave++) { + const rings: Ring[] = []; + for (let i = 0; i < 4000; i++) rings.push(ring(`x${wave}r${i}-`, 4)); + for (const r of rings) total += ringLength(r); + } + let survivorSum = 0; + for (const tree of survivors) { + survivorSum = (survivorSum + checksum(tree)) % 1000003; + const leaf = tree.children[1]!.children[2]!; + survivorSum = (survivorSum + (leaf.parent === tree.children[1] ? 1 : 0)) % 1000003; + } + console.log(`round ${round}: total ${total} kept ${sum} survivors ${survivors.length} ${survivorSum}`); +}