diff --git a/packages/compiler/src/backend/llvm/expr-records.ts b/packages/compiler/src/backend/llvm/expr-records.ts index e65a356eb..9e3a5e9b0 100644 --- a/packages/compiler/src/backend/llvm/expr-records.ts +++ b/packages/compiler/src/backend/llvm/expr-records.ts @@ -61,9 +61,11 @@ export function emitUnionWiden( /** The identity union conversion (IR unionRetype). Two tagged unions * share the operand's box: its +1 becomes the result, so no box is built * and no payload is retained. The operand is a whole-value use in every - * lifetime analysis, so it is never a stack box. A nullable-pointer union - * on either side, or a source union whose boxes a checked-dynamic cast has - * cached (and refreshes in place), rebuilds each listed arm instead. */ + * lifetime analysis, so it is never a stack box. Two pointer unions of one + * encoding share the pointer the same way. A pointer union on only one + * side, pointer unions whose encodings differ for an arm the value may + * hold, or a source union whose boxes a checked-dynamic cast has cached (and + * refreshes in place), rebuild each listed arm instead. */ export function emitUnionRetype( host: LlvmEmitterContext, e: IrExpr & { kind: "unionRetype" }, @@ -75,14 +77,15 @@ export function emitUnionRetype( if (from && to) { // Equal encodings make the pointer itself the target value. A shape // union derives its record arms from the record and a class union its - // class arms from the class id, so only the unit encodings must agree. + // class arms from the class id, so only the unit encodings must agree, + // and only when the value may hold one of the source's unit arms: a + // narrowing that drops `undefined` or `null` keeps the very pointer. + const holdsUnit = e.tags.some((tag) => tag === from.unitTag || tag === from.nullTag); + const units = !holdsUnit || (from.unitTag === to.unitTag && from.nullTag === to.nullTag); const same = from.repr === "single" && to.repr === "single" - ? from.refTag === to.refTag && from.unitTag === to.unitTag && from.nullTag === to.nullTag - : from.repr !== "single" && - from.repr === to.repr && - from.unitTag === to.unitTag && - from.nullTag === to.nullTag; + ? from.refTag === to.refTag && units + : from.repr !== "single" && from.repr === to.repr && units; if (same) return host.own({ name: value.name, type: e.type }); return host.own({ name: rebuildRetype(host, value.name, e), type: e.type }); } diff --git a/packages/compiler/src/backend/llvm/union-retype.test.ts b/packages/compiler/src/backend/llvm/union-retype.test.ts index 502940e71..bc82e7e44 100644 --- a/packages/compiler/src/backend/llvm/union-retype.test.ts +++ b/packages/compiler/src/backend/llvm/union-retype.test.ts @@ -268,3 +268,51 @@ test("record unions retype as the record pointer itself", async () => { // No record of the visitor is ever boxed. expect(ir).not.toMatch(/call ptr @scr_union_new_ref/); }); + +// Children that may be absent, as IR nodes have: reading one after its +// `=== undefined` or `=== null` test narrows a shape union with a unit arm +// to the shape union without it. +const OPTIONAL_CHILDREN = ` +type Num = { kind: "num"; value: number }; +type Ref = { kind: "ref"; id: string }; +type Call = { kind: "call"; callee: Expr; receiver?: Expr; args: Expr[] }; +type Init = { kind: "init"; value: Expr | null }; +type Expr = Num | Ref | Call | Init; +function count(e: Expr): number { + switch (e.kind) { + case "num": + case "ref": + return 1; + case "call": + return ( + 1 + + count(e.callee) + + (e.receiver === undefined ? 0 : count(e.receiver)) + + e.args.reduce((n, a) => n + count(a), 0) + ); + case "init": + return 1 + (e.value === null ? 0 : count(e.value)); + } +} +const tree: Expr = { + kind: "call", + callee: { kind: "ref", id: "f" }, + receiver: { kind: "init", value: { kind: "num", value: 1 } }, + args: [{ kind: "init", value: null }], +}; +console.log(count(tree)); +`; + +test("dropping a shape union's unit arm keeps the record pointer", async () => { + const { module, ir } = await llvmFor(OPTIONAL_CHILDREN, false); + const identities = module.functions.filter( + (fn) => + fn.name.startsWith("%union.retag.") && + JSON.stringify(fn.body).includes('"kind":"unionRetype"'), + ); + expect(identities.length).toBeGreaterThan(0); + for (const helper of identities) { + const symbol = `sc_bf__x25_union_retag_${helper.name.slice("%union.retag.".length)}`; + expect(body(ir, symbol)).not.toMatch(/retype\.built/); + } +}); diff --git a/packages/runtime/src/scr_cycle.c b/packages/runtime/src/scr_cycle.c index a7157f9af..02ad86aa1 100644 --- a/packages/runtime/src/scr_cycle.c +++ b/packages/runtime/src/scr_cycle.c @@ -235,6 +235,7 @@ typedef struct { bool falling; bool idle_points; size_t death_due_live; + bool probe_due; bool keep_base; size_t trigger; size_t alloc_trigger; @@ -286,6 +287,7 @@ static _Thread_local ScrCycThread scr_cyc_thread = {.gen_limit = SCR_CYC_OLD}; #define scr_cyc_falling (scr_cyc_thread.falling) #define scr_cyc_idle_points (scr_cyc_thread.idle_points) #define scr_cyc_death_due_live (scr_cyc_thread.death_due_live) +#define scr_cyc_probe_due (scr_cyc_thread.probe_due) #define scr_cyc_keep_base (scr_cyc_thread.keep_base) #define scr_cyc_kinds (scr_cyc_thread.kinds) #define scr_cyc_kinds_cap (scr_cyc_thread.kinds_cap) @@ -640,6 +642,10 @@ static SCR_TL bool scr_cyc_falling = false, scr_cyc_keep_base = false; #ifndef SCR_WORKERS static SCR_TL bool scr_cyc_idle_points = false; static SCR_TL size_t scr_cyc_death_due_live = 0; +/* A nursery probe window opened since the last scheduled pass (see "level + * bypass"): the next pass the release path would run at the nursery runs at + * the mature level instead. */ +static SCR_TL bool scr_cyc_probe_due = false; #endif /* The nursery trigger a program sets for itself @@ -976,9 +982,11 @@ static SCR_CYC_SCHEDULING void scr_cyc_collect_due(void) { if (!by_growth && !by_death && scr_roots[SCR_CYC_OLD].n < scr_cyc_backlog_threshold(SCR_CYC_OLD)) { gen = SCR_CYC_MATURE; by_growth = scr_cyc_grown(SCR_CYC_MATURE); - if (!by_growth && scr_roots[SCR_CYC_MATURE].n < scr_cyc_backlog_threshold(SCR_CYC_MATURE)) + if (!by_growth && scr_roots[SCR_CYC_MATURE].n < scr_cyc_backlog_threshold(SCR_CYC_MATURE) && + !scr_cyc_probe_due) gen = SCR_CYC_NURSERY; } + scr_cyc_probe_due = false; scr_cyc_scheduled_pass(gen, by_growth, by_death); } @@ -1059,6 +1067,16 @@ bool scr_cyc_collect_idle(void) { * a probe window every candidate of the type stays at the level, so the * cycles it builds during the window die there. * + * Those young cycles can still be held by older garbage: a dead structure + * the bypass promoted before the window that points at objects built in it. + * A restricted pass reads that edge as external, so the window frees none + * of them, judges the type long-lived again, and the garbage on both sides + * waits for a mature pass, which the growth schedule may not run until the + * heap has grown a multiple of its size. So a nursery window that opens + * makes the next release-triggered pass that would run at the nursery run + * at the mature level: it reclaims the promoted garbage once, and the window + * judges the young cycles on their own. + * * This changes only which generation an object is judged in. Any * generation assignment is sound (a restricted pass treats edges from older * objects as external), and a bypassing candidate is buffered like any @@ -1110,7 +1128,8 @@ static inline ScrCycKind *scr_cyc_kind(uint32_t kind) { /* Whether a candidate of this type at level `gen` (nursery or mature) * skips that level's passes (see above). The last candidate a bypass sends - * past the level opens a probe window, with fresh counts. */ + * past the level opens a probe window, with fresh counts, and a nursery + * window asks for a mature pass first. */ static inline __attribute__((always_inline)) bool scr_cyc_bypassed(const ScrCycType *type, unsigned gen) { uint32_t kind = type->kind; @@ -1119,9 +1138,12 @@ static inline __attribute__((always_inline)) bool scr_cyc_bypassed(const ScrCycT if (k->bypass == 0) return false; if (--k->bypass == 0) { k->walked = k->freed = 0; + if (gen == SCR_CYC_NURSERY) { + scr_cyc_probe_due = true; #if SCR_CYC_INLINE_BUFFERING - if (gen == SCR_CYC_NURSERY) scr_cyc_nursery_bypassing--; + scr_cyc_nursery_bypassing--; #endif + } } return true; } diff --git a/packages/runtime/test/test_cycle.c b/packages/runtime/test/test_cycle.c index 961d43586..f18d29ff8 100644 --- a/packages/runtime/test/test_cycle.c +++ b/packages/runtime/test/test_cycle.c @@ -1039,6 +1039,75 @@ static void check_nursery_bypass(void) { scr_collect_cycles(); } +/* A nursery probe window that opens makes the next release-triggered pass + * that would run at the nursery run at the mature level, which reclaims + * dead promoted rings a nursery pass stops at. */ +static size_t window_kind_freed, window_ring_freed; +static void window_kind_free(void *obj) { + window_kind_freed++; + node_free(obj); +} +static void window_ring_free(void *obj) { + window_ring_freed++; + node_free(obj); +} +static ScrCycType window_kind_type = { &node_trace, &window_kind_free, 9 }; +static ScrCycType window_ring_type = { &node_trace, &window_ring_free, 0 }; + +/* A candidate of the window kind that dies once it has been routed, as the + * next pass would route it: it counts down a bypass without growing the + * heap. */ +static void release_window_candidate(void) { + Node *node = scr_cyc_alloc(sizeof(*node), &window_kind_type); + node->rc = 2; + release_live(node); + scr_cyc_route_bypassed_at(); + node->rc = 0; + scr_cyc_on_dead(node); + window_kind_free(node); +} + +static void check_probe_window_runs_mature_pass(void) { + /* Every release collects at a threshold of 1, and passes with only a few + * candidates walk too few of them to decide a window. */ + if (configured_nursery_threshold() < 8) return; + enum { WINDOW = 8192, INTERVAL = 16, LIVE = WINDOW, RING = 4 }; + Node **live = calloc(LIVE, sizeof(*live)); + if (!live) scr_trap("cycle test: out of memory\n"); + for (size_t i = 0; i < LIVE; i++) { + live[i] = scr_cyc_alloc(sizeof(*live[i]), &window_kind_type); + live[i]->rc = 1; + read_live(live[i]); + scr_cyc_pass_at(SCR_CYC_NURSERY); + } + scr_collect_cycles(); + /* A pass that an earlier window asked for runs now. */ + run_release_passes(); + + /* A dead ring promoted to the mature level: nursery passes stop at it. */ + Node *ring = make_ring_of(RING, &window_ring_type); + ring->rc++; + release_live(ring); + scr_cyc_pass_at(SCR_CYC_NURSERY); + release_live(ring); + run_release_passes(); + check(window_ring_freed == 0, "a nursery pass freed a dead mature ring"); + + /* The window kind's candidates go past the nursery until a probe window + * opens; the next release-triggered pass runs at the mature level. */ + for (size_t i = 0; i < (size_t)WINDOW * INTERVAL; i++) release_window_candidate(); + run_release_passes(); + check(window_ring_freed == RING, "the pass after a probe window opened missed a dead mature ring"); + + for (size_t i = 0; i < LIVE; i++) { + live[i]->rc = 0; + scr_cyc_on_dead(live[i]); + window_kind_free(live[i]); + } + free(live); + scr_collect_cycles(); +} + /* The mature level's bypass: a type whose mature candidates keep surviving * goes straight to the old generation, its dead cycles are found by full * passes, and once its objects die at the mature level again it is judged @@ -1719,6 +1788,7 @@ int main(void) { check_death_brings_full_pass(); check_nursery_bypass(); check_mature_bypass(); + check_probe_window_runs_mature_pass(); check_probe_sees_candidate_rings(); check_rebuffer_past_mature(); check_allocation_growth_runs_passes();