Keep the pointer when a narrowing drops a pointer union's unit arm (#2517)

packages/compiler/src/backend/llvm/expr-records.ts

@@ -61,9 +61,11 @@ export function emitUnionWiden(
 /** The identity union conversion (IR unionRetype). Two tagged unions
  * share the operand's box: its +1 becomes the result, so no box is built
  * and no payload is retained. The operand is a whole-value use in every
- * lifetime analysis, so it is never a stack box. A nullable-pointer union
- * on either side, or a source union whose boxes a checked-dynamic cast has
- * cached (and refreshes in place), rebuilds each listed arm instead. */
+ * lifetime analysis, so it is never a stack box. Two pointer unions of one
+ * encoding share the pointer the same way. A pointer union on only one
+ * side, pointer unions whose encodings differ for an arm the value may
+ * hold, or a source union whose boxes a checked-dynamic cast has cached (and
+ * refreshes in place), rebuild each listed arm instead. */
 export function emitUnionRetype(
   host: LlvmEmitterContext,
   e: IrExpr & { kind: "unionRetype" },
@@ -75,14 +77,15 @@ export function emitUnionRetype(
   if (from && to) {
     // Equal encodings make the pointer itself the target value. A shape
     // union derives its record arms from the record and a class union its
-    // class arms from the class id, so only the unit encodings must agree.
+    // class arms from the class id, so only the unit encodings must agree,
+    // and only when the value may hold one of the source's unit arms: a
+    // narrowing that drops `undefined` or `null` keeps the very pointer.
+    const holdsUnit = e.tags.some((tag) => tag === from.unitTag || tag === from.nullTag);
+    const units = !holdsUnit || (from.unitTag === to.unitTag && from.nullTag === to.nullTag);
     const same =
       from.repr === "single" && to.repr === "single"
-        ? from.refTag === to.refTag && from.unitTag === to.unitTag && from.nullTag === to.nullTag
-        : from.repr !== "single" &&
-          from.repr === to.repr &&
-          from.unitTag === to.unitTag &&
-          from.nullTag === to.nullTag;
+        ? from.refTag === to.refTag && units
+        : from.repr !== "single" && from.repr === to.repr && units;
     if (same) return host.own({ name: value.name, type: e.type });
     return host.own({ name: rebuildRetype(host, value.name, e), type: e.type });
   }

packages/compiler/src/backend/llvm/union-retype.test.ts

@@ -268,3 +268,51 @@ test("record unions retype as the record pointer itself", async () => {
   // No record of the visitor is ever boxed.
   expect(ir).not.toMatch(/call ptr @scr_union_new_ref/);
 });
+
+// Children that may be absent, as IR nodes have: reading one after its
+// `=== undefined` or `=== null` test narrows a shape union with a unit arm
+// to the shape union without it.
+const OPTIONAL_CHILDREN = `
+type Num = { kind: "num"; value: number };
+type Ref = { kind: "ref"; id: string };
+type Call = { kind: "call"; callee: Expr; receiver?: Expr; args: Expr[] };
+type Init = { kind: "init"; value: Expr | null };
+type Expr = Num | Ref | Call | Init;
+function count(e: Expr): number {
+  switch (e.kind) {
+    case "num":
+    case "ref":
+      return 1;
+    case "call":
+      return (
+        1 +
+        count(e.callee) +
+        (e.receiver === undefined ? 0 : count(e.receiver)) +
+        e.args.reduce((n, a) => n + count(a), 0)
+      );
+    case "init":
+      return 1 + (e.value === null ? 0 : count(e.value));
+  }
+}
+const tree: Expr = {
+  kind: "call",
+  callee: { kind: "ref", id: "f" },
+  receiver: { kind: "init", value: { kind: "num", value: 1 } },
+  args: [{ kind: "init", value: null }],
+};
+console.log(count(tree));
+`;
+
+test("dropping a shape union's unit arm keeps the record pointer", async () => {
+  const { module, ir } = await llvmFor(OPTIONAL_CHILDREN, false);
+  const identities = module.functions.filter(
+    (fn) =>
+      fn.name.startsWith("%union.retag.") &&
+      JSON.stringify(fn.body).includes('"kind":"unionRetype"'),
+  );
+  expect(identities.length).toBeGreaterThan(0);
+  for (const helper of identities) {
+    const symbol = `sc_bf__x25_union_retag_${helper.name.slice("%union.retag.".length)}`;
+    expect(body(ir, symbol)).not.toMatch(/retype\.built/);
+  }
+});

packages/runtime/src/scr_cycle.c

@@ -235,6 +235,7 @@ typedef struct {
   bool falling;
   bool idle_points;
   size_t death_due_live;
+  bool probe_due;
   bool keep_base;
   size_t trigger;
   size_t alloc_trigger;
@@ -286,6 +287,7 @@ static _Thread_local ScrCycThread scr_cyc_thread = {.gen_limit = SCR_CYC_OLD};
 #define scr_cyc_falling (scr_cyc_thread.falling)
 #define scr_cyc_idle_points (scr_cyc_thread.idle_points)
 #define scr_cyc_death_due_live (scr_cyc_thread.death_due_live)
+#define scr_cyc_probe_due (scr_cyc_thread.probe_due)
 #define scr_cyc_keep_base (scr_cyc_thread.keep_base)
 #define scr_cyc_kinds (scr_cyc_thread.kinds)
 #define scr_cyc_kinds_cap (scr_cyc_thread.kinds_cap)
@@ -640,6 +642,10 @@ static SCR_TL bool scr_cyc_falling = false, scr_cyc_keep_base = false;
 #ifndef SCR_WORKERS
 static SCR_TL bool scr_cyc_idle_points = false;
 static SCR_TL size_t scr_cyc_death_due_live = 0;
+/* A nursery probe window opened since the last scheduled pass (see "level
+ * bypass"): the next pass the release path would run at the nursery runs at
+ * the mature level instead. */
+static SCR_TL bool scr_cyc_probe_due = false;
 #endif
 
 /* The nursery trigger a program sets for itself
@@ -976,9 +982,11 @@ static SCR_CYC_SCHEDULING void scr_cyc_collect_due(void) {
   if (!by_growth && !by_death && scr_roots[SCR_CYC_OLD].n < scr_cyc_backlog_threshold(SCR_CYC_OLD)) {
     gen = SCR_CYC_MATURE;
     by_growth = scr_cyc_grown(SCR_CYC_MATURE);
-    if (!by_growth && scr_roots[SCR_CYC_MATURE].n < scr_cyc_backlog_threshold(SCR_CYC_MATURE))
+    if (!by_growth && scr_roots[SCR_CYC_MATURE].n < scr_cyc_backlog_threshold(SCR_CYC_MATURE) &&
+        !scr_cyc_probe_due)
       gen = SCR_CYC_NURSERY;
   }
+  scr_cyc_probe_due = false;
   scr_cyc_scheduled_pass(gen, by_growth, by_death);
 }
 
@@ -1059,6 +1067,16 @@ bool scr_cyc_collect_idle(void) {
  * a probe window every candidate of the type stays at the level, so the
  * cycles it builds during the window die there.
  *
+ * Those young cycles can still be held by older garbage: a dead structure
+ * the bypass promoted before the window that points at objects built in it.
+ * A restricted pass reads that edge as external, so the window frees none
+ * of them, judges the type long-lived again, and the garbage on both sides
+ * waits for a mature pass, which the growth schedule may not run until the
+ * heap has grown a multiple of its size. So a nursery window that opens
+ * makes the next release-triggered pass that would run at the nursery run
+ * at the mature level: it reclaims the promoted garbage once, and the window
+ * judges the young cycles on their own.
+ *
  * This changes only which generation an object is judged in. Any
  * generation assignment is sound (a restricted pass treats edges from older
  * objects as external), and a bypassing candidate is buffered like any
@@ -1110,7 +1128,8 @@ static inline ScrCycKind *scr_cyc_kind(uint32_t kind) {
 
 /* Whether a candidate of this type at level `gen` (nursery or mature)
  * skips that level's passes (see above). The last candidate a bypass sends
- * past the level opens a probe window, with fresh counts. */
+ * past the level opens a probe window, with fresh counts, and a nursery
+ * window asks for a mature pass first. */
 static inline __attribute__((always_inline)) bool scr_cyc_bypassed(const ScrCycType *type,
                                                                   unsigned gen) {
   uint32_t kind = type->kind;
@@ -1119,9 +1138,12 @@ static inline __attribute__((always_inline)) bool scr_cyc_bypassed(const ScrCycT
   if (k->bypass == 0) return false;
   if (--k->bypass == 0) {
     k->walked = k->freed = 0;
+    if (gen == SCR_CYC_NURSERY) {
+      scr_cyc_probe_due = true;
 #if SCR_CYC_INLINE_BUFFERING
-    if (gen == SCR_CYC_NURSERY) scr_cyc_nursery_bypassing--;
+      scr_cyc_nursery_bypassing--;
 #endif
+    }
   }
   return true;
 }

packages/runtime/test/test_cycle.c

@@ -1039,6 +1039,75 @@ static void check_nursery_bypass(void) {
   scr_collect_cycles();
 }
 
+/* A nursery probe window that opens makes the next release-triggered pass
+ * that would run at the nursery run at the mature level, which reclaims
+ * dead promoted rings a nursery pass stops at. */
+static size_t window_kind_freed, window_ring_freed;
+static void window_kind_free(void *obj) {
+  window_kind_freed++;
+  node_free(obj);
+}
+static void window_ring_free(void *obj) {
+  window_ring_freed++;
+  node_free(obj);
+}
+static ScrCycType window_kind_type = { &node_trace, &window_kind_free, 9 };
+static ScrCycType window_ring_type = { &node_trace, &window_ring_free, 0 };
+
+/* A candidate of the window kind that dies once it has been routed, as the
+ * next pass would route it: it counts down a bypass without growing the
+ * heap. */
+static void release_window_candidate(void) {
+  Node *node = scr_cyc_alloc(sizeof(*node), &window_kind_type);
+  node->rc = 2;
+  release_live(node);
+  scr_cyc_route_bypassed_at();
+  node->rc = 0;
+  scr_cyc_on_dead(node);
+  window_kind_free(node);
+}
+
+static void check_probe_window_runs_mature_pass(void) {
+  /* Every release collects at a threshold of 1, and passes with only a few
+   * candidates walk too few of them to decide a window. */
+  if (configured_nursery_threshold() < 8) return;
+  enum { WINDOW = 8192, INTERVAL = 16, LIVE = WINDOW, RING = 4 };
+  Node **live = calloc(LIVE, sizeof(*live));
+  if (!live) scr_trap("cycle test: out of memory\n");
+  for (size_t i = 0; i < LIVE; i++) {
+    live[i] = scr_cyc_alloc(sizeof(*live[i]), &window_kind_type);
+    live[i]->rc = 1;
+    read_live(live[i]);
+    scr_cyc_pass_at(SCR_CYC_NURSERY);
+  }
+  scr_collect_cycles();
+  /* A pass that an earlier window asked for runs now. */
+  run_release_passes();
+
+  /* A dead ring promoted to the mature level: nursery passes stop at it. */
+  Node *ring = make_ring_of(RING, &window_ring_type);
+  ring->rc++;
+  release_live(ring);
+  scr_cyc_pass_at(SCR_CYC_NURSERY);
+  release_live(ring);
+  run_release_passes();
+  check(window_ring_freed == 0, "a nursery pass freed a dead mature ring");
+
+  /* The window kind's candidates go past the nursery until a probe window
+   * opens; the next release-triggered pass runs at the mature level. */
+  for (size_t i = 0; i < (size_t)WINDOW * INTERVAL; i++) release_window_candidate();
+  run_release_passes();
+  check(window_ring_freed == RING, "the pass after a probe window opened missed a dead mature ring");
+
+  for (size_t i = 0; i < LIVE; i++) {
+    live[i]->rc = 0;
+    scr_cyc_on_dead(live[i]);
+    window_kind_free(live[i]);
+  }
+  free(live);
+  scr_collect_cycles();
+}
+
 /* The mature level's bypass: a type whose mature candidates keep surviving
  * goes straight to the old generation, its dead cycles are found by full
  * passes, and once its objects die at the mature level again it is judged
@@ -1719,6 +1788,7 @@ int main(void) {
   check_death_brings_full_pass();
   check_nursery_bypass();
   check_mature_bypass();
+  check_probe_window_runs_mature_pass();
   check_probe_sees_candidate_rings();
   check_rebuffer_past_mature();
   check_allocation_growth_runs_passes();