Collect the frontend's cycles when the native compiler disposes it (#1671)

packages/compiler/native/host.c

@@ -44,3 +44,15 @@ uint8_t scriptc_native_private_directory(const uint8_t *bytes, size_t size, uint
   return ok;
 #endif
 }
+
+/* The runtime's full sweep after a phase's data died (scr_rc.h). */
+void scr_collect_cycles_phase(void);
+
+/* Reclaims the cycles a finished compiler phase leaves behind. The
+ * frontend's syntax trees and program state refer to each other, so
+ * reference counting frees almost none of them when the frontend is
+ * disposed, and the collector's schedule alone may not run a full pass
+ * before the build ends. */
+void scriptc_native_collect_cycles(void) {
+  scr_collect_cycles_phase();
+}

packages/compiler/native/host.ffi.json

@@ -1,6 +1,7 @@
 {
   "ffi_format": 6,
   "functions": [
-    { "name": "compilerNativePrivateDirectory", "symbol": "scriptc_native_private_directory", "params": ["string", "bool"], "returns": "bool" }
+    { "name": "compilerNativePrivateDirectory", "symbol": "scriptc_native_private_directory", "params": ["string", "bool"], "returns": "bool" },
+    { "name": "compilerNativeCollectCycles", "symbol": "scriptc_native_collect_cycles", "params": [], "returns": "void" }
   ]
 }

packages/compiler/src/executable/prepare.ts

@@ -1,5 +1,6 @@
 import type { CompileFailure, CompileRequestOptions } from "../compile-types.js";
 import type { FfiProfile } from "../ffi/ffi-manifest.js";
+import { frontendDisposed } from "../frontend/dispose-hook.js";
 import {
   checkerPanicDiag,
   npmStaticRefusalDiag,
@@ -112,6 +113,7 @@ export function prepareExecutableModule(
     }
   } finally {
     fe.dispose();
+    frontendDisposed();
     timing("frontend-dispose");
   }
 

packages/compiler/src/frontend/dispose-hook.ts

@@ -0,0 +1,14 @@
+/* What runs right after a build disposes its frontend. The native compiler
+ * sets it to a full cycle collection (native/cli.ts): the frontend's syntax
+ * trees and program state refer to each other, so disposing them leaves
+ * cyclic garbage that reference counting cannot free. Under Node it is
+ * unset. */
+let afterDispose: (() => void) | null = null;
+
+export function setFrontendDisposedHook(hook: () => void): void {
+  afterDispose = hook;
+}
+
+export function frontendDisposed(): void {
+  if (afterDispose !== null) afterDispose();
+}

packages/compiler/src/library/prepare.ts

@@ -1,5 +1,6 @@
 import { dirname, resolve } from "node:path";
 import type { CompileFailure } from "../compile-types.js";
+import { frontendDisposed } from "../frontend/dispose-hook.js";
 import {
   fenceSpeculativeWasiFunctions,
   moduleWasiUnavailableSurface,
@@ -585,6 +586,7 @@ export function prepareLibrary(
     sourceTexts = fe.sourceTexts();
   } finally {
     fe.dispose();
+    frontendDisposed();
   }
   const mod = lowered.module!;
   timing("frontend-dispose");

packages/compiler/src/native/cli.ts

@@ -10,6 +10,7 @@ import {
 } from "../backend/runtime-pack-native.js";
 import { mergeNativeProfiles } from "../backend/native-tools.js";
 import { setDeclarationRoot } from "../frontend/dts-paths.js";
+import { setFrontendDisposedHook } from "../frontend/dispose-hook.js";
 import { FrontendServices } from "../frontend/services.js";
 import { createNativeTs7Api } from "../frontend/ts7/native-api.js";
 import { resolveProvenanceSourcesWithParser } from "../frontend/provenance-core.js";
@@ -21,8 +22,10 @@ import { buildSanitizedRuntime } from "./sanitizer.js";
 import { runCompilerTask } from "./task.js";
 
 declare function compilerNativePrivateDirectory(path: string, harden: boolean): boolean;
+declare function compilerNativeCollectCycles(): void;
 
 async function main(): Promise<number> {
+  setFrontendDisposedHook(() => compilerNativeCollectCycles());
   const manifestPath = process.env["SCRIPTC_TOOLCHAIN"] ?? process.execPath + ".json";
   let compiler: NativeCompiler | null = null;
   let cache: ReturnType<typeof openNativeCache> = null;

packages/runtime/src/scr_cycle.c

@@ -2195,6 +2195,23 @@ void scr_collect_cycles(void) {
   }
 }
 
+/* The full sweep, for a caller that knows a phase's data has just died (a
+ * compiler dropping its frontend). Such data is mostly cyclic, so reference
+ * counting frees little of it and the schedule cannot see it die. The
+ * growth fractions the phase widened, while its live data kept passes from
+ * finding anything, and the death rule's backoff then start over from the
+ * heap the sweep leaves, as after a productive pass. Otherwise the next
+ * phase's garbage would accumulate until the heap grew to that widened
+ * fraction. */
+void scr_collect_cycles_phase(void) {
+  scr_collect_cycles();
+  for (unsigned gen = SCR_CYC_MATURE; gen < SCR_CYC_NGENS; gen++) {
+    scr_cyc_growth_shift[gen] = 0;
+    scr_cyc_growth_streak[gen] = 0;
+  }
+  scr_cyc_death_shift = 0;
+}
+
 /* Worker TLS becomes unreachable when its OS thread returns. Release the
  * reusable traversal buffers after all context-owned roots are retired. */
 void scr_cyc_context_cleanup(void) {

packages/runtime/src/scr_rc.h

@@ -619,6 +619,7 @@ static inline void scr_cyc_dead(void *obj) {
  * after and wants nothing reclaimable left), and it costs a walk of the
  * live heap — do not put it on a per-turn path. */
 void scr_collect_cycles(void);
+void scr_collect_cycles_phase(void);
 void scr_cyc_context_cleanup(void);
 
 /* One pass on the normal generational schedule, for callers that reach a

packages/runtime/test/test_cycle.c

@@ -536,6 +536,29 @@ static void check_mixed_dead_structure_narrows(void) {
   check(live <= narrowed, "a pass that freed a dead mature and old structure did not narrow the growth fraction");
 }
 
+/* The phase sweep (scr_collect_cycles_phase) is for a caller that knows a
+ * phase's data has just died: it sweeps, and the growth schedule starts
+ * over. After passes that find nothing have widened the old level's
+ * fraction to the cap, the plain sweep leaves the next dead cycle to the
+ * cap, while after the phase sweep it goes once the heap has grown the base
+ * quarter past the floor. */
+static void check_phase_sweep_restarts_schedule(void) {
+  enum { FLOOR = 4096 };
+  size_t cap = configured_growth_cap();
+  if (cap < 2) return; /* the base fraction is already the cap */
+  size_t slack = 2 * configured_nursery_threshold() + 2;
+  size_t due = FLOOR + FLOOR * cap;
+  size_t live = 0;
+  for (int i = 0; i < 4; i++) live = grow_dead_cycle_until_full_pass(0, false);
+  check(live > due, "passes that found nothing did not widen the growth fraction to the cap");
+  scr_collect_cycles();
+  live = grow_dead_cycle_until_full_pass(0, false);
+  check(live > due, "the plain sweep changed the growth schedule");
+  scr_collect_cycles_phase();
+  live = grow_dead_cycle_until_full_pass(0, false);
+  check(live <= FLOOR + FLOOR / 4 + slack, "the phase sweep did not restart the growth schedule");
+}
+
 static void check_deep_ring(void) {
   enum { DEPTH = 100000 };
   size_t before = freed;
@@ -1433,6 +1456,7 @@ int main(void) {
   check_growth_cap_bounds_floating_garbage();
   check_mixed_dead_structure_narrows();
   check_death_waits_for_idle();
+  check_phase_sweep_restarts_schedule();
   check_death_brings_full_pass();
   check_nursery_bypass();
   check_mature_bypass();