From 050bbd852c15a2cf720b987bbb58c093b0b3836c Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Wed, 21 Oct 2026 09:09:19 -0500 Subject: [PATCH] Collect the frontend's cycles when the native compiler disposes it The frontend's syntax trees and program state refer to each other, so reference counting frees almost none of them when a build disposes the frontend. Whether a cycle pass reclaimed them before the build ended depended on where the collector's schedule happened to be. Passes during the frontend found its data live and widened the growth fractions, so the next pass waited until the heap had grown well past its size at disposal, and the dead frontend and the emitter's garbage stayed until the build ended. The native compiler now runs scr_collect_cycles_phase right after it disposes the frontend, through a host function. The runtime entry point sweeps, then restarts the growth fractions and the death rule's backoff from the heap the sweep leaves, as after a productive pass. Under Node the hook is unset. --- packages/compiler/native/host.c | 12 ++++++++++ packages/compiler/native/host.ffi.json | 3 ++- packages/compiler/src/executable/prepare.ts | 2 ++ .../compiler/src/frontend/dispose-hook.ts | 14 +++++++++++ packages/compiler/src/library/prepare.ts | 2 ++ packages/compiler/src/native/cli.ts | 3 +++ packages/runtime/src/scr_cycle.c | 17 +++++++++++++ packages/runtime/src/scr_rc.h | 1 + packages/runtime/test/test_cycle.c | 24 +++++++++++++++++++ 9 files changed, 77 insertions(+), 1 deletion(-) create mode 100644 packages/compiler/src/frontend/dispose-hook.ts diff --git a/packages/compiler/native/host.c b/packages/compiler/native/host.c index 75c2dcbaf..6aa1b2a28 100644 --- a/packages/compiler/native/host.c +++ b/packages/compiler/native/host.c @@ -44,3 +44,15 @@ uint8_t scriptc_native_private_directory(const uint8_t *bytes, size_t size, uint return ok; #endif } + +/* The runtime's full sweep after a phase's data died (scr_rc.h). */ +void scr_collect_cycles_phase(void); + +/* Reclaims the cycles a finished compiler phase leaves behind. The + * frontend's syntax trees and program state refer to each other, so + * reference counting frees almost none of them when the frontend is + * disposed, and the collector's schedule alone may not run a full pass + * before the build ends. */ +void scriptc_native_collect_cycles(void) { + scr_collect_cycles_phase(); +} diff --git a/packages/compiler/native/host.ffi.json b/packages/compiler/native/host.ffi.json index f2a738054..d7e0e92aa 100644 --- a/packages/compiler/native/host.ffi.json +++ b/packages/compiler/native/host.ffi.json @@ -1,6 +1,7 @@ { "ffi_format": 6, "functions": [ - { "name": "compilerNativePrivateDirectory", "symbol": "scriptc_native_private_directory", "params": ["string", "bool"], "returns": "bool" } + { "name": "compilerNativePrivateDirectory", "symbol": "scriptc_native_private_directory", "params": ["string", "bool"], "returns": "bool" }, + { "name": "compilerNativeCollectCycles", "symbol": "scriptc_native_collect_cycles", "params": [], "returns": "void" } ] } diff --git a/packages/compiler/src/executable/prepare.ts b/packages/compiler/src/executable/prepare.ts index d97497c70..4afae7c7d 100644 --- a/packages/compiler/src/executable/prepare.ts +++ b/packages/compiler/src/executable/prepare.ts @@ -1,5 +1,6 @@ import type { CompileFailure, CompileRequestOptions } from "../compile-types.js"; import type { FfiProfile } from "../ffi/ffi-manifest.js"; +import { frontendDisposed } from "../frontend/dispose-hook.js"; import { checkerPanicDiag, npmStaticRefusalDiag, @@ -112,6 +113,7 @@ export function prepareExecutableModule( } } finally { fe.dispose(); + frontendDisposed(); timing("frontend-dispose"); } diff --git a/packages/compiler/src/frontend/dispose-hook.ts b/packages/compiler/src/frontend/dispose-hook.ts new file mode 100644 index 000000000..ccc9b19aa --- /dev/null +++ b/packages/compiler/src/frontend/dispose-hook.ts @@ -0,0 +1,14 @@ +/* What runs right after a build disposes its frontend. The native compiler + * sets it to a full cycle collection (native/cli.ts): the frontend's syntax + * trees and program state refer to each other, so disposing them leaves + * cyclic garbage that reference counting cannot free. Under Node it is + * unset. */ +let afterDispose: (() => void) | null = null; + +export function setFrontendDisposedHook(hook: () => void): void { + afterDispose = hook; +} + +export function frontendDisposed(): void { + if (afterDispose !== null) afterDispose(); +} diff --git a/packages/compiler/src/library/prepare.ts b/packages/compiler/src/library/prepare.ts index 298bb43bb..fdaf8f506 100644 --- a/packages/compiler/src/library/prepare.ts +++ b/packages/compiler/src/library/prepare.ts @@ -1,5 +1,6 @@ import { dirname, resolve } from "node:path"; import type { CompileFailure } from "../compile-types.js"; +import { frontendDisposed } from "../frontend/dispose-hook.js"; import { fenceSpeculativeWasiFunctions, moduleWasiUnavailableSurface, @@ -585,6 +586,7 @@ export function prepareLibrary( sourceTexts = fe.sourceTexts(); } finally { fe.dispose(); + frontendDisposed(); } const mod = lowered.module!; timing("frontend-dispose"); diff --git a/packages/compiler/src/native/cli.ts b/packages/compiler/src/native/cli.ts index d7520ae0d..ed8d52293 100644 --- a/packages/compiler/src/native/cli.ts +++ b/packages/compiler/src/native/cli.ts @@ -10,6 +10,7 @@ import { } from "../backend/runtime-pack-native.js"; import { mergeNativeProfiles } from "../backend/native-tools.js"; import { setDeclarationRoot } from "../frontend/dts-paths.js"; +import { setFrontendDisposedHook } from "../frontend/dispose-hook.js"; import { FrontendServices } from "../frontend/services.js"; import { createNativeTs7Api } from "../frontend/ts7/native-api.js"; import { resolveProvenanceSourcesWithParser } from "../frontend/provenance-core.js"; @@ -21,8 +22,10 @@ import { buildSanitizedRuntime } from "./sanitizer.js"; import { runCompilerTask } from "./task.js"; declare function compilerNativePrivateDirectory(path: string, harden: boolean): boolean; +declare function compilerNativeCollectCycles(): void; async function main(): Promise { + setFrontendDisposedHook(() => compilerNativeCollectCycles()); const manifestPath = process.env["SCRIPTC_TOOLCHAIN"] ?? process.execPath + ".json"; let compiler: NativeCompiler | null = null; let cache: ReturnType = null; diff --git a/packages/runtime/src/scr_cycle.c b/packages/runtime/src/scr_cycle.c index 93180deca..ce4dfa5fa 100644 --- a/packages/runtime/src/scr_cycle.c +++ b/packages/runtime/src/scr_cycle.c @@ -2195,6 +2195,23 @@ void scr_collect_cycles(void) { } } +/* The full sweep, for a caller that knows a phase's data has just died (a + * compiler dropping its frontend). Such data is mostly cyclic, so reference + * counting frees little of it and the schedule cannot see it die. The + * growth fractions the phase widened, while its live data kept passes from + * finding anything, and the death rule's backoff then start over from the + * heap the sweep leaves, as after a productive pass. Otherwise the next + * phase's garbage would accumulate until the heap grew to that widened + * fraction. */ +void scr_collect_cycles_phase(void) { + scr_collect_cycles(); + for (unsigned gen = SCR_CYC_MATURE; gen < SCR_CYC_NGENS; gen++) { + scr_cyc_growth_shift[gen] = 0; + scr_cyc_growth_streak[gen] = 0; + } + scr_cyc_death_shift = 0; +} + /* Worker TLS becomes unreachable when its OS thread returns. Release the * reusable traversal buffers after all context-owned roots are retired. */ void scr_cyc_context_cleanup(void) { diff --git a/packages/runtime/src/scr_rc.h b/packages/runtime/src/scr_rc.h index 889202975..39e962512 100644 --- a/packages/runtime/src/scr_rc.h +++ b/packages/runtime/src/scr_rc.h @@ -601,6 +601,7 @@ static inline void scr_cyc_dead(void *obj) { * after and wants nothing reclaimable left), and it costs a walk of the * live heap — do not put it on a per-turn path. */ void scr_collect_cycles(void); +void scr_collect_cycles_phase(void); void scr_cyc_context_cleanup(void); /* One pass on the normal generational schedule, for callers that reach a diff --git a/packages/runtime/test/test_cycle.c b/packages/runtime/test/test_cycle.c index d4deec3c3..723e9519f 100644 --- a/packages/runtime/test/test_cycle.c +++ b/packages/runtime/test/test_cycle.c @@ -536,6 +536,29 @@ static void check_mixed_dead_structure_narrows(void) { check(live <= narrowed, "a pass that freed a dead mature and old structure did not narrow the growth fraction"); } +/* The phase sweep (scr_collect_cycles_phase) is for a caller that knows a + * phase's data has just died: it sweeps, and the growth schedule starts + * over. After passes that find nothing have widened the old level's + * fraction to the cap, the plain sweep leaves the next dead cycle to the + * cap, while after the phase sweep it goes once the heap has grown the base + * quarter past the floor. */ +static void check_phase_sweep_restarts_schedule(void) { + enum { FLOOR = 4096 }; + size_t cap = configured_growth_cap(); + if (cap < 2) return; /* the base fraction is already the cap */ + size_t slack = 2 * configured_nursery_threshold() + 2; + size_t due = FLOOR + FLOOR * cap; + size_t live = 0; + for (int i = 0; i < 4; i++) live = grow_dead_cycle_until_full_pass(0, false); + check(live > due, "passes that found nothing did not widen the growth fraction to the cap"); + scr_collect_cycles(); + live = grow_dead_cycle_until_full_pass(0, false); + check(live > due, "the plain sweep changed the growth schedule"); + scr_collect_cycles_phase(); + live = grow_dead_cycle_until_full_pass(0, false); + check(live <= FLOOR + FLOOR / 4 + slack, "the phase sweep did not restart the growth schedule"); +} + static void check_deep_ring(void) { enum { DEPTH = 100000 }; size_t before = freed; @@ -1433,6 +1456,7 @@ int main(void) { check_growth_cap_bounds_floating_garbage(); check_mixed_dead_structure_narrows(); check_death_waits_for_idle(); + check_phase_sweep_restarts_schedule(); check_death_brings_full_pass(); check_nursery_bypass(); check_mature_bypass();