From 0cc067430843f259fdcd1834bb4336e6d7121db3 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Wed, 21 Oct 2026 08:23:18 -0500 Subject: [PATCH] Prepare the first build of an executable in a frontend process The frontend, lowering and LLVM emission of an optimized executable build run in a child process when no earlier build of the same invocation emitted its native program object. The parent passes the LLVM text to the native helper as the file the child wrote, so the helper runs after the pages the frontend used are released. Later builds of the invocation find most native partitions cached and peak in the frontend, so they, along with development builds and builds with SCRIPTC_FRONTEND_PROCESS=0, prepare their input in-process. The child runs with the parent's Node options without --eval, --print and their code, or inspector, watch and test runner options. With SCRIPTC_TIMING=1, it reports its phases on the parent's clock. --- packages/cli/test/frontend-process.test.ts | 172 ++++++++++ .../compiler/src/backend/native-codegen.ts | 54 ++- .../src/executable/executable-cache.test.ts | 37 ++ .../src/executable/executable-cache.ts | 33 ++ .../src/frontend-process-args.test.ts | 56 ++++ .../compiler/src/frontend-process-args.ts | 54 +++ packages/compiler/src/frontend-process.ts | 18 + packages/compiler/src/index.ts | 315 ++++++++++++++++-- packages/compiler/src/timing.ts | 17 +- 9 files changed, 714 insertions(+), 42 deletions(-) create mode 100644 packages/cli/test/frontend-process.test.ts create mode 100644 packages/compiler/src/frontend-process-args.test.ts create mode 100644 packages/compiler/src/frontend-process-args.ts create mode 100644 packages/compiler/src/frontend-process.ts diff --git a/packages/cli/test/frontend-process.test.ts b/packages/cli/test/frontend-process.test.ts new file mode 100644 index 000000000..8f7f85e4d --- /dev/null +++ b/packages/cli/test/frontend-process.test.ts @@ -0,0 +1,172 @@ +import { execFile } from "node:child_process"; +import { mkdir, mkdtemp, readdir, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { pathToFileURL } from "node:url"; +import { promisify } from "node:util"; +import { expect, test } from "vitest"; +import { precompiledRuntimePackTarget } from "../../compiler/src/startup-cache.js"; + +// The first optimized executable build of an invocation prepares its input in a frontend process +// (prepareExecutableInputInChild in the compiler's index.ts) when the compiler runs from its built +// files, as the CLI's bootstrap does. Later builds of the invocation, and builds with +// SCRIPTC_FRONTEND_PROCESS=0, prepare it in the compiler's own process. +const execFileAsync = promisify(execFile); +const repoRoot = join(import.meta.dirname, "../../.."); +const bootstrap = join(repoRoot, "packages/cli/dist/bootstrap.js"); +const compilerApi = pathToFileURL(join(repoRoot, "packages/compiler/dist/index.js")).href; +const helperTarget = precompiledRuntimePackTarget(); + +interface Build { + code: number; + stdout: string; + stderr: string; +} + +async function build( + entry: string, + outPath: string, + cacheRoot: string, + inProcess: boolean, +): Promise { + const env: NodeJS.ProcessEnv = { + ...process.env, + SCRIPTC_CACHE_DIR: cacheRoot, + SCRIPTC_TIMING: "1", + }; + delete env["SCRIPTC_NO_CACHE"]; + if (inProcess) env["SCRIPTC_FRONTEND_PROCESS"] = "0"; + else delete env["SCRIPTC_FRONTEND_PROCESS"]; + try { + const { stdout, stderr } = await execFileAsync( + process.execPath, + [bootstrap, "build", entry, "-o", outPath], + { env, maxBuffer: 8 * 1024 * 1024 }, + ); + return { code: 0, stdout, stderr }; + } catch (error) { + const failed = error as { code?: number; stdout?: string; stderr?: string }; + return { code: failed.code ?? 1, stdout: failed.stdout ?? "", stderr: failed.stderr ?? "" }; + } +} + +/** The build's output without its timing lines, whose times differ between runs. */ +const withoutTiming = (text: string): string => + text + .split("\n") + .filter((line) => !line.startsWith("scriptc timing") && !line.startsWith("scriptc lowering")) + .join("\n"); + +test.skipIf(helperTarget === null)( + "the frontend process builds the executable, diagnostics and cache entries the in-process build does", + async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-frontend-process-")); + try { + const src = join(dir, "src"); + await mkdir(src); + await writeFile( + join(src, "shapes.ts"), + "export const area = (w: number, h: number): number => w * h;\n", + ); + await writeFile( + join(src, "main.ts"), + 'import { area } from "./shapes.js";\nconsole.log(area(3, 4), [1, 2, 3].map((x) => x * 2).join(","));\n', + ); + const entry = join(src, "main.ts"); + const out = (name: string): string => join(dir, name, "program"); + await mkdir(join(dir, "child")); + await mkdir(join(dir, "inline")); + + const child = await build(entry, out("child"), join(dir, "cache-child"), false); + const inline = await build(entry, out("inline"), join(dir, "cache-inline"), true); + expect(child.code).toBe(0); + expect(inline.code).toBe(0); + expect(child.stderr).toContain('"phase":"frontend-process"'); + expect(inline.stderr).not.toContain('"phase":"frontend-process"'); + expect(await readFile(out("child"))).toEqual(await readFile(out("inline"))); + expect(await readFile(join(dir, "child", "main.ll"), "utf8")).toBe( + await readFile(join(dir, "inline", "main.ll"), "utf8"), + ); + expect((await execFileAsync(out("child"))).stdout).toBe("12 2,4,6\n"); + // The request and result files are removed. + expect( + (await readdir(join(dir, "child"))).filter((name) => name.includes("frontend")), + ).toEqual([]); + + // A repeat finds the executable the frontend process's build published. An edit rebuilds in + // the compiler's own process. + const repeat = await build(entry, out("child"), join(dir, "cache-child"), false); + expect(repeat.code).toBe(0); + expect(repeat.stderr).not.toContain("scriptc lowering"); + expect(repeat.stderr).not.toContain('"phase":"frontend-process"'); + await writeFile( + join(src, "shapes.ts"), + "export const area = (w: number, h: number): number => w * h + 1;\n", + ); + const edited = await build(entry, out("child"), join(dir, "cache-child"), false); + expect(edited.code).toBe(0); + expect(edited.stderr).toContain("scriptc lowering"); + expect(edited.stderr).not.toContain('"phase":"frontend-process"'); + expect((await execFileAsync(out("child"))).stdout).toBe("13 2,4,6\n"); + + // Without the executable cache entries (as for a program whose frontend inputs cannot be + // cached), the rebuild still runs in the compiler's own process: the first build recorded + // its native program object. + await rm(join(dir, "cache-child", "early-exe"), { recursive: true, force: true }); + await rm(join(dir, "cache-child", "early-exe-route"), { recursive: true, force: true }); + const uncached = await build(entry, out("child"), join(dir, "cache-child"), false); + expect(uncached.code).toBe(0); + expect(uncached.stderr).toContain("scriptc lowering"); + expect(uncached.stderr).not.toContain('"phase":"frontend-process"'); + expect((await execFileAsync(out("child"))).stdout).toBe("13 2,4,6\n"); + + // A program the frontend rejects reports the same diagnostics either way. The first build of + // the invocation in an empty cache runs the frontend process. + await writeFile(join(src, "main.ts"), 'const n: number = "not a number";\nconsole.log(n);\n'); + const childError = await build(entry, out("child"), join(dir, "cache-child-error"), false); + const inlineError = await build(entry, out("inline"), join(dir, "cache-inline"), true); + expect(childError.stderr).toContain('"phase":"frontend-process"'); + expect(childError.code).not.toBe(0); + expect(childError.code).toBe(inlineError.code); + expect(withoutTiming(childError.stderr)).toBe(withoutTiming(inlineError.stderr)); + expect(withoutTiming(childError.stderr)).toContain("not a number"); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }, + 240_000, +); + +test.skipIf(helperTarget === null)( + "a compile called from evaluated code runs its frontend process once", + async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-frontend-process-eval-")); + try { + const entry = join(dir, "main.ts"); + await writeFile(entry, 'console.log("from eval");\n'); + const { stdout, stderr } = await execFileAsync( + process.execPath, + [ + "--input-type=module", + "--eval", + `import { compile } from ${JSON.stringify(compilerApi)}; + const result = await compile(process.argv[1], { outDir: process.argv[2], outPath: process.argv[3] }); + console.log("compiled", result.ok);`, + entry, + dir, + join(dir, "program"), + ], + { + env: { ...process.env, SCRIPTC_CACHE_DIR: join(dir, "cache"), SCRIPTC_TIMING: "1" }, + maxBuffer: 8 * 1024 * 1024, + }, + ); + expect(stdout).toBe("compiled true\n"); + expect(stderr).toContain('"phase":"frontend-process"'); + expect((await execFileAsync(join(dir, "program"))).stdout).toBe("from eval\n"); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }, + 240_000, +); diff --git a/packages/compiler/src/backend/native-codegen.ts b/packages/compiler/src/backend/native-codegen.ts index 6aab5880a..ff411922a 100644 --- a/packages/compiler/src/backend/native-codegen.ts +++ b/packages/compiler/src/backend/native-codegen.ts @@ -1,8 +1,8 @@ import { execFile } from "node:child_process"; -import { createHash } from "node:crypto"; -import { constants } from "node:fs"; +import { createHash, type Hash } from "node:crypto"; +import { closeSync, constants, openSync, readSync } from "node:fs"; import { createRequire } from "node:module"; -import { access, chmod, mkdir, readFile, rename, rm, stat } from "node:fs/promises"; +import { access, chmod, copyFile, mkdir, readFile, rename, rm, stat } from "node:fs/promises"; import { dirname, join } from "node:path"; import { promisify } from "node:util"; import { @@ -68,9 +68,31 @@ export interface NativeCodegenArtifact { const resolvedHelperCache = new Map>(); +/** LLVM text already written to `file`, of `bytes` bytes, which a build passes on without reading it. */ +export interface LlvmFile { + readonly file: string; + readonly bytes: number; +} + +/** LLVM text as a string, as parts to concatenate, or as a file. */ +export type LlvmInput = string | readonly string[] | LlvmFile; + +export function isLlvmFile(llvm: LlvmInput): llvm is LlvmFile { + return typeof llvm === "object" && !Array.isArray(llvm); +} + +/** The size of `llvm` in bytes. */ +export function llvmInputBytes(llvm: LlvmInput): number { + if (isLlvmFile(llvm)) return llvm.bytes; + return (typeof llvm === "string" ? [llvm] : llvm).reduce( + (bytes, part) => bytes + Buffer.byteLength(part), + 0, + ); +} + export interface NativeCodegenOptions { outputPath: string; - llvm: string | readonly string[]; + llvm: LlvmInput; outputKind: NativeCodegenOutputKind; sourcePath: string; optimization?: "0" | "1" | "2" | "3" | "s" | "z"; @@ -271,14 +293,31 @@ async function resolveHelper( return pending; } +/** Adds the bytes of `file` to `hash`, as `hash.update` of the text written there would. */ +function updateHashFromFile(hash: Hash, file: string): void { + const fd = openSync(file, "r"); + try { + const buffer = Buffer.allocUnsafe(4 * 1024 * 1024); + for (;;) { + const read = readSync(fd, buffer, 0, buffer.length, null); + if (read === 0) break; + hash.update(buffer.subarray(0, read)); + } + } finally { + closeSync(fd); + } +} + function cacheKey( options: NativeCodegenOptions, target: NativeTargetSpec, helper: HelperIdentity, ): string { const hash = createHash("sha256").update("scriptc-native-codegen-v1\0"); - for (const part of typeof options.llvm === "string" ? [options.llvm] : options.llvm) - hash.update(part); + if (isLlvmFile(options.llvm)) updateHashFromFile(hash, options.llvm.file); + else + for (const part of typeof options.llvm === "string" ? [options.llvm] : options.llvm) + hash.update(part); return hash .update("\0") .update(JSON.stringify(target)) @@ -391,7 +430,8 @@ export async function emitNativeArtifact( // concurrent build of another entry that has the same output stem. const input = privateSiblingPath(options.outputPath, "native-input"); try { - await writeLlvmSource(input, options.llvm, 0o600); + if (isLlvmFile(options.llvm)) await copyFile(options.llvm.file, input); + else await writeLlvmSource(input, options.llvm, 0o600); await invoke(helper.binaryPath, [ "emit", "--input", diff --git a/packages/compiler/src/executable/executable-cache.test.ts b/packages/compiler/src/executable/executable-cache.test.ts index ae3c3b7eb..5a633bb45 100644 --- a/packages/compiler/src/executable/executable-cache.test.ts +++ b/packages/compiler/src/executable/executable-cache.test.ts @@ -13,9 +13,11 @@ import { } from "../backend/native-toolchain.js"; import { installDarwinDebugSymbols, readDarwinDebugSymbols } from "../backend/debug-symbols.js"; import { + hasEarlyExecutableNativeBuild, publishEarlyExecutableCache, readEarlyExecutableCache, readRoutedExecutableCache, + recordEarlyExecutableNativeBuild, type EarlyExecutableCacheOptions, type EarlyExecutableNativeFeatures, } from "./executable-cache.js"; @@ -594,3 +596,38 @@ test("the route index still isolates eager and deferred native environments", as )?.executableRestored, ).toBe(true); }); + +test("a recorded native build is found by its invocation", async () => { + const f = await fixture(); + expect(await hasEarlyExecutableNativeBuild(null, f.options)).toBe(false); + expect(await hasEarlyExecutableNativeBuild(f.root, f.options)).toBe(false); + await recordEarlyExecutableNativeBuild(null, f.options); + await recordEarlyExecutableNativeBuild(f.root, f.options); + expect(await hasEarlyExecutableNativeBuild(f.root, f.options)).toBe(true); + // The record does not depend on the compiler implementation or the native environment. + const rebuilt: EarlyExecutableCacheOptions = { + ...f.options, + implementation: "b".repeat(64), + nativeEnvironment: "another driver", + }; + expect(await hasEarlyExecutableNativeBuild(f.root, rebuilt)).toBe(true); + expect( + await hasEarlyExecutableNativeBuild(f.root, { + ...f.options, + outPath: join(f.options.outDir, "other"), + }), + ).toBe(false); + expect(await hasEarlyExecutableNativeBuild(f.root, { ...f.options, sanitize: true })).toBe(false); + // Publishing an executable does not record a native build. + const other = { ...f.options, outPath: join(f.options.outDir, "published") }; + const tracker = new FrontendInputTracker(); + tracker.run(() => trackedReadFile(f.source)); + await publishEarlyExecutableCache(f.root, other, { + llvmPath: f.llvmPath, + irPath: f.irPath, + native, + executableRestored: false, + frontend: tracker.snapshot(), + }); + expect(await hasEarlyExecutableNativeBuild(f.root, other)).toBe(false); +}); diff --git a/packages/compiler/src/executable/executable-cache.ts b/packages/compiler/src/executable/executable-cache.ts index 8f7014dad..1ed23a038 100644 --- a/packages/compiler/src/executable/executable-cache.ts +++ b/packages/compiler/src/executable/executable-cache.ts @@ -327,6 +327,39 @@ function routePath( return join(root, "early-exe-route", routeKey(options)); } +function nativeBuildMarkerPath( + root: string, + options: Omit, +): string { + return join(root, "early-exe-native-build", routeKey(options)); +} + +/** Records that a build of the invocation `options` emitted its native program object, so the + * native helper's partition cache holds its partitions. Unlike the route, the record does not + * depend on the build publishing its executable. */ +export async function recordEarlyExecutableNativeBuild( + root: string | null, + options: Omit, +): Promise { + if (root === null) return; + const marker = nativeBuildMarkerPath(root, options); + await mkdir(dirname(marker), { recursive: true, mode: 0o700 }); + await writeFile(marker, "", { mode: 0o600 }); +} + +/** Whether an earlier build of the invocation `options` recorded its native program object + * (`recordEarlyExecutableNativeBuild`). False when there is no cache. */ +export async function hasEarlyExecutableNativeBuild( + root: string | null, + options: Omit, +): Promise { + if (root === null) return false; + return stat(nativeBuildMarkerPath(root, options)).then( + (info) => info.isFile(), + () => false, + ); +} + function routeIntegrity(stamp: Omit): string { return createHash("sha256") .update("early-executable-route-stamp-v2\0") diff --git a/packages/compiler/src/frontend-process-args.test.ts b/packages/compiler/src/frontend-process-args.test.ts new file mode 100644 index 000000000..a6e9b52e5 --- /dev/null +++ b/packages/compiler/src/frontend-process-args.test.ts @@ -0,0 +1,56 @@ +import { expect, test } from "vitest"; +import { frontendProcessExecArgv } from "./frontend-process-args.js"; + +test("the frontend process keeps heap limits and loaders", () => { + expect( + frontendProcessExecArgv([ + "--max-old-space-size=8192", + "--stack-size=4000", + "--import", + "tsx", + "-r", + "./setup.cjs", + "--conditions=development", + "--enable-source-maps", + ]), + ).toEqual([ + "--max-old-space-size=8192", + "--stack-size=4000", + "--import", + "tsx", + "-r", + "./setup.cjs", + "--conditions=development", + "--enable-source-maps", + ]); +}); + +test("the frontend process drops evaluated code, inspectors, watch and test runner options", () => { + expect( + frontendProcessExecArgv([ + "--max-old-space-size=8192", + "--input-type=module", + "--eval", + "await compile(process.argv[1], {})", + ]), + ).toEqual(["--max-old-space-size=8192"]); + expect(frontendProcessExecArgv(["-e", "code", "-p", "code", "-pe", "code"])).toEqual([]); + expect( + frontendProcessExecArgv(["--eval=code", "--print=code", "--input-type", "module"]), + ).toEqual([]); + expect( + frontendProcessExecArgv([ + "--inspect", + "--inspect-brk=9229", + "--inspect-port", + "9230", + "--watch", + "--watch-path=src", + "--test", + "--test-name-pattern", + "build", + "--test-reporter=spec", + "--expose-gc", + ]), + ).toEqual(["--expose-gc"]); +}); diff --git a/packages/compiler/src/frontend-process-args.ts b/packages/compiler/src/frontend-process-args.ts new file mode 100644 index 000000000..dde9d002f --- /dev/null +++ b/packages/compiler/src/frontend-process-args.ts @@ -0,0 +1,54 @@ +/** Options of the compiler's own process that the frontend process (frontend-process.ts) must not + * receive, with the options among them that take their value as the next argument: the code of + * --eval and --print would run in place of the frontend process's entry, and the inspector, watch + * and test runner options belong to the process that was started with them. */ +const OPTIONS_WITH_VALUES = new Set([ + "-e", + "--eval", + "-p", + "--print", + "--input-type", + "--inspect-port", + "--debug-port", + "--watch-path", + "--test-concurrency", + "--test-coverage-branches", + "--test-coverage-exclude", + "--test-coverage-functions", + "--test-coverage-include", + "--test-coverage-lines", + "--test-global-setup", + "--test-isolation", + "--test-name-pattern", + "--test-reporter", + "--test-reporter-destination", + "--test-shard", + "--test-skip-pattern", + "--test-timeout", +]); +const OPTIONS = new Set(["-i", "--interactive", "-c", "--check", "--test"]); +const OPTION_PREFIXES = ["--inspect", "--debug-port", "--watch", "--test-", "--input-type="]; + +/** The Node.js options of the frontend process: `execArgv` without the options that run code + * (--eval, --print), open an inspector, or watch or test, so that its own entry runs with the + * compiler process's remaining options, such as heap limits and loaders. */ +export function frontendProcessExecArgv(execArgv: readonly string[]): string[] { + const kept: string[] = []; + for (let index = 0; index < execArgv.length; index++) { + const option = execArgv[index]!; + // A short option group such as -pe takes the code that follows it. + if (OPTIONS_WITH_VALUES.has(option) || /^-[a-z]*[ep][a-z]*$/i.test(option)) { + index++; + continue; + } + if ( + OPTIONS.has(option) || + option.startsWith("--eval=") || + option.startsWith("--print=") || + OPTION_PREFIXES.some((prefix) => option.startsWith(prefix)) + ) + continue; + kept.push(option); + } + return kept; +} diff --git a/packages/compiler/src/frontend-process.ts b/packages/compiler/src/frontend-process.ts new file mode 100644 index 000000000..a3d59516f --- /dev/null +++ b/packages/compiler/src/frontend-process.ts @@ -0,0 +1,18 @@ +// The frontend process of an executable build (FrontendProcess and runFrontendProcess in index.ts): +// loads the compiler, then reads the paths of its request and result from stdin. Without them it +// exits: the build found its executable without it. +import { enableCompileCache } from "node:module"; + +try { + enableCompileCache(); +} catch { + // Bytecode caching only speeds up loading. +} +const { runFrontendProcess } = await import("./index.js"); +let input = ""; +for await (const chunk of process.stdin) input += String(chunk); +const [requestPath, resultPath] = input.split("\n"); +if (requestPath !== undefined && requestPath !== "" && resultPath !== undefined) + await runFrontendProcess(requestPath, resultPath); +// The frontend's own helpers (the checker process, timers) must not keep this process alive. +process.exit(0); diff --git a/packages/compiler/src/index.ts b/packages/compiler/src/index.ts index 7dfcb2179..6e9717135 100644 --- a/packages/compiler/src/index.ts +++ b/packages/compiler/src/index.ts @@ -1,5 +1,5 @@ import { optimizationField, type NativeOptimization } from "./backend/optimization.js"; -import { compilationTiming, type CompilationTiming } from "./timing.js"; +import { compilationTiming, type CompilationTiming, type TimingClock } from "./timing.js"; import { prepareExecutableModule } from "./executable/prepare.js"; import { prepareLibrary, @@ -45,8 +45,12 @@ export type { } from "./compile-types.js"; import { compilePackedLibrary } from "./backend/library-pack.js"; import { InternalCompilerError } from "./errors.js"; +import { frontendProcessExecArgv } from "./frontend-process-args.js"; +import { type ChildProcess, spawn } from "node:child_process"; +import { existsSync } from "node:fs"; import { mkdir, readFile, rename, rm, writeFile } from "node:fs/promises"; import { basename, dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; import { clearCcCaches, configuredTargetPlatform, @@ -66,6 +70,8 @@ import { emitLlvmModuleSource, LlvmUnsupportedError } from "./backend/llvm/emitt import { inlineRcOption } from "./backend/llvm/shapes.js"; import { emitNativeArtifact, + llvmInputBytes, + type LlvmInput, NativeCodegenError, writeLlvmSource, } from "./backend/native-codegen.js"; @@ -115,7 +121,11 @@ import { clearResolveCaches } from "./frontend/resolve.js"; import { loadFfiProfile, type FfiProfile } from "./ffi/ffi-manifest.js"; import { executableLinkFeatures } from "./backend/executable-features.js"; import { youngNurseryRequested } from "./backend/young-nursery.js"; -import { FrontendInputTracker, trackedReadFile } from "./frontend/input-tracker.js"; +import { + type FrontendInputSnapshot, + FrontendInputTracker, + trackedReadFile, +} from "./frontend/input-tracker.js"; import { libraryFrontendImplementationFingerprint, publishEarlyLibraryCache, @@ -127,9 +137,11 @@ import { type SemanticLibraryCacheHit, } from "./library/library-cache.js"; import { + hasEarlyExecutableNativeBuild, publishEarlyExecutableCache, publishEarlyExecutableRoute, readEarlyExecutableCache, + recordEarlyExecutableNativeBuild, type EarlyExecutableCacheOptions, type EarlyExecutableNativeFeatures, } from "./executable/executable-cache.js"; @@ -410,10 +422,16 @@ export async function compile( clearCompileSessionCaches(); const frontendInputs = new FrontendInputTracker(); const warnings: CompileWarningSink = { warnings: [], sourceTexts: null }; - const result = await frontendInputs.run(() => - compileTracked(entryPath, opts, frontendInputs, warnings), - ); - return withCompileWarnings(result, warnings); + // compileTracked starts it for a build with no earlier build of the same invocation. + const frontendProcess = preparesInChildProcess(opts) ? new FrontendProcess() : undefined; + try { + const result = await frontendInputs.run(() => + compileTracked(entryPath, opts, frontendInputs, warnings, frontendProcess), + ); + return withCompileWarnings(result, warnings); + } finally { + frontendProcess?.release(); + } } /** Divergence warnings collected while a compile lowers (none when a cache @@ -601,7 +619,7 @@ interface NativeProgramObject { async function emitNativeProgramObject( entryPath: string, opts: CompileRequestOptions, - llvm: string | readonly string[], + llvm: LlvmInput, features: EarlyExecutableNativeFeatures, ): Promise { const stem = basename(entryPath).replace(/\.(ts|mts|cts|js|mjs|cjs)$/, ""); @@ -612,10 +630,7 @@ async function emitNativeProgramObject( const linkPath = `${privateSiblingPath(artifactPath, "native-program-object")}.o`; // The validated program-object artifact is a single object by contract. const target = nativeCodegenTarget(); - const llvmBytes = (typeof llvm === "string" ? [llvm] : llvm).reduce( - (bytes, part) => bytes + Buffer.byteLength(part), - 0, - ); + const llvmBytes = llvmInputBytes(llvm); const partitions = opts.nativeProgramObject === true || target === null ? 1 : nativeProgramPartitions(llvmBytes); // Optimized programs import small runtime functions from the bitcode of @@ -685,7 +700,7 @@ function runtimePackDiagnostic(error: RuntimePackError, entryPath: string): ScrD } interface PreparedExecutable { - llvmSource: string | readonly string[]; + llvmSource: LlvmInput; llvmPath: string; irPath: string | undefined; nativeFeatures: EarlyExecutableNativeFeatures; @@ -840,13 +855,229 @@ async function prepareExecutableInput( return { llvmSource, llvmPath, irPath, nativeFeatures, programSplit, sourceTexts }; } +/** + * The child process that prepares an executable build's input: frontend, lowering and LLVM emission + * (`runFrontendProcess`). Node's heap keeps the pages a build's frontend used for the life of the + * process, so running them in a process of their own leaves the native helper the memory they took. + */ +const frontendProcessEntry = fileURLToPath(new URL("./frontend-process.js", import.meta.url)); + +/** + * Whether `prepareExecutableInputInChild` may prepare this build: optimized executables, when the + * compiler runs from its built files. Development builds split the LLVM text in this process, and + * SCRIPTC_FRONTEND_PROCESS=0 keeps the frontend in-process. + * + * compileTracked starts the process only when no earlier build of the same invocation emitted its + * native program object (`hasEarlyExecutableNativeBuild`). A rebuild finds most native partitions + * in the helper's cache, so its memory peaks in the frontend, and the parent process would add to + * that peak. + */ +function preparesInChildProcess(opts: CompileRequestOptions): boolean { + return ( + process.env["SCRIPTC_FRONTEND_PROCESS"] !== "0" && + (opts.outputKind ?? "exe") === "exe" && + (opts.optimization ?? "release") !== "dev" && + existsSync(frontendProcessEntry) + ); +} + +/** + * The frontend process (frontend-process.ts) of one compile. `start` launches it; it then waits on + * its stdin for the paths of a request and its result (`run`), and exits once its stdin closes + * without them (`release`). + */ +class FrontendProcess { + private child: ChildProcess | undefined; + private exited: Promise | undefined; + private sent = false; + + /** Whether `start` launched the process. */ + get started(): boolean { + return this.child !== undefined; + } + + start(): void { + if (this.child !== undefined) return; + const execArgv = frontendProcessExecArgv(process.execArgv); + const child = spawn(process.execPath, [...execArgv, frontendProcessEntry], { + stdio: ["pipe", "inherit", "inherit"], + }); + this.exited = new Promise((resolveExit) => { + child.on("error", () => resolveExit(null)); + child.on("exit", (code) => resolveExit(code)); + }); + // A failed start surfaces as a failed run; the build then prepares its input in-process. + child.stdin!.on("error", () => undefined); + this.child = child; + } + + /** Sends the request at `requestPath`; resolves with the process's exit code. */ + run(requestPath: string, resultPath: string): Promise { + if (this.child === undefined || this.exited === undefined || this.sent) + return Promise.resolve(null); + this.sent = true; + this.child.stdin!.end(`${requestPath}\n${resultPath}\n`); + return this.exited; + } + + /** Lets the process exit if no request was sent. */ + release(): void { + if (this.child === undefined || this.sent) return; + this.sent = true; + this.child.stdin!.end(); + // The build is done with it: this process need not wait for it to exit. + this.child.unref(); + (this.child.stdin as { unref?: () => void }).unref?.(); + } +} + +interface FrontendProcessRequest { + entryPath: string; + opts: CompileRequestOptions; + buildPlatform: string; + /** The compile's SCRIPTC_TIMING clock, which the frontend process's phases continue. */ + timing: TimingClock; +} + +/** What the frontend process sends back. A thrown error comes back as `failed`. */ +type FrontendProcessResult = + | { + kind: "prepared"; + llvmPath: string; + llvmBytes: number; + irPath: string | undefined; + nativeFeatures: EarlyExecutableNativeFeatures; + sourceTexts: [string, string][]; + warnings: ScrDiagnostic[]; + frontend: FrontendInputSnapshot; + } + | { + kind: "diagnostics"; + diagnostics: ScrDiagnostic[]; + sourceTexts: [string, string][]; + warnings: ScrDiagnostic[]; + frontend: FrontendInputSnapshot; + } + | { kind: "failed" }; + +/** + * `prepareExecutableInput` in the frontend process, with the LLVM text passed on as its file. The + * process's frontend inputs join `frontendInputs` and its warnings `warningSink`. If it fails, the + * input is prepared here, so the build reports the same errors. + */ +async function prepareExecutableInputInChild( + frontendProcess: FrontendProcess, + entryPath: string, + opts: CompileRequestOptions, + ffi: FfiProfile | null, + buildPlatform: string, + timing: CompilationTiming, + timingClock: TimingClock, + frontendInputs: FrontendInputTracker, + warningSink?: CompileWarningSink, +): Promise { + await mkdir(opts.outDir, { recursive: true }); + const requestPath = privateSiblingPath(join(opts.outDir, "frontend-request.json"), "frontend"); + const resultPath = privateSiblingPath(join(opts.outDir, "frontend-result.json"), "frontend"); + let result: FrontendProcessResult = { kind: "failed" }; + try { + const request: FrontendProcessRequest = { entryPath, opts, buildPlatform, timing: timingClock }; + await writeFile(requestPath, JSON.stringify(request), { mode: 0o600 }); + const exitCode = await frontendProcess.run(requestPath, resultPath); + if (exitCode === 0) + result = JSON.parse(await readFile(resultPath, "utf8")) as FrontendProcessResult; + } catch { + result = { kind: "failed" }; + } finally { + await rm(requestPath, { force: true }); + await rm(resultPath, { force: true }); + } + if (result.kind === "failed") + return prepareExecutableInput(entryPath, opts, ffi, buildPlatform, timing, warningSink); + timing("frontend-process"); + for (const probe of result.frontend.probes) frontendInputs.record(probe); + if (!result.frontend.stable) frontendInputs.markUnstable(); + const sourceTexts = new Map(result.sourceTexts); + if (warningSink !== undefined && result.warnings.length > 0) { + warningSink.warnings = result.warnings; + warningSink.sourceTexts = sourceTexts; + } + if (result.kind === "diagnostics") + return { ok: false, diagnostics: result.diagnostics, sourceTexts }; + return { + llvmSource: { file: result.llvmPath, bytes: result.llvmBytes }, + llvmPath: result.llvmPath, + irPath: result.irPath, + nativeFeatures: result.nativeFeatures, + programSplit: null, + sourceTexts, + }; +} + +/** + * The frontend process (frontend-process.ts): prepares the executable input of the request at + * `requestPath` as `compileTracked` would in-process, and writes what it prepared, without the LLVM + * text, to `resultPath`. + */ +export async function runFrontendProcess(requestPath: string, resultPath: string): Promise { + const request = JSON.parse(await readFile(requestPath, "utf8")) as FrontendProcessRequest; + clearCompileSessionCaches(); + const frontendInputs = new FrontendInputTracker(); + const warnings: CompileWarningSink = { warnings: [], sourceTexts: null }; + let result: FrontendProcessResult = { kind: "failed" }; + try { + let ffi: FfiProfile | null = null; + if (request.opts.ffiProfilePath !== undefined) { + const loaded = loadFfiProfile(resolve(request.opts.ffiProfilePath)); + if (!loaded.ok) throw new InternalCompilerError("the FFI profile could not be reloaded"); + ffi = loaded.profile; + } + const prepared = await frontendInputs.run(() => + prepareExecutableInput( + request.entryPath, + request.opts, + ffi, + request.buildPlatform, + compilationTiming(request.timing), + warnings, + ), + ); + const frontend = frontendInputs.snapshot(); + if (!("ok" in prepared)) { + result = { + kind: "prepared", + llvmPath: prepared.llvmPath, + llvmBytes: llvmInputBytes(prepared.llvmSource), + irPath: prepared.irPath, + nativeFeatures: prepared.nativeFeatures, + sourceTexts: [...prepared.sourceTexts], + warnings: warnings.warnings, + frontend, + }; + } else if (!prepared.ok) { + result = { + kind: "diagnostics", + diagnostics: prepared.diagnostics, + sourceTexts: [...prepared.sourceTexts], + warnings: warnings.warnings, + frontend, + }; + } + } catch { + result = { kind: "failed" }; + } + await writeFile(resultPath, JSON.stringify(result), { mode: 0o600 }); +} + async function compileTracked( entryPath: string, opts: CompileRequestOptions, frontendInputs: FrontendInputTracker, warningSink?: CompileWarningSink, + frontendProcess?: FrontendProcess, ): Promise { const timing = compilationTiming(); + const timingStart = Date.now(); entryPath = resolve(entryPath); const outputKind = opts.outputKind ?? "exe"; if ( @@ -1055,16 +1286,10 @@ async function compileTracked( if (outputKind === "exe") { const helperObjectRoute = opts.nativeProgramObject === true || usesPrecompiledRuntimePack(opts, "llvm"); - // Package content and native driver discovery are independent inputs to - // the same cache key. Start both before waiting so edits do not pay their - // filesystem and subprocess latency serially. - const [implementation, nativeEnvironment] = await Promise.all([ - compilerImplementationIdentity(), - helperObjectRoute - ? executableLinkerEnvironmentFingerprint(process.env, nativeCodegenTarget()?.defaultLinker) - : executableNativeEnvironmentFingerprint(), - ]); - earlyCacheOptions = { + const routeOptions: Omit< + EarlyExecutableCacheOptions, + "implementation" | "implementationDependencies" | "nativeEnvironment" + > = { entryPath, outDir: opts.outDir, outPath: opts.outPath, @@ -1094,8 +1319,27 @@ async function compileTracked( ? resolvePlatformLinker(process.env, nativeCodegenTarget()?.defaultLinker) : (process.env["SCRIPTC_CC"] ?? "clang"), ], - nativeEnvironment, nodeVersion: process.version, + }; + // The frontend process prepares the first build of an invocation (preparesInChildProcess). + // Started here, it loads the compiler while this process computes the rest of the cache key. + if ( + frontendProcess !== undefined && + !(await hasEarlyExecutableNativeBuild(cacheRoot, routeOptions)) + ) + frontendProcess.start(); + // Package content and native driver discovery are independent inputs to + // the same cache key. Start both before waiting so edits do not pay their + // filesystem and subprocess latency serially. + const [implementation, nativeEnvironment] = await Promise.all([ + compilerImplementationIdentity(), + helperObjectRoute + ? executableLinkerEnvironmentFingerprint(process.env, nativeCodegenTarget()?.defaultLinker) + : executableNativeEnvironmentFingerprint(), + ]); + earlyCacheOptions = { + ...routeOptions, + nativeEnvironment, implementation: implementation.digest, implementationDependencies: implementation.dependencies, }; @@ -1227,14 +1471,20 @@ async function compileTracked( }; } timing("executable-cache-miss"); - const prepared = await prepareExecutableInput( - entryPath, - opts, - ffi, - buildPlatform, - timing, - warningSink, - ); + const prepared = + frontendProcess?.started === true + ? await prepareExecutableInputInChild( + frontendProcess, + entryPath, + opts, + ffi, + buildPlatform, + timing, + { start: timingStart, previous: Date.now() }, + frontendInputs, + warningSink, + ) + : await prepareExecutableInput(entryPath, opts, ffi, buildPlatform, timing, warningSink); if ("ok" in prepared) return prepared; const { llvmSource, llvmPath, irPath, nativeFeatures, programSplit, sourceTexts } = prepared; const backend = "llvm" as const; @@ -1262,6 +1512,9 @@ async function compileTracked( nativeFeatures, ); timing("native-object"); + await recordEarlyExecutableNativeBuild(cacheRoot, executableCacheOptions).catch( + () => undefined, + ); } catch (err) { if (!(err instanceof NativeCodegenError)) throw err; return { diff --git a/packages/compiler/src/timing.ts b/packages/compiler/src/timing.ts index 79c07cb7e..6d9f6844d 100644 --- a/packages/compiler/src/timing.ts +++ b/packages/compiler/src/timing.ts @@ -1,11 +1,20 @@ export type CompilationTiming = (phase: string, detail?: Record) => void; +/** The start of a timing and its latest phase, as `Date.now()` values. */ +export interface TimingClock { + start: number; + previous: number; +} + /** Opt-in phase durations shared by the Node seed and native compiler. - * The disabled path does not read the clock or write compiler output. */ -export function compilationTiming(): CompilationTiming { + * The disabled path does not read the clock or write compiler output. + * With `clock`, the phases continue a timing that started in another + * process, so a frontend process reports them as its parent compile would. */ +export function compilationTiming(clock?: TimingClock): CompilationTiming { if (process.env["SCRIPTC_TIMING"] !== "1") return () => {}; - const start = performance.now(); - let previous = start; + const opened = performance.now(); + const start = clock === undefined ? opened : opened - (Date.now() - clock.start); + let previous = clock === undefined ? start : opened - (Date.now() - clock.previous); return (phase, detail = {}) => { const now = performance.now(); process.stderr.write(