diff --git a/packages/cli/test/frontend-process.test.ts b/packages/cli/test/frontend-process.test.ts new file mode 100644 index 000000000..8f7f85e4d --- /dev/null +++ b/packages/cli/test/frontend-process.test.ts @@ -0,0 +1,172 @@ +import { execFile } from "node:child_process"; +import { mkdir, mkdtemp, readdir, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { pathToFileURL } from "node:url"; +import { promisify } from "node:util"; +import { expect, test } from "vitest"; +import { precompiledRuntimePackTarget } from "../../compiler/src/startup-cache.js"; + +// The first optimized executable build of an invocation prepares its input in a frontend process +// (prepareExecutableInputInChild in the compiler's index.ts) when the compiler runs from its built +// files, as the CLI's bootstrap does. Later builds of the invocation, and builds with +// SCRIPTC_FRONTEND_PROCESS=0, prepare it in the compiler's own process. +const execFileAsync = promisify(execFile); +const repoRoot = join(import.meta.dirname, "../../.."); +const bootstrap = join(repoRoot, "packages/cli/dist/bootstrap.js"); +const compilerApi = pathToFileURL(join(repoRoot, "packages/compiler/dist/index.js")).href; +const helperTarget = precompiledRuntimePackTarget(); + +interface Build { + code: number; + stdout: string; + stderr: string; +} + +async function build( + entry: string, + outPath: string, + cacheRoot: string, + inProcess: boolean, +): Promise { + const env: NodeJS.ProcessEnv = { + ...process.env, + SCRIPTC_CACHE_DIR: cacheRoot, + SCRIPTC_TIMING: "1", + }; + delete env["SCRIPTC_NO_CACHE"]; + if (inProcess) env["SCRIPTC_FRONTEND_PROCESS"] = "0"; + else delete env["SCRIPTC_FRONTEND_PROCESS"]; + try { + const { stdout, stderr } = await execFileAsync( + process.execPath, + [bootstrap, "build", entry, "-o", outPath], + { env, maxBuffer: 8 * 1024 * 1024 }, + ); + return { code: 0, stdout, stderr }; + } catch (error) { + const failed = error as { code?: number; stdout?: string; stderr?: string }; + return { code: failed.code ?? 1, stdout: failed.stdout ?? "", stderr: failed.stderr ?? "" }; + } +} + +/** The build's output without its timing lines, whose times differ between runs. */ +const withoutTiming = (text: string): string => + text + .split("\n") + .filter((line) => !line.startsWith("scriptc timing") && !line.startsWith("scriptc lowering")) + .join("\n"); + +test.skipIf(helperTarget === null)( + "the frontend process builds the executable, diagnostics and cache entries the in-process build does", + async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-frontend-process-")); + try { + const src = join(dir, "src"); + await mkdir(src); + await writeFile( + join(src, "shapes.ts"), + "export const area = (w: number, h: number): number => w * h;\n", + ); + await writeFile( + join(src, "main.ts"), + 'import { area } from "./shapes.js";\nconsole.log(area(3, 4), [1, 2, 3].map((x) => x * 2).join(","));\n', + ); + const entry = join(src, "main.ts"); + const out = (name: string): string => join(dir, name, "program"); + await mkdir(join(dir, "child")); + await mkdir(join(dir, "inline")); + + const child = await build(entry, out("child"), join(dir, "cache-child"), false); + const inline = await build(entry, out("inline"), join(dir, "cache-inline"), true); + expect(child.code).toBe(0); + expect(inline.code).toBe(0); + expect(child.stderr).toContain('"phase":"frontend-process"'); + expect(inline.stderr).not.toContain('"phase":"frontend-process"'); + expect(await readFile(out("child"))).toEqual(await readFile(out("inline"))); + expect(await readFile(join(dir, "child", "main.ll"), "utf8")).toBe( + await readFile(join(dir, "inline", "main.ll"), "utf8"), + ); + expect((await execFileAsync(out("child"))).stdout).toBe("12 2,4,6\n"); + // The request and result files are removed. + expect( + (await readdir(join(dir, "child"))).filter((name) => name.includes("frontend")), + ).toEqual([]); + + // A repeat finds the executable the frontend process's build published. An edit rebuilds in + // the compiler's own process. + const repeat = await build(entry, out("child"), join(dir, "cache-child"), false); + expect(repeat.code).toBe(0); + expect(repeat.stderr).not.toContain("scriptc lowering"); + expect(repeat.stderr).not.toContain('"phase":"frontend-process"'); + await writeFile( + join(src, "shapes.ts"), + "export const area = (w: number, h: number): number => w * h + 1;\n", + ); + const edited = await build(entry, out("child"), join(dir, "cache-child"), false); + expect(edited.code).toBe(0); + expect(edited.stderr).toContain("scriptc lowering"); + expect(edited.stderr).not.toContain('"phase":"frontend-process"'); + expect((await execFileAsync(out("child"))).stdout).toBe("13 2,4,6\n"); + + // Without the executable cache entries (as for a program whose frontend inputs cannot be + // cached), the rebuild still runs in the compiler's own process: the first build recorded + // its native program object. + await rm(join(dir, "cache-child", "early-exe"), { recursive: true, force: true }); + await rm(join(dir, "cache-child", "early-exe-route"), { recursive: true, force: true }); + const uncached = await build(entry, out("child"), join(dir, "cache-child"), false); + expect(uncached.code).toBe(0); + expect(uncached.stderr).toContain("scriptc lowering"); + expect(uncached.stderr).not.toContain('"phase":"frontend-process"'); + expect((await execFileAsync(out("child"))).stdout).toBe("13 2,4,6\n"); + + // A program the frontend rejects reports the same diagnostics either way. The first build of + // the invocation in an empty cache runs the frontend process. + await writeFile(join(src, "main.ts"), 'const n: number = "not a number";\nconsole.log(n);\n'); + const childError = await build(entry, out("child"), join(dir, "cache-child-error"), false); + const inlineError = await build(entry, out("inline"), join(dir, "cache-inline"), true); + expect(childError.stderr).toContain('"phase":"frontend-process"'); + expect(childError.code).not.toBe(0); + expect(childError.code).toBe(inlineError.code); + expect(withoutTiming(childError.stderr)).toBe(withoutTiming(inlineError.stderr)); + expect(withoutTiming(childError.stderr)).toContain("not a number"); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }, + 240_000, +); + +test.skipIf(helperTarget === null)( + "a compile called from evaluated code runs its frontend process once", + async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-frontend-process-eval-")); + try { + const entry = join(dir, "main.ts"); + await writeFile(entry, 'console.log("from eval");\n'); + const { stdout, stderr } = await execFileAsync( + process.execPath, + [ + "--input-type=module", + "--eval", + `import { compile } from ${JSON.stringify(compilerApi)}; + const result = await compile(process.argv[1], { outDir: process.argv[2], outPath: process.argv[3] }); + console.log("compiled", result.ok);`, + entry, + dir, + join(dir, "program"), + ], + { + env: { ...process.env, SCRIPTC_CACHE_DIR: join(dir, "cache"), SCRIPTC_TIMING: "1" }, + maxBuffer: 8 * 1024 * 1024, + }, + ); + expect(stdout).toBe("compiled true\n"); + expect(stderr).toContain('"phase":"frontend-process"'); + expect((await execFileAsync(join(dir, "program"))).stdout).toBe("from eval\n"); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }, + 240_000, +); diff --git a/packages/compiler/src/backend/native-codegen.ts b/packages/compiler/src/backend/native-codegen.ts index 6aab5880a..ff411922a 100644 --- a/packages/compiler/src/backend/native-codegen.ts +++ b/packages/compiler/src/backend/native-codegen.ts @@ -1,8 +1,8 @@ import { execFile } from "node:child_process"; -import { createHash } from "node:crypto"; -import { constants } from "node:fs"; +import { createHash, type Hash } from "node:crypto"; +import { closeSync, constants, openSync, readSync } from "node:fs"; import { createRequire } from "node:module"; -import { access, chmod, mkdir, readFile, rename, rm, stat } from "node:fs/promises"; +import { access, chmod, copyFile, mkdir, readFile, rename, rm, stat } from "node:fs/promises"; import { dirname, join } from "node:path"; import { promisify } from "node:util"; import { @@ -68,9 +68,31 @@ export interface NativeCodegenArtifact { const resolvedHelperCache = new Map>(); +/** LLVM text already written to `file`, of `bytes` bytes, which a build passes on without reading it. */ +export interface LlvmFile { + readonly file: string; + readonly bytes: number; +} + +/** LLVM text as a string, as parts to concatenate, or as a file. */ +export type LlvmInput = string | readonly string[] | LlvmFile; + +export function isLlvmFile(llvm: LlvmInput): llvm is LlvmFile { + return typeof llvm === "object" && !Array.isArray(llvm); +} + +/** The size of `llvm` in bytes. */ +export function llvmInputBytes(llvm: LlvmInput): number { + if (isLlvmFile(llvm)) return llvm.bytes; + return (typeof llvm === "string" ? [llvm] : llvm).reduce( + (bytes, part) => bytes + Buffer.byteLength(part), + 0, + ); +} + export interface NativeCodegenOptions { outputPath: string; - llvm: string | readonly string[]; + llvm: LlvmInput; outputKind: NativeCodegenOutputKind; sourcePath: string; optimization?: "0" | "1" | "2" | "3" | "s" | "z"; @@ -271,14 +293,31 @@ async function resolveHelper( return pending; } +/** Adds the bytes of `file` to `hash`, as `hash.update` of the text written there would. */ +function updateHashFromFile(hash: Hash, file: string): void { + const fd = openSync(file, "r"); + try { + const buffer = Buffer.allocUnsafe(4 * 1024 * 1024); + for (;;) { + const read = readSync(fd, buffer, 0, buffer.length, null); + if (read === 0) break; + hash.update(buffer.subarray(0, read)); + } + } finally { + closeSync(fd); + } +} + function cacheKey( options: NativeCodegenOptions, target: NativeTargetSpec, helper: HelperIdentity, ): string { const hash = createHash("sha256").update("scriptc-native-codegen-v1\0"); - for (const part of typeof options.llvm === "string" ? [options.llvm] : options.llvm) - hash.update(part); + if (isLlvmFile(options.llvm)) updateHashFromFile(hash, options.llvm.file); + else + for (const part of typeof options.llvm === "string" ? [options.llvm] : options.llvm) + hash.update(part); return hash .update("\0") .update(JSON.stringify(target)) @@ -391,7 +430,8 @@ export async function emitNativeArtifact( // concurrent build of another entry that has the same output stem. const input = privateSiblingPath(options.outputPath, "native-input"); try { - await writeLlvmSource(input, options.llvm, 0o600); + if (isLlvmFile(options.llvm)) await copyFile(options.llvm.file, input); + else await writeLlvmSource(input, options.llvm, 0o600); await invoke(helper.binaryPath, [ "emit", "--input", diff --git a/packages/compiler/src/executable/executable-cache.test.ts b/packages/compiler/src/executable/executable-cache.test.ts index ae3c3b7eb..5a633bb45 100644 --- a/packages/compiler/src/executable/executable-cache.test.ts +++ b/packages/compiler/src/executable/executable-cache.test.ts @@ -13,9 +13,11 @@ import { } from "../backend/native-toolchain.js"; import { installDarwinDebugSymbols, readDarwinDebugSymbols } from "../backend/debug-symbols.js"; import { + hasEarlyExecutableNativeBuild, publishEarlyExecutableCache, readEarlyExecutableCache, readRoutedExecutableCache, + recordEarlyExecutableNativeBuild, type EarlyExecutableCacheOptions, type EarlyExecutableNativeFeatures, } from "./executable-cache.js"; @@ -594,3 +596,38 @@ test("the route index still isolates eager and deferred native environments", as )?.executableRestored, ).toBe(true); }); + +test("a recorded native build is found by its invocation", async () => { + const f = await fixture(); + expect(await hasEarlyExecutableNativeBuild(null, f.options)).toBe(false); + expect(await hasEarlyExecutableNativeBuild(f.root, f.options)).toBe(false); + await recordEarlyExecutableNativeBuild(null, f.options); + await recordEarlyExecutableNativeBuild(f.root, f.options); + expect(await hasEarlyExecutableNativeBuild(f.root, f.options)).toBe(true); + // The record does not depend on the compiler implementation or the native environment. + const rebuilt: EarlyExecutableCacheOptions = { + ...f.options, + implementation: "b".repeat(64), + nativeEnvironment: "another driver", + }; + expect(await hasEarlyExecutableNativeBuild(f.root, rebuilt)).toBe(true); + expect( + await hasEarlyExecutableNativeBuild(f.root, { + ...f.options, + outPath: join(f.options.outDir, "other"), + }), + ).toBe(false); + expect(await hasEarlyExecutableNativeBuild(f.root, { ...f.options, sanitize: true })).toBe(false); + // Publishing an executable does not record a native build. + const other = { ...f.options, outPath: join(f.options.outDir, "published") }; + const tracker = new FrontendInputTracker(); + tracker.run(() => trackedReadFile(f.source)); + await publishEarlyExecutableCache(f.root, other, { + llvmPath: f.llvmPath, + irPath: f.irPath, + native, + executableRestored: false, + frontend: tracker.snapshot(), + }); + expect(await hasEarlyExecutableNativeBuild(f.root, other)).toBe(false); +}); diff --git a/packages/compiler/src/executable/executable-cache.ts b/packages/compiler/src/executable/executable-cache.ts index 8f7014dad..1ed23a038 100644 --- a/packages/compiler/src/executable/executable-cache.ts +++ b/packages/compiler/src/executable/executable-cache.ts @@ -327,6 +327,39 @@ function routePath( return join(root, "early-exe-route", routeKey(options)); } +function nativeBuildMarkerPath( + root: string, + options: Omit, +): string { + return join(root, "early-exe-native-build", routeKey(options)); +} + +/** Records that a build of the invocation `options` emitted its native program object, so the + * native helper's partition cache holds its partitions. Unlike the route, the record does not + * depend on the build publishing its executable. */ +export async function recordEarlyExecutableNativeBuild( + root: string | null, + options: Omit, +): Promise { + if (root === null) return; + const marker = nativeBuildMarkerPath(root, options); + await mkdir(dirname(marker), { recursive: true, mode: 0o700 }); + await writeFile(marker, "", { mode: 0o600 }); +} + +/** Whether an earlier build of the invocation `options` recorded its native program object + * (`recordEarlyExecutableNativeBuild`). False when there is no cache. */ +export async function hasEarlyExecutableNativeBuild( + root: string | null, + options: Omit, +): Promise { + if (root === null) return false; + return stat(nativeBuildMarkerPath(root, options)).then( + (info) => info.isFile(), + () => false, + ); +} + function routeIntegrity(stamp: Omit): string { return createHash("sha256") .update("early-executable-route-stamp-v2\0") diff --git a/packages/compiler/src/frontend-process-args.test.ts b/packages/compiler/src/frontend-process-args.test.ts new file mode 100644 index 000000000..a6e9b52e5 --- /dev/null +++ b/packages/compiler/src/frontend-process-args.test.ts @@ -0,0 +1,56 @@ +import { expect, test } from "vitest"; +import { frontendProcessExecArgv } from "./frontend-process-args.js"; + +test("the frontend process keeps heap limits and loaders", () => { + expect( + frontendProcessExecArgv([ + "--max-old-space-size=8192", + "--stack-size=4000", + "--import", + "tsx", + "-r", + "./setup.cjs", + "--conditions=development", + "--enable-source-maps", + ]), + ).toEqual([ + "--max-old-space-size=8192", + "--stack-size=4000", + "--import", + "tsx", + "-r", + "./setup.cjs", + "--conditions=development", + "--enable-source-maps", + ]); +}); + +test("the frontend process drops evaluated code, inspectors, watch and test runner options", () => { + expect( + frontendProcessExecArgv([ + "--max-old-space-size=8192", + "--input-type=module", + "--eval", + "await compile(process.argv[1], {})", + ]), + ).toEqual(["--max-old-space-size=8192"]); + expect(frontendProcessExecArgv(["-e", "code", "-p", "code", "-pe", "code"])).toEqual([]); + expect( + frontendProcessExecArgv(["--eval=code", "--print=code", "--input-type", "module"]), + ).toEqual([]); + expect( + frontendProcessExecArgv([ + "--inspect", + "--inspect-brk=9229", + "--inspect-port", + "9230", + "--watch", + "--watch-path=src", + "--test", + "--test-name-pattern", + "build", + "--test-reporter=spec", + "--expose-gc", + ]), + ).toEqual(["--expose-gc"]); +}); diff --git a/packages/compiler/src/frontend-process-args.ts b/packages/compiler/src/frontend-process-args.ts new file mode 100644 index 000000000..dde9d002f --- /dev/null +++ b/packages/compiler/src/frontend-process-args.ts @@ -0,0 +1,54 @@ +/** Options of the compiler's own process that the frontend process (frontend-process.ts) must not + * receive, with the options among them that take their value as the next argument: the code of + * --eval and --print would run in place of the frontend process's entry, and the inspector, watch + * and test runner options belong to the process that was started with them. */ +const OPTIONS_WITH_VALUES = new Set([ + "-e", + "--eval", + "-p", + "--print", + "--input-type", + "--inspect-port", + "--debug-port", + "--watch-path", + "--test-concurrency", + "--test-coverage-branches", + "--test-coverage-exclude", + "--test-coverage-functions", + "--test-coverage-include", + "--test-coverage-lines", + "--test-global-setup", + "--test-isolation", + "--test-name-pattern", + "--test-reporter", + "--test-reporter-destination", + "--test-shard", + "--test-skip-pattern", + "--test-timeout", +]); +const OPTIONS = new Set(["-i", "--interactive", "-c", "--check", "--test"]); +const OPTION_PREFIXES = ["--inspect", "--debug-port", "--watch", "--test-", "--input-type="]; + +/** The Node.js options of the frontend process: `execArgv` without the options that run code + * (--eval, --print), open an inspector, or watch or test, so that its own entry runs with the + * compiler process's remaining options, such as heap limits and loaders. */ +export function frontendProcessExecArgv(execArgv: readonly string[]): string[] { + const kept: string[] = []; + for (let index = 0; index < execArgv.length; index++) { + const option = execArgv[index]!; + // A short option group such as -pe takes the code that follows it. + if (OPTIONS_WITH_VALUES.has(option) || /^-[a-z]*[ep][a-z]*$/i.test(option)) { + index++; + continue; + } + if ( + OPTIONS.has(option) || + option.startsWith("--eval=") || + option.startsWith("--print=") || + OPTION_PREFIXES.some((prefix) => option.startsWith(prefix)) + ) + continue; + kept.push(option); + } + return kept; +} diff --git a/packages/compiler/src/frontend-process.ts b/packages/compiler/src/frontend-process.ts new file mode 100644 index 000000000..a3d59516f --- /dev/null +++ b/packages/compiler/src/frontend-process.ts @@ -0,0 +1,18 @@ +// The frontend process of an executable build (FrontendProcess and runFrontendProcess in index.ts): +// loads the compiler, then reads the paths of its request and result from stdin. Without them it +// exits: the build found its executable without it. +import { enableCompileCache } from "node:module"; + +try { + enableCompileCache(); +} catch { + // Bytecode caching only speeds up loading. +} +const { runFrontendProcess } = await import("./index.js"); +let input = ""; +for await (const chunk of process.stdin) input += String(chunk); +const [requestPath, resultPath] = input.split("\n"); +if (requestPath !== undefined && requestPath !== "" && resultPath !== undefined) + await runFrontendProcess(requestPath, resultPath); +// The frontend's own helpers (the checker process, timers) must not keep this process alive. +process.exit(0); diff --git a/packages/compiler/src/index.ts b/packages/compiler/src/index.ts index 7dfcb2179..6e9717135 100644 --- a/packages/compiler/src/index.ts +++ b/packages/compiler/src/index.ts @@ -1,5 +1,5 @@ import { optimizationField, type NativeOptimization } from "./backend/optimization.js"; -import { compilationTiming, type CompilationTiming } from "./timing.js"; +import { compilationTiming, type CompilationTiming, type TimingClock } from "./timing.js"; import { prepareExecutableModule } from "./executable/prepare.js"; import { prepareLibrary, @@ -45,8 +45,12 @@ export type { } from "./compile-types.js"; import { compilePackedLibrary } from "./backend/library-pack.js"; import { InternalCompilerError } from "./errors.js"; +import { frontendProcessExecArgv } from "./frontend-process-args.js"; +import { type ChildProcess, spawn } from "node:child_process"; +import { existsSync } from "node:fs"; import { mkdir, readFile, rename, rm, writeFile } from "node:fs/promises"; import { basename, dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; import { clearCcCaches, configuredTargetPlatform, @@ -66,6 +70,8 @@ import { emitLlvmModuleSource, LlvmUnsupportedError } from "./backend/llvm/emitt import { inlineRcOption } from "./backend/llvm/shapes.js"; import { emitNativeArtifact, + llvmInputBytes, + type LlvmInput, NativeCodegenError, writeLlvmSource, } from "./backend/native-codegen.js"; @@ -115,7 +121,11 @@ import { clearResolveCaches } from "./frontend/resolve.js"; import { loadFfiProfile, type FfiProfile } from "./ffi/ffi-manifest.js"; import { executableLinkFeatures } from "./backend/executable-features.js"; import { youngNurseryRequested } from "./backend/young-nursery.js"; -import { FrontendInputTracker, trackedReadFile } from "./frontend/input-tracker.js"; +import { + type FrontendInputSnapshot, + FrontendInputTracker, + trackedReadFile, +} from "./frontend/input-tracker.js"; import { libraryFrontendImplementationFingerprint, publishEarlyLibraryCache, @@ -127,9 +137,11 @@ import { type SemanticLibraryCacheHit, } from "./library/library-cache.js"; import { + hasEarlyExecutableNativeBuild, publishEarlyExecutableCache, publishEarlyExecutableRoute, readEarlyExecutableCache, + recordEarlyExecutableNativeBuild, type EarlyExecutableCacheOptions, type EarlyExecutableNativeFeatures, } from "./executable/executable-cache.js"; @@ -410,10 +422,16 @@ export async function compile( clearCompileSessionCaches(); const frontendInputs = new FrontendInputTracker(); const warnings: CompileWarningSink = { warnings: [], sourceTexts: null }; - const result = await frontendInputs.run(() => - compileTracked(entryPath, opts, frontendInputs, warnings), - ); - return withCompileWarnings(result, warnings); + // compileTracked starts it for a build with no earlier build of the same invocation. + const frontendProcess = preparesInChildProcess(opts) ? new FrontendProcess() : undefined; + try { + const result = await frontendInputs.run(() => + compileTracked(entryPath, opts, frontendInputs, warnings, frontendProcess), + ); + return withCompileWarnings(result, warnings); + } finally { + frontendProcess?.release(); + } } /** Divergence warnings collected while a compile lowers (none when a cache @@ -601,7 +619,7 @@ interface NativeProgramObject { async function emitNativeProgramObject( entryPath: string, opts: CompileRequestOptions, - llvm: string | readonly string[], + llvm: LlvmInput, features: EarlyExecutableNativeFeatures, ): Promise { const stem = basename(entryPath).replace(/\.(ts|mts|cts|js|mjs|cjs)$/, ""); @@ -612,10 +630,7 @@ async function emitNativeProgramObject( const linkPath = `${privateSiblingPath(artifactPath, "native-program-object")}.o`; // The validated program-object artifact is a single object by contract. const target = nativeCodegenTarget(); - const llvmBytes = (typeof llvm === "string" ? [llvm] : llvm).reduce( - (bytes, part) => bytes + Buffer.byteLength(part), - 0, - ); + const llvmBytes = llvmInputBytes(llvm); const partitions = opts.nativeProgramObject === true || target === null ? 1 : nativeProgramPartitions(llvmBytes); // Optimized programs import small runtime functions from the bitcode of @@ -685,7 +700,7 @@ function runtimePackDiagnostic(error: RuntimePackError, entryPath: string): ScrD } interface PreparedExecutable { - llvmSource: string | readonly string[]; + llvmSource: LlvmInput; llvmPath: string; irPath: string | undefined; nativeFeatures: EarlyExecutableNativeFeatures; @@ -840,13 +855,229 @@ async function prepareExecutableInput( return { llvmSource, llvmPath, irPath, nativeFeatures, programSplit, sourceTexts }; } +/** + * The child process that prepares an executable build's input: frontend, lowering and LLVM emission + * (`runFrontendProcess`). Node's heap keeps the pages a build's frontend used for the life of the + * process, so running them in a process of their own leaves the native helper the memory they took. + */ +const frontendProcessEntry = fileURLToPath(new URL("./frontend-process.js", import.meta.url)); + +/** + * Whether `prepareExecutableInputInChild` may prepare this build: optimized executables, when the + * compiler runs from its built files. Development builds split the LLVM text in this process, and + * SCRIPTC_FRONTEND_PROCESS=0 keeps the frontend in-process. + * + * compileTracked starts the process only when no earlier build of the same invocation emitted its + * native program object (`hasEarlyExecutableNativeBuild`). A rebuild finds most native partitions + * in the helper's cache, so its memory peaks in the frontend, and the parent process would add to + * that peak. + */ +function preparesInChildProcess(opts: CompileRequestOptions): boolean { + return ( + process.env["SCRIPTC_FRONTEND_PROCESS"] !== "0" && + (opts.outputKind ?? "exe") === "exe" && + (opts.optimization ?? "release") !== "dev" && + existsSync(frontendProcessEntry) + ); +} + +/** + * The frontend process (frontend-process.ts) of one compile. `start` launches it; it then waits on + * its stdin for the paths of a request and its result (`run`), and exits once its stdin closes + * without them (`release`). + */ +class FrontendProcess { + private child: ChildProcess | undefined; + private exited: Promise | undefined; + private sent = false; + + /** Whether `start` launched the process. */ + get started(): boolean { + return this.child !== undefined; + } + + start(): void { + if (this.child !== undefined) return; + const execArgv = frontendProcessExecArgv(process.execArgv); + const child = spawn(process.execPath, [...execArgv, frontendProcessEntry], { + stdio: ["pipe", "inherit", "inherit"], + }); + this.exited = new Promise((resolveExit) => { + child.on("error", () => resolveExit(null)); + child.on("exit", (code) => resolveExit(code)); + }); + // A failed start surfaces as a failed run; the build then prepares its input in-process. + child.stdin!.on("error", () => undefined); + this.child = child; + } + + /** Sends the request at `requestPath`; resolves with the process's exit code. */ + run(requestPath: string, resultPath: string): Promise { + if (this.child === undefined || this.exited === undefined || this.sent) + return Promise.resolve(null); + this.sent = true; + this.child.stdin!.end(`${requestPath}\n${resultPath}\n`); + return this.exited; + } + + /** Lets the process exit if no request was sent. */ + release(): void { + if (this.child === undefined || this.sent) return; + this.sent = true; + this.child.stdin!.end(); + // The build is done with it: this process need not wait for it to exit. + this.child.unref(); + (this.child.stdin as { unref?: () => void }).unref?.(); + } +} + +interface FrontendProcessRequest { + entryPath: string; + opts: CompileRequestOptions; + buildPlatform: string; + /** The compile's SCRIPTC_TIMING clock, which the frontend process's phases continue. */ + timing: TimingClock; +} + +/** What the frontend process sends back. A thrown error comes back as `failed`. */ +type FrontendProcessResult = + | { + kind: "prepared"; + llvmPath: string; + llvmBytes: number; + irPath: string | undefined; + nativeFeatures: EarlyExecutableNativeFeatures; + sourceTexts: [string, string][]; + warnings: ScrDiagnostic[]; + frontend: FrontendInputSnapshot; + } + | { + kind: "diagnostics"; + diagnostics: ScrDiagnostic[]; + sourceTexts: [string, string][]; + warnings: ScrDiagnostic[]; + frontend: FrontendInputSnapshot; + } + | { kind: "failed" }; + +/** + * `prepareExecutableInput` in the frontend process, with the LLVM text passed on as its file. The + * process's frontend inputs join `frontendInputs` and its warnings `warningSink`. If it fails, the + * input is prepared here, so the build reports the same errors. + */ +async function prepareExecutableInputInChild( + frontendProcess: FrontendProcess, + entryPath: string, + opts: CompileRequestOptions, + ffi: FfiProfile | null, + buildPlatform: string, + timing: CompilationTiming, + timingClock: TimingClock, + frontendInputs: FrontendInputTracker, + warningSink?: CompileWarningSink, +): Promise { + await mkdir(opts.outDir, { recursive: true }); + const requestPath = privateSiblingPath(join(opts.outDir, "frontend-request.json"), "frontend"); + const resultPath = privateSiblingPath(join(opts.outDir, "frontend-result.json"), "frontend"); + let result: FrontendProcessResult = { kind: "failed" }; + try { + const request: FrontendProcessRequest = { entryPath, opts, buildPlatform, timing: timingClock }; + await writeFile(requestPath, JSON.stringify(request), { mode: 0o600 }); + const exitCode = await frontendProcess.run(requestPath, resultPath); + if (exitCode === 0) + result = JSON.parse(await readFile(resultPath, "utf8")) as FrontendProcessResult; + } catch { + result = { kind: "failed" }; + } finally { + await rm(requestPath, { force: true }); + await rm(resultPath, { force: true }); + } + if (result.kind === "failed") + return prepareExecutableInput(entryPath, opts, ffi, buildPlatform, timing, warningSink); + timing("frontend-process"); + for (const probe of result.frontend.probes) frontendInputs.record(probe); + if (!result.frontend.stable) frontendInputs.markUnstable(); + const sourceTexts = new Map(result.sourceTexts); + if (warningSink !== undefined && result.warnings.length > 0) { + warningSink.warnings = result.warnings; + warningSink.sourceTexts = sourceTexts; + } + if (result.kind === "diagnostics") + return { ok: false, diagnostics: result.diagnostics, sourceTexts }; + return { + llvmSource: { file: result.llvmPath, bytes: result.llvmBytes }, + llvmPath: result.llvmPath, + irPath: result.irPath, + nativeFeatures: result.nativeFeatures, + programSplit: null, + sourceTexts, + }; +} + +/** + * The frontend process (frontend-process.ts): prepares the executable input of the request at + * `requestPath` as `compileTracked` would in-process, and writes what it prepared, without the LLVM + * text, to `resultPath`. + */ +export async function runFrontendProcess(requestPath: string, resultPath: string): Promise { + const request = JSON.parse(await readFile(requestPath, "utf8")) as FrontendProcessRequest; + clearCompileSessionCaches(); + const frontendInputs = new FrontendInputTracker(); + const warnings: CompileWarningSink = { warnings: [], sourceTexts: null }; + let result: FrontendProcessResult = { kind: "failed" }; + try { + let ffi: FfiProfile | null = null; + if (request.opts.ffiProfilePath !== undefined) { + const loaded = loadFfiProfile(resolve(request.opts.ffiProfilePath)); + if (!loaded.ok) throw new InternalCompilerError("the FFI profile could not be reloaded"); + ffi = loaded.profile; + } + const prepared = await frontendInputs.run(() => + prepareExecutableInput( + request.entryPath, + request.opts, + ffi, + request.buildPlatform, + compilationTiming(request.timing), + warnings, + ), + ); + const frontend = frontendInputs.snapshot(); + if (!("ok" in prepared)) { + result = { + kind: "prepared", + llvmPath: prepared.llvmPath, + llvmBytes: llvmInputBytes(prepared.llvmSource), + irPath: prepared.irPath, + nativeFeatures: prepared.nativeFeatures, + sourceTexts: [...prepared.sourceTexts], + warnings: warnings.warnings, + frontend, + }; + } else if (!prepared.ok) { + result = { + kind: "diagnostics", + diagnostics: prepared.diagnostics, + sourceTexts: [...prepared.sourceTexts], + warnings: warnings.warnings, + frontend, + }; + } + } catch { + result = { kind: "failed" }; + } + await writeFile(resultPath, JSON.stringify(result), { mode: 0o600 }); +} + async function compileTracked( entryPath: string, opts: CompileRequestOptions, frontendInputs: FrontendInputTracker, warningSink?: CompileWarningSink, + frontendProcess?: FrontendProcess, ): Promise { const timing = compilationTiming(); + const timingStart = Date.now(); entryPath = resolve(entryPath); const outputKind = opts.outputKind ?? "exe"; if ( @@ -1055,16 +1286,10 @@ async function compileTracked( if (outputKind === "exe") { const helperObjectRoute = opts.nativeProgramObject === true || usesPrecompiledRuntimePack(opts, "llvm"); - // Package content and native driver discovery are independent inputs to - // the same cache key. Start both before waiting so edits do not pay their - // filesystem and subprocess latency serially. - const [implementation, nativeEnvironment] = await Promise.all([ - compilerImplementationIdentity(), - helperObjectRoute - ? executableLinkerEnvironmentFingerprint(process.env, nativeCodegenTarget()?.defaultLinker) - : executableNativeEnvironmentFingerprint(), - ]); - earlyCacheOptions = { + const routeOptions: Omit< + EarlyExecutableCacheOptions, + "implementation" | "implementationDependencies" | "nativeEnvironment" + > = { entryPath, outDir: opts.outDir, outPath: opts.outPath, @@ -1094,8 +1319,27 @@ async function compileTracked( ? resolvePlatformLinker(process.env, nativeCodegenTarget()?.defaultLinker) : (process.env["SCRIPTC_CC"] ?? "clang"), ], - nativeEnvironment, nodeVersion: process.version, + }; + // The frontend process prepares the first build of an invocation (preparesInChildProcess). + // Started here, it loads the compiler while this process computes the rest of the cache key. + if ( + frontendProcess !== undefined && + !(await hasEarlyExecutableNativeBuild(cacheRoot, routeOptions)) + ) + frontendProcess.start(); + // Package content and native driver discovery are independent inputs to + // the same cache key. Start both before waiting so edits do not pay their + // filesystem and subprocess latency serially. + const [implementation, nativeEnvironment] = await Promise.all([ + compilerImplementationIdentity(), + helperObjectRoute + ? executableLinkerEnvironmentFingerprint(process.env, nativeCodegenTarget()?.defaultLinker) + : executableNativeEnvironmentFingerprint(), + ]); + earlyCacheOptions = { + ...routeOptions, + nativeEnvironment, implementation: implementation.digest, implementationDependencies: implementation.dependencies, }; @@ -1227,14 +1471,20 @@ async function compileTracked( }; } timing("executable-cache-miss"); - const prepared = await prepareExecutableInput( - entryPath, - opts, - ffi, - buildPlatform, - timing, - warningSink, - ); + const prepared = + frontendProcess?.started === true + ? await prepareExecutableInputInChild( + frontendProcess, + entryPath, + opts, + ffi, + buildPlatform, + timing, + { start: timingStart, previous: Date.now() }, + frontendInputs, + warningSink, + ) + : await prepareExecutableInput(entryPath, opts, ffi, buildPlatform, timing, warningSink); if ("ok" in prepared) return prepared; const { llvmSource, llvmPath, irPath, nativeFeatures, programSplit, sourceTexts } = prepared; const backend = "llvm" as const; @@ -1262,6 +1512,9 @@ async function compileTracked( nativeFeatures, ); timing("native-object"); + await recordEarlyExecutableNativeBuild(cacheRoot, executableCacheOptions).catch( + () => undefined, + ); } catch (err) { if (!(err instanceof NativeCodegenError)) throw err; return { diff --git a/packages/compiler/src/timing.ts b/packages/compiler/src/timing.ts index 79c07cb7e..6d9f6844d 100644 --- a/packages/compiler/src/timing.ts +++ b/packages/compiler/src/timing.ts @@ -1,11 +1,20 @@ export type CompilationTiming = (phase: string, detail?: Record) => void; +/** The start of a timing and its latest phase, as `Date.now()` values. */ +export interface TimingClock { + start: number; + previous: number; +} + /** Opt-in phase durations shared by the Node seed and native compiler. - * The disabled path does not read the clock or write compiler output. */ -export function compilationTiming(): CompilationTiming { + * The disabled path does not read the clock or write compiler output. + * With `clock`, the phases continue a timing that started in another + * process, so a frontend process reports them as its parent compile would. */ +export function compilationTiming(clock?: TimingClock): CompilationTiming { if (process.env["SCRIPTC_TIMING"] !== "1") return () => {}; - const start = performance.now(); - let previous = start; + const opened = performance.now(); + const start = clock === undefined ? opened : opened - (Date.now() - clock.start); + let previous = clock === undefined ? start : opened - (Date.now() - clock.previous); return (phase, detail = {}) => { const now = performance.now(); process.stderr.write(