diff --git a/packages/compiler/src/backend/llvm/lib-shared.ts b/packages/compiler/src/backend/llvm/lib-shared.ts index 1c7604c34..9d85547eb 100644 --- a/packages/compiler/src/backend/llvm/lib-shared.ts +++ b/packages/compiler/src/backend/llvm/lib-shared.ts @@ -1754,6 +1754,9 @@ export const USES_TIMERS_LIB_FNS = new Set([ "timers.setImmediateFnValue", "timers.immediatePromise", "dyn.builtinFunction", - // An asynchronous call into Node needs the loop to settle. + // A call into Node needs the loop: an asynchronous one to settle, and + // either kind may leave Node with ref'd timers or I/O, which the loop + // drives (scr_node_island.c) before the program exits. "nodeIsland.call", + "nodeIsland.callSync", ]); diff --git a/packages/compiler/src/index.ts b/packages/compiler/src/index.ts index c66e37901..0571b04b0 100644 --- a/packages/compiler/src/index.ts +++ b/packages/compiler/src/index.ts @@ -108,7 +108,7 @@ import { nodeIslandNodeDiag, type ScrDiagnostic, } from "./diagnostics/diagnostic.js"; -import { libnodeRecord } from "./node-islands/libnode.js"; +import { libnodeCacheKey, libnodeRecord } from "./node-islands/libnode.js"; import { loadLibraryProfile, type LibraryProfile } from "./library/library-profile.js"; import { clearFenceEvalCaches, decorateLibraryRefusals } from "./library/fence-eval.js"; import { @@ -1375,8 +1375,8 @@ async function compileTracked( if (outputKind === "exe") { const helperObjectRoute = opts.nativeProgramObject === true || usesPrecompiledRuntimePack(opts, "llvm"); - // Calls into Node record the libnode pack (node-islands/libnode.ts). - const libnodeKey = opts.libnode ?? process.env["SCRIPTC_LIBNODE"]; + // Calls into Node record the libnode pack, keyed by its manifest (node-islands/libnode.ts). + const libnodeKey = libnodeCacheKey(opts.libnode); const routeOptions: Omit< EarlyExecutableCacheOptions, "implementation" | "implementationDependencies" | "nativeEnvironment" @@ -1446,6 +1446,17 @@ async function compileTracked( if (earlyCacheOptions === null) { throw new InternalCompilerError("executable cache hit without executable cache options"); } + // The pack a --libnode=beside build places next to the executable is an output of the + // build, like the executable: a restored executable that calls into Node gets it again. + if (earlyHit.native.nodeIsland === true && opts.libnode === "beside") { + const recorded = libnodeRecord(opts.libnode, opts.outPath); + if (typeof recorded === "string") + return { + ok: false, + diagnostics: [nodeIslandNodeDiag(recorded, entryPath)], + sourceTexts: new Map(), + }; + } const executableCacheOptions = earlyCacheOptions; // Route/proof metadata is independently evictable. A full-compiler // fallback that still finds the validated payload repairs that lightweight diff --git a/packages/compiler/src/node-islands/libnode.test.ts b/packages/compiler/src/node-islands/libnode.test.ts index 107e4aceb..6fadedcec 100644 --- a/packages/compiler/src/node-islands/libnode.test.ts +++ b/packages/compiler/src/node-islands/libnode.test.ts @@ -1,8 +1,9 @@ -import { existsSync, mkdirSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { describe, expect, test } from "vitest"; -import { libnodeRecord, NODE_BRIDGE_ABI } from "./libnode.js"; +import { USES_TIMERS_LIB_FNS } from "../backend/llvm/lib-shared.js"; +import { libnodeCacheKey, libnodeRecord, NODE_BRIDGE_ABI } from "./libnode.js"; function pack(nodeVersion: string, bridgeAbi = NODE_BRIDGE_ABI): string { const dir = mkdtempSync(join(tmpdir(), "scriptc-libnode-")); @@ -55,6 +56,19 @@ describe("the recorded libnode pack", () => { expect(existsSync(join(out, "app.libnode/manifest.json"))).toBe(true); }); + test("beside places the pack again after the outputs were deleted", () => { + // A cached executable restored without its pack gets it again (compile, index.ts). + const dir = pack("24.22.0"); + const out = mkdtempSync(join(tmpdir(), "scriptc-libnode-out-")); + libnodeRecord("beside", join(out, "app"), { SCRIPTC_LIBNODE: dir }); + rmSync(join(out, "app.libnode"), { recursive: true, force: true }); + expect(libnodeRecord("beside", join(out, "app"), { SCRIPTC_LIBNODE: dir })).toEqual({ + libnode: "$ORIGIN/app.libnode", + nodeVersion: "24.22.0", + }); + expect(readFileSync(join(out, "app.libnode/libscr_node_bridge.so"), "utf8")).toBe("bridge"); + }); + test("a pack the deployment places beside the executable needs no manifest at build time", () => { expect(libnodeRecord("$ORIGIN/node", "/out/app", {})).toEqual({ libnode: "$ORIGIN/node", @@ -62,3 +76,35 @@ describe("the recorded libnode pack", () => { }); }); }); + +describe("the libnode pack in an executable's cache key", () => { + test("is absent without --libnode or SCRIPTC_LIBNODE", () => { + expect(libnodeCacheKey(undefined, {})).toBeUndefined(); + }); + + test("changes when the pack in the same directory is upgraded", () => { + const dir = pack("24.22.0"); + const before = libnodeCacheKey(dir, {}); + expect(libnodeCacheKey(dir, {})).toBe(before); + writeFileSync( + join(dir, "manifest.json"), + JSON.stringify({ nodeVersion: "24.23.0", bridgeAbi: NODE_BRIDGE_ABI, files: {} }), + ); + expect(libnodeCacheKey(dir, {})).not.toBe(before); + }); + + test("names the pack beside copies from", () => { + const a = pack("24.22.0"); + const b = pack("24.22.0"); + expect(libnodeCacheKey("beside", { SCRIPTC_LIBNODE: a })).not.toBe( + libnodeCacheKey("beside", { SCRIPTC_LIBNODE: b }), + ); + expect(libnodeCacheKey("beside", { SCRIPTC_LIBNODE: a })).not.toBe(libnodeCacheKey(a, {})); + }); +}); + +test("calls into Node keep the program's event loop running", () => { + // Node's ref'd timers and I/O run on the program's loop (scr_node_island.c). + expect(USES_TIMERS_LIB_FNS.has("nodeIsland.call")).toBe(true); + expect(USES_TIMERS_LIB_FNS.has("nodeIsland.callSync")).toBe(true); +}); diff --git a/packages/compiler/src/node-islands/libnode.ts b/packages/compiler/src/node-islands/libnode.ts index defb13869..2ba056023 100644 --- a/packages/compiler/src/node-islands/libnode.ts +++ b/packages/compiler/src/node-islands/libnode.ts @@ -12,6 +12,7 @@ * as `.libnode/`, recorded as `$ORIGIN/.libnode`, so the * executable and its Node travel together; a recorded `$ORIGIN/` path is * also the hook for bundling Node into the executable itself later. */ +import { createHash } from "node:crypto"; import { copyFileSync, linkSync, mkdirSync, readFileSync, rmSync, statSync } from "node:fs"; import { basename, isAbsolute, join, resolve } from "node:path"; @@ -109,3 +110,28 @@ export function libnodeRecord( placeBeside(source, manifest, `${outPath}.libnode`); return { libnode: `${ORIGIN}${name}`, nodeVersion: manifest.nodeVersion }; } + +/** The part of an executable's cache key that names its libnode pack, or + * undefined when neither --libnode nor SCRIPTC_LIBNODE is set. It holds the + * option, the pack directory and a digest of the pack's manifest.json, so + * that a pack upgraded in place (another Node version, bridge ABI or file + * list) gives cached executables a new key instead of their old record. */ +export function libnodeCacheKey( + option: string | undefined, + env: NodeJS.ProcessEnv = process.env, +): string | undefined { + const fromEnv = env["SCRIPTC_LIBNODE"]; + if (option === undefined && fromEnv === undefined) return undefined; + const source = option !== undefined && option !== "beside" ? option : (fromEnv ?? ""); + let manifest = "none"; + if (isAbsolute(source)) { + try { + manifest = createHash("sha256") + .update(readFileSync(join(source, "manifest.json"))) + .digest("hex"); + } catch { + // A missing pack fails the build that records it (libnodeRecord). + } + } + return `${option ?? ""}|${source}|${manifest}`; +} diff --git a/packages/runtime/src/scr_runtime.h b/packages/runtime/src/scr_runtime.h index bf01a4a9b..79c4a105f 100644 --- a/packages/runtime/src/scr_runtime.h +++ b/packages/runtime/src/scr_runtime.h @@ -77,5 +77,6 @@ #include "scr_fs_cp.h" #include "scr_wasm_abi.h" #include "scr_wasm.h" +#include "scr_node_island.h" #endif /* SCR_RUNTIME_H */ diff --git a/tests/harness/corpus-costs/6800-node-call-sync/main.ts.json b/tests/harness/corpus-costs/6800-node-call-sync/main.ts.json new file mode 100644 index 000000000..3ca32050d --- /dev/null +++ b/tests/harness/corpus-costs/6800-node-call-sync/main.ts.json @@ -0,0 +1 @@ +{ "plain": 2856, "san": 39153 } diff --git a/tests/harness/corpus-costs/6801-node-call-errors/main.ts.json b/tests/harness/corpus-costs/6801-node-call-errors/main.ts.json new file mode 100644 index 000000000..6583fe71c --- /dev/null +++ b/tests/harness/corpus-costs/6801-node-call-errors/main.ts.json @@ -0,0 +1 @@ +{ "plain": 1928, "san": 35656 } diff --git a/tests/harness/corpus-costs/6802-node-call-async/main.ts.json b/tests/harness/corpus-costs/6802-node-call-async/main.ts.json new file mode 100644 index 000000000..3aa5290bd --- /dev/null +++ b/tests/harness/corpus-costs/6802-node-call-async/main.ts.json @@ -0,0 +1 @@ +{ "plain": 3224, "san": 32694 } diff --git a/tests/harness/corpus-costs/6803-node-call-exit/main.ts.json b/tests/harness/corpus-costs/6803-node-call-exit/main.ts.json new file mode 100644 index 000000000..134fe7954 --- /dev/null +++ b/tests/harness/corpus-costs/6803-node-call-exit/main.ts.json @@ -0,0 +1 @@ +{ "plain": 2860, "san": 20343 }