From 5dc4a4daa613019fee32f8775fa99bb9e2eccedd Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Tue, 27 Oct 2026 15:04:45 -0500 Subject: [PATCH] Map large executables' text with 2 MiB pages --- docs/content/docs/platforms.mdx | 2 + .../compiler/src/backend/huge-text.test.ts | 89 ++++++++++++++ packages/compiler/src/backend/huge-text.ts | 110 ++++++++++++++++++ packages/compiler/src/backend/link-plan.ts | 9 ++ packages/compiler/src/backend/linker.ts | 4 + packages/compiler/src/native/compiler.ts | 10 ++ packages/runtime/src/scr_lib.c | 42 +++++++ 7 files changed, 266 insertions(+) create mode 100644 packages/compiler/src/backend/huge-text.test.ts create mode 100644 packages/compiler/src/backend/huge-text.ts diff --git a/docs/content/docs/platforms.mdx b/docs/content/docs/platforms.mdx index 72cd65a3c..462a043ba 100644 --- a/docs/content/docs/platforms.mdx +++ b/docs/content/docs/platforms.mdx @@ -53,6 +53,8 @@ fib-linux: ELF 64-bit LSB executable, ARM aarch64, version 1 (SYSV), dynamically Linux executables use the native Linux implementations of the event loop (epoll), servers, TLS (with the system's CA bundle), and `fs.watch`, and they are tested against Node.js on Linux, including Alpine. +An executable with at least 4 MiB of program code is linked with 2 MiB segment alignment when lld or zig links it, so the kernel can map its code with 2 MiB pages instead of 4 KiB pages. This takes effect when transparent huge pages are set to `madvise` or `always` and the file system keeps files in the page cache as 2 MiB folios (XFS, for example). scriptc writes such an executable with one write so that the page cache holds it that way; a copy written in small chunks maps with 4 KiB pages until it is read from disk again. The alignment also aligns the executable's load address to 2 MiB, which leaves 19 bits of address randomization for it instead of 28. Smaller executables, and links by GNU ld, keep 4 KiB alignment. + ### Windows (x86_64) ```console diff --git a/packages/compiler/src/backend/huge-text.test.ts b/packages/compiler/src/backend/huge-text.test.ts new file mode 100644 index 000000000..39ef52408 --- /dev/null +++ b/packages/compiler/src/backend/huge-text.test.ts @@ -0,0 +1,89 @@ +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, expect, test } from "vitest"; +import { + HUGE_TEXT_MIN_CODE_BYTES, + elfObjectCodeBytes, + hugeTextLinkerArgs, + hugeTextProgram, +} from "./huge-text.js"; + +const dirs: string[] = []; +afterEach(() => { + for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true }); +}); + +/** An ELF64 little-endian relocatable object's headers: one section per + * entry, as [type, flags, size]. Section contents are left out, since only + * the headers are read. */ +function elfObject(sections: readonly [number, number, number][]): Buffer { + const shoff = 64; + const bytes = Buffer.alloc(shoff + (sections.length + 1) * 64); + bytes.writeUInt32BE(0x7f454c46, 0); + bytes[4] = 2; // ELFCLASS64 + bytes[5] = 1; // ELFDATA2LSB + bytes[6] = 1; + bytes.writeUInt16LE(1, 16); // ET_REL + bytes.writeUInt16LE(62, 18); // EM_X86_64 + bytes.writeBigUInt64LE(BigInt(shoff), 40); + bytes.writeUInt16LE(64, 52); + bytes.writeUInt16LE(64, 58); + bytes.writeUInt16LE(sections.length + 1, 60); + sections.forEach(([type, flags, size], index) => { + const at = shoff + (index + 1) * 64; + bytes.writeUInt32LE(type, at + 4); + bytes.writeBigUInt64LE(BigInt(flags), at + 8); + bytes.writeBigUInt64LE(BigInt(size), at + 32); + }); + return bytes; +} + +function file(name: string, bytes: Buffer): string { + const dir = mkdtempSync(join(tmpdir(), "scriptc-huge-text-")); + dirs.push(dir); + const path = join(dir, name); + writeFileSync(path, bytes); + return path; +} + +const PROGBITS = 1; +const NOBITS = 8; +const ALLOC = 0x2; +const EXECINSTR = 0x4; + +test("an object's code is the size of its executable PROGBITS sections", () => { + const object = file( + "program.o", + elfObject([ + [PROGBITS, ALLOC | EXECINSTR, 3 << 20], + [PROGBITS, ALLOC | EXECINSTR, 1000], + [PROGBITS, ALLOC, 5 << 20], + [NOBITS, ALLOC, 7 << 20], + ]), + ); + expect(elfObjectCodeBytes(object)).toBe((3 << 20) + 1000); + expect(elfObjectCodeBytes(file("not-elf.o", Buffer.from("not an object")))).toBe(0); + expect(elfObjectCodeBytes(join(object, "..", "missing.o"))).toBe(0); +}); + +test("only Linux programs with an interior 2 MiB window of code link for huge pages", () => { + const linux = { platform: "linux" } as const; + const half = file( + "half.o", + elfObject([[PROGBITS, ALLOC | EXECINSTR, HUGE_TEXT_MIN_CODE_BYTES / 2]]), + ); + const small = file("small.o", elfObject([[PROGBITS, ALLOC | EXECINSTR, 1 << 20]])); + expect(hugeTextProgram(linux, [half, small])).toBe(false); + // Partitions of one program count together. + expect(hugeTextProgram(linux, [half, half])).toBe(true); + expect(hugeTextProgram({ platform: "darwin" } as const, [half, half])).toBe(false); + expect(hugeTextProgram(linux, [])).toBe(false); +}); + +test("the 2 MiB alignment applies to lld and zig links of huge-text programs", () => { + expect(hugeTextLinkerArgs(true, true)).toEqual(["-Wl,-z,max-page-size=0x200000"]); + // GNU ld pads the file to the alignment. + expect(hugeTextLinkerArgs(true, false)).toEqual([]); + expect(hugeTextLinkerArgs(false, true)).toEqual([]); +}); diff --git a/packages/compiler/src/backend/huge-text.ts b/packages/compiler/src/backend/huge-text.ts new file mode 100644 index 000000000..4db824074 --- /dev/null +++ b/packages/compiler/src/backend/huge-text.ts @@ -0,0 +1,110 @@ +/** + * Text on 2 MiB pages for large Linux executables. + * + * The kernel maps a 2 MiB window of an executable's text with one huge page + * when the window's virtual address and file offset are both 2 MiB aligned + * and the page cache holds the window as one 2 MiB folio. A link with + * `-z max-page-size=0x200000` keeps every segment's address congruent to its + * file offset modulo 2 MiB, so every 2 MiB window inside the text segment + * qualifies. lld and zig add no file padding for it; GNU ld pads the file to + * the alignment, so links that fall back to it leave the flag out. + * + * The alignment also aligns a position-independent executable's load + * address to 2 MiB, which leaves 19 bits of address randomization for it + * instead of 28. Only executables whose program code fills an interior 2 MiB + * window can gain from huge pages, so smaller executables link as before. + * + * lld writes its output through a memory mapping, which leaves the file in + * the page cache as 4 KiB folios. A large executable is published by writing + * its bytes with one write instead, which leaves 2 MiB folios where the file + * system supports them. The runtime advises the interior windows huge pages + * at startup (scr_lib.c), so a file that is read back from disk is read as + * 2 MiB folios too. + */ +import { + chmodSync, + closeSync, + openSync, + readFileSync, + readSync, + renameSync, + statSync, + writeFileSync, +} from "node:fs"; +import type { NativeTargetSpec } from "./targets.js"; + +/** Program code from which an executable is linked for huge pages: text of + * this size has at least one 2 MiB window inside it wherever it starts. */ +export const HUGE_TEXT_MIN_CODE_BYTES = 4 << 20; + +const SHF_EXECINSTR = 0x4n; + +/** The bytes of executable sections in an ELF64 little-endian object, or 0 + * when the file is not one or cannot be read. Reads only the file and + * section headers. */ +export function elfObjectCodeBytes(path: string): number { + let fd: number; + try { + fd = openSync(path, "r"); + } catch { + return 0; + } + try { + const header = Buffer.alloc(64); + if (readSync(fd, header, 0, 64, 0) !== 64) return 0; + const view = new DataView(header.buffer, header.byteOffset, header.byteLength); + if (view.getUint32(0, false) !== 0x7f454c46 || header[4] !== 2 || header[5] !== 1) return 0; + const shoff = Number(view.getBigUint64(40, true)); + const shentsize = view.getUint16(58, true); + let shnum = view.getUint16(60, true); + if (shentsize !== 64 || shoff === 0) return 0; + if (shnum === 0) { + const first = Buffer.alloc(64); + if (readSync(fd, first, 0, 64, shoff) !== 64) return 0; + shnum = Number(new DataView(first.buffer, first.byteOffset, 64).getBigUint64(32, true)); + } + const table = Buffer.alloc(shnum * 64); + if (readSync(fd, table, 0, table.length, shoff) !== table.length) return 0; + const sections = new DataView(table.buffer, table.byteOffset, table.byteLength); + let code = 0; + for (let i = 0; i < shnum; i++) { + const at = i * 64; + // SHT_PROGBITS sections with SHF_EXECINSTR. + if (sections.getUint32(at + 4, true) !== 1) continue; + if ((sections.getBigUint64(at + 8, true) & SHF_EXECINSTR) === 0n) continue; + code += Number(sections.getBigUint64(at + 32, true)); + } + return code; + } finally { + closeSync(fd); + } +} + +/** Whether a link of these program objects is laid out for huge pages. */ +export function hugeTextProgram( + target: Pick, + programObjects: readonly string[], +): boolean { + if (target.platform !== "linux") return false; + let code = 0; + for (const object of programObjects) { + code += elfObjectCodeBytes(object); + if (code >= HUGE_TEXT_MIN_CODE_BYTES) return true; + } + return false; +} + +/** The link arguments that align a huge-text executable's segments to + * 2 MiB, for links by lld or zig. */ +export function hugeTextLinkerArgs(hugeText: boolean, linksWithLldOrZig: boolean): string[] { + return hugeText && linksWithLldOrZig ? ["-Wl,-z,max-page-size=0x200000"] : []; +} + +/** Rewrites a linked huge-text executable with one write, so the page cache + * holds it in 2 MiB folios; its caller then publishes it as usual. */ +export function rewriteHugeTextExecutable(linked: string): void { + const written = `${linked}.written`; + writeFileSync(written, readFileSync(linked)); + chmodSync(written, statSync(linked).mode & 0o7777); + renameSync(written, linked); +} diff --git a/packages/compiler/src/backend/link-plan.ts b/packages/compiler/src/backend/link-plan.ts index c3c0cb5a7..19cff2d5f 100644 --- a/packages/compiler/src/backend/link-plan.ts +++ b/packages/compiler/src/backend/link-plan.ts @@ -13,6 +13,7 @@ import type { NativeArtifactDependency } from "./native/contracts.js"; import { needsDarwinDebugSymbols } from "./debug-symbols.js"; import { loadRuntimePack, type RuntimePackSelection } from "./runtime-pack.js"; import { executableLinkInputs } from "./link-plan-core.js"; +import { hugeTextProgram } from "./huge-text.js"; import type { NativeTargetSpec, WindowsSubsystem } from "./targets.js"; export interface NativeLinkPlan { @@ -30,6 +31,8 @@ export interface NativeLinkPlan { darwinDebugSymbols?: boolean; /** Linked with `--static`: the linked executable is checked before use. */ staticLink?: boolean; + /** Program code large enough for text on huge pages (huge-text.ts). */ + hugeText?: boolean; } export async function createNativeLinkPlan(options: { @@ -85,5 +88,11 @@ export async function createNativeLinkPlan(options: { programObjectDependencies: [...(options.programObjectDependencies ?? [])], runtimePack, ...(options.staticLink === true ? { staticLink: true } : {}), + ...(hugeTextProgram(options.target, [ + options.programObject, + ...(options.programPartitions ?? []), + ]) + ? { hugeText: true } + : {}), }; } diff --git a/packages/compiler/src/backend/linker.ts b/packages/compiler/src/backend/linker.ts index fe476360d..ab76597ed 100644 --- a/packages/compiler/src/backend/linker.ts +++ b/packages/compiler/src/backend/linker.ts @@ -24,6 +24,7 @@ import { type NativeArtifactDependency } from "./native/contracts.js"; import { RuntimePackError, stageRuntimePackArtifacts } from "./runtime-pack.js"; import type { NativeLinkPlan } from "./link-plan.js"; import { inputsWithoutLld, lldLinkerArgs } from "./lld.js"; +import { hugeTextLinkerArgs, rewriteHugeTextExecutable } from "./huge-text.js"; import { systemLibraryLinkerArgs } from "./link-plan-core.js"; import { createDarwinDebugSymbols, @@ -248,11 +249,13 @@ export async function linkNativeExecutable( stagedRoot = staged.root; // Linux links with lld when the default driver finds it. const lld = explicitLinker === undefined ? lldLinkerArgs(plan.target.platform, linker) : []; + const zig = explicitLinker === undefined && plan.target.defaultLinker === "zig"; const linkArgs = (withLld: boolean): string[] => { const inputs = plan.inputs.map((input) => staged.replacements.get(input) ?? input); return [ ...(explicitLinker === undefined ? plan.target.defaultLinkerArgs : []), ...(withLld ? lld : []), + ...hugeTextLinkerArgs(plan.hugeText === true, (withLld && lld.length !== 0) || zig), ...plan.driverFlags, ...mapArgs, ...(withLld ? inputs : inputsWithoutLld(plan.target.platform, inputs)), @@ -298,6 +301,7 @@ export async function linkNativeExecutable( }); const output = await stat(privateOut); if (!output.isFile() || output.size === 0) throw new Error("linker produced no executable"); + if (plan.hugeText === true) rewriteHugeTextExecutable(privateOut); if (mapArgs.length !== 0) { const refusal = staticExecutableRefusal(plan.target, true, await readFile(linkMap, "utf8")); if (refusal !== null) throw new NativeCodegenError("SC3002", refusal); diff --git a/packages/compiler/src/native/compiler.ts b/packages/compiler/src/native/compiler.ts index b46778827..19351a81a 100644 --- a/packages/compiler/src/native/compiler.ts +++ b/packages/compiler/src/native/compiler.ts @@ -75,6 +75,11 @@ import { splitLlvmLibraryProgram } from "../backend/llvm/split.js"; import { prepareNativeExecutable, prepareNativeLibrary } from "./prepare.js"; import { buildSanitizedRuntime, sanitizerDriver, sanitizerFlags } from "./sanitizer.js"; import { inputsWithoutLld, lldLinkerArgs } from "../backend/lld.js"; +import { + hugeTextLinkerArgs, + hugeTextProgram, + rewriteHugeTextExecutable, +} from "../backend/huge-text.js"; import { openNativeExecutableCache } from "./executable-cache.js"; import { LinkerJobs } from "./linker-job.js"; import { nativeFileIdentity } from "./file-identity.js"; @@ -637,11 +642,15 @@ export class NativeCompiler { }); // Linux links with lld when the driver finds it. const lld = options.sanitize ? [] : lldLinkerArgs(target.platform, toolchain.linker); + const hugeText = + !options.sanitize && hugeTextProgram(target, [object, ...programObjects.slice(1)]); + const zig = process.env["SCRIPTC_LINKER"] === undefined && target.defaultLinker === "zig"; const linkMap = join(stage, "link.map"); const mapArgs = staticLinkMapArgs(target, options.staticLink === true, linkMap); const linkArgsWith = (withLld: boolean): string[] => [ ...(options.sanitize ? ["-fsanitize=address"] : toolchain.linkerArgs), ...(withLld ? lld : []), + ...hugeTextLinkerArgs(hugeText, (withLld && lld.length !== 0) || zig), ...plan.driverFlags, ...mapArgs, ...(withLld ? plan.inputs : inputsWithoutLld(target.platform, plan.inputs)), @@ -679,6 +688,7 @@ export class NativeCompiler { } timing("native-link"); requireNativeArtifact(stagedOutput); + if (hugeText) rewriteHugeTextExecutable(stagedOutput); if (mapArgs.length !== 0) { const refusal = staticExecutableRefusal(target, true, readFileSync(linkMap, "utf8")); if (refusal !== null) throw new NativeCodegenError("SC3002", refusal); diff --git a/packages/runtime/src/scr_lib.c b/packages/runtime/src/scr_lib.c index ad42a5824..e8009d25a 100644 --- a/packages/runtime/src/scr_lib.c +++ b/packages/runtime/src/scr_lib.c @@ -2066,6 +2066,48 @@ bool scr_process_has_exit_code(void) { return scr_exit_code_assigned || scr_exit_code_hint_get() != 0; } +#if defined(__linux__) && !defined(SCR_LIB) +#include +#include +/* Text on huge pages (huge-text.ts). An executable with enough program code + * is linked with 2 MiB segment alignment, so each 2 MiB window inside its + * text segment starts at a 2 MiB file offset, and the kernel maps a window + * with one huge page when the page cache holds it as one 2 MiB folio. + * MADV_HUGEPAGE on those windows makes the kernel read a window that is not + * cached as a 2 MiB folio. glibc runs it from .preinit_array, before _init + * and the constructors fault in any of the text; musl, which has no + * .preinit_array, runs it as a constructor. Executables with 4 KiB alignment + * make no call. */ +#define SCR_TEXT_HUGE_PAGE ((uintptr_t)2 << 20) + +static int scr_text_segment(struct dl_phdr_info *info, size_t size, void *data) { + (void)size; + uintptr_t *range = data; + for (size_t i = 0; i < info->dlpi_phnum; i++) { + const ElfW(Phdr) *ph = &info->dlpi_phdr[i]; + if (ph->p_type != PT_LOAD || !(ph->p_flags & PF_X) || ph->p_align < SCR_TEXT_HUGE_PAGE) continue; + range[0] = (uintptr_t)info->dlpi_addr + (uintptr_t)ph->p_vaddr; + range[1] = range[0] + (uintptr_t)ph->p_memsz; + break; + } + return 1; /* the first object is the executable */ +} + +static bool scr_text_huge_pages_done; + +__attribute__((constructor)) static void scr_text_huge_pages(void) { + if (scr_text_huge_pages_done) return; + scr_text_huge_pages_done = true; + uintptr_t range[2] = {0, 0}; + (void)dl_iterate_phdr(scr_text_segment, range); + uintptr_t lo = (range[0] + SCR_TEXT_HUGE_PAGE - 1) & ~(SCR_TEXT_HUGE_PAGE - 1); + uintptr_t hi = range[1] & ~(SCR_TEXT_HUGE_PAGE - 1); + if (hi > lo) (void)madvise((void *)lo, hi - lo, MADV_HUGEPAGE); +} +__attribute__((section(".preinit_array"), used)) static void (*const scr_text_huge_pages_entry)(void) = + scr_text_huge_pages; +#endif + #if defined(__linux__) #include #include