Map large executables' text with 2 MiB pages

docs/content/docs/platforms.mdx

@@ -53,6 +53,8 @@ fib-linux: ELF 64-bit LSB executable, ARM aarch64, version 1 (SYSV), dynamically
 
 Linux executables use the native Linux implementations of the event loop (epoll), servers, TLS (with the system's CA bundle), and `fs.watch`, and they are tested against Node.js on Linux, including Alpine.
 
+An executable with at least 4 MiB of program code is linked with 2 MiB segment alignment when lld or zig links it, so the kernel can map its code with 2 MiB pages instead of 4 KiB pages. This takes effect when transparent huge pages are set to `madvise` or `always` and the file system keeps files in the page cache as 2 MiB folios (XFS, for example). scriptc writes such an executable with one write so that the page cache holds it that way; a copy written in small chunks maps with 4 KiB pages until it is read from disk again. The alignment also aligns the executable's load address to 2 MiB, which leaves 19 bits of address randomization for it instead of 28. Smaller executables, and links by GNU ld, keep 4 KiB alignment.
+
 ### Windows (x86_64)
 
 ```console

packages/compiler/src/backend/huge-text.test.ts

@@ -0,0 +1,89 @@
+import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterEach, expect, test } from "vitest";
+import {
+  HUGE_TEXT_MIN_CODE_BYTES,
+  elfObjectCodeBytes,
+  hugeTextLinkerArgs,
+  hugeTextProgram,
+} from "./huge-text.js";
+
+const dirs: string[] = [];
+afterEach(() => {
+  for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true });
+});
+
+/** An ELF64 little-endian relocatable object's headers: one section per
+ * entry, as [type, flags, size]. Section contents are left out, since only
+ * the headers are read. */
+function elfObject(sections: readonly [number, number, number][]): Buffer {
+  const shoff = 64;
+  const bytes = Buffer.alloc(shoff + (sections.length + 1) * 64);
+  bytes.writeUInt32BE(0x7f454c46, 0);
+  bytes[4] = 2; // ELFCLASS64
+  bytes[5] = 1; // ELFDATA2LSB
+  bytes[6] = 1;
+  bytes.writeUInt16LE(1, 16); // ET_REL
+  bytes.writeUInt16LE(62, 18); // EM_X86_64
+  bytes.writeBigUInt64LE(BigInt(shoff), 40);
+  bytes.writeUInt16LE(64, 52);
+  bytes.writeUInt16LE(64, 58);
+  bytes.writeUInt16LE(sections.length + 1, 60);
+  sections.forEach(([type, flags, size], index) => {
+    const at = shoff + (index + 1) * 64;
+    bytes.writeUInt32LE(type, at + 4);
+    bytes.writeBigUInt64LE(BigInt(flags), at + 8);
+    bytes.writeBigUInt64LE(BigInt(size), at + 32);
+  });
+  return bytes;
+}
+
+function file(name: string, bytes: Buffer): string {
+  const dir = mkdtempSync(join(tmpdir(), "scriptc-huge-text-"));
+  dirs.push(dir);
+  const path = join(dir, name);
+  writeFileSync(path, bytes);
+  return path;
+}
+
+const PROGBITS = 1;
+const NOBITS = 8;
+const ALLOC = 0x2;
+const EXECINSTR = 0x4;
+
+test("an object's code is the size of its executable PROGBITS sections", () => {
+  const object = file(
+    "program.o",
+    elfObject([
+      [PROGBITS, ALLOC | EXECINSTR, 3 << 20],
+      [PROGBITS, ALLOC | EXECINSTR, 1000],
+      [PROGBITS, ALLOC, 5 << 20],
+      [NOBITS, ALLOC, 7 << 20],
+    ]),
+  );
+  expect(elfObjectCodeBytes(object)).toBe((3 << 20) + 1000);
+  expect(elfObjectCodeBytes(file("not-elf.o", Buffer.from("not an object")))).toBe(0);
+  expect(elfObjectCodeBytes(join(object, "..", "missing.o"))).toBe(0);
+});
+
+test("only Linux programs with an interior 2 MiB window of code link for huge pages", () => {
+  const linux = { platform: "linux" } as const;
+  const half = file(
+    "half.o",
+    elfObject([[PROGBITS, ALLOC | EXECINSTR, HUGE_TEXT_MIN_CODE_BYTES / 2]]),
+  );
+  const small = file("small.o", elfObject([[PROGBITS, ALLOC | EXECINSTR, 1 << 20]]));
+  expect(hugeTextProgram(linux, [half, small])).toBe(false);
+  // Partitions of one program count together.
+  expect(hugeTextProgram(linux, [half, half])).toBe(true);
+  expect(hugeTextProgram({ platform: "darwin" } as const, [half, half])).toBe(false);
+  expect(hugeTextProgram(linux, [])).toBe(false);
+});
+
+test("the 2 MiB alignment applies to lld and zig links of huge-text programs", () => {
+  expect(hugeTextLinkerArgs(true, true)).toEqual(["-Wl,-z,max-page-size=0x200000"]);
+  // GNU ld pads the file to the alignment.
+  expect(hugeTextLinkerArgs(true, false)).toEqual([]);
+  expect(hugeTextLinkerArgs(false, true)).toEqual([]);
+});

packages/compiler/src/backend/huge-text.ts

@@ -0,0 +1,110 @@
+/**
+ * Text on 2 MiB pages for large Linux executables.
+ *
+ * The kernel maps a 2 MiB window of an executable's text with one huge page
+ * when the window's virtual address and file offset are both 2 MiB aligned
+ * and the page cache holds the window as one 2 MiB folio. A link with
+ * `-z max-page-size=0x200000` keeps every segment's address congruent to its
+ * file offset modulo 2 MiB, so every 2 MiB window inside the text segment
+ * qualifies. lld and zig add no file padding for it; GNU ld pads the file to
+ * the alignment, so links that fall back to it leave the flag out.
+ *
+ * The alignment also aligns a position-independent executable's load
+ * address to 2 MiB, which leaves 19 bits of address randomization for it
+ * instead of 28. Only executables whose program code fills an interior 2 MiB
+ * window can gain from huge pages, so smaller executables link as before.
+ *
+ * lld writes its output through a memory mapping, which leaves the file in
+ * the page cache as 4 KiB folios. A large executable is published by writing
+ * its bytes with one write instead, which leaves 2 MiB folios where the file
+ * system supports them. The runtime advises the interior windows huge pages
+ * at startup (scr_lib.c), so a file that is read back from disk is read as
+ * 2 MiB folios too.
+ */
+import {
+  chmodSync,
+  closeSync,
+  openSync,
+  readFileSync,
+  readSync,
+  renameSync,
+  statSync,
+  writeFileSync,
+} from "node:fs";
+import type { NativeTargetSpec } from "./targets.js";
+
+/** Program code from which an executable is linked for huge pages: text of
+ * this size has at least one 2 MiB window inside it wherever it starts. */
+export const HUGE_TEXT_MIN_CODE_BYTES = 4 << 20;
+
+const SHF_EXECINSTR = 0x4n;
+
+/** The bytes of executable sections in an ELF64 little-endian object, or 0
+ * when the file is not one or cannot be read. Reads only the file and
+ * section headers. */
+export function elfObjectCodeBytes(path: string): number {
+  let fd: number;
+  try {
+    fd = openSync(path, "r");
+  } catch {
+    return 0;
+  }
+  try {
+    const header = Buffer.alloc(64);
+    if (readSync(fd, header, 0, 64, 0) !== 64) return 0;
+    const view = new DataView(header.buffer, header.byteOffset, header.byteLength);
+    if (view.getUint32(0, false) !== 0x7f454c46 || header[4] !== 2 || header[5] !== 1) return 0;
+    const shoff = Number(view.getBigUint64(40, true));
+    const shentsize = view.getUint16(58, true);
+    let shnum = view.getUint16(60, true);
+    if (shentsize !== 64 || shoff === 0) return 0;
+    if (shnum === 0) {
+      const first = Buffer.alloc(64);
+      if (readSync(fd, first, 0, 64, shoff) !== 64) return 0;
+      shnum = Number(new DataView(first.buffer, first.byteOffset, 64).getBigUint64(32, true));
+    }
+    const table = Buffer.alloc(shnum * 64);
+    if (readSync(fd, table, 0, table.length, shoff) !== table.length) return 0;
+    const sections = new DataView(table.buffer, table.byteOffset, table.byteLength);
+    let code = 0;
+    for (let i = 0; i < shnum; i++) {
+      const at = i * 64;
+      // SHT_PROGBITS sections with SHF_EXECINSTR.
+      if (sections.getUint32(at + 4, true) !== 1) continue;
+      if ((sections.getBigUint64(at + 8, true) & SHF_EXECINSTR) === 0n) continue;
+      code += Number(sections.getBigUint64(at + 32, true));
+    }
+    return code;
+  } finally {
+    closeSync(fd);
+  }
+}
+
+/** Whether a link of these program objects is laid out for huge pages. */
+export function hugeTextProgram(
+  target: Pick<NativeTargetSpec, "platform">,
+  programObjects: readonly string[],
+): boolean {
+  if (target.platform !== "linux") return false;
+  let code = 0;
+  for (const object of programObjects) {
+    code += elfObjectCodeBytes(object);
+    if (code >= HUGE_TEXT_MIN_CODE_BYTES) return true;
+  }
+  return false;
+}
+
+/** The link arguments that align a huge-text executable's segments to
+ * 2 MiB, for links by lld or zig. */
+export function hugeTextLinkerArgs(hugeText: boolean, linksWithLldOrZig: boolean): string[] {
+  return hugeText && linksWithLldOrZig ? ["-Wl,-z,max-page-size=0x200000"] : [];
+}
+
+/** Rewrites a linked huge-text executable with one write, so the page cache
+ * holds it in 2 MiB folios; its caller then publishes it as usual. */
+export function rewriteHugeTextExecutable(linked: string): void {
+  const written = `${linked}.written`;
+  writeFileSync(written, readFileSync(linked));
+  chmodSync(written, statSync(linked).mode & 0o7777);
+  renameSync(written, linked);
+}

packages/compiler/src/backend/link-plan.ts

@@ -13,6 +13,7 @@ import type { NativeArtifactDependency } from "./native/contracts.js";
 import { needsDarwinDebugSymbols } from "./debug-symbols.js";
 import { loadRuntimePack, type RuntimePackSelection } from "./runtime-pack.js";
 import { executableLinkInputs } from "./link-plan-core.js";
+import { hugeTextProgram } from "./huge-text.js";
 import type { NativeTargetSpec, WindowsSubsystem } from "./targets.js";
 
 export interface NativeLinkPlan {
@@ -30,6 +31,8 @@ export interface NativeLinkPlan {
   darwinDebugSymbols?: boolean;
   /** Linked with `--static`: the linked executable is checked before use. */
   staticLink?: boolean;
+  /** Program code large enough for text on huge pages (huge-text.ts). */
+  hugeText?: boolean;
 }
 
 export async function createNativeLinkPlan(options: {
@@ -85,5 +88,11 @@ export async function createNativeLinkPlan(options: {
     programObjectDependencies: [...(options.programObjectDependencies ?? [])],
     runtimePack,
     ...(options.staticLink === true ? { staticLink: true } : {}),
+    ...(hugeTextProgram(options.target, [
+      options.programObject,
+      ...(options.programPartitions ?? []),
+    ])
+      ? { hugeText: true }
+      : {}),
   };
 }

packages/compiler/src/backend/linker.ts

@@ -24,6 +24,7 @@ import { type NativeArtifactDependency } from "./native/contracts.js";
 import { RuntimePackError, stageRuntimePackArtifacts } from "./runtime-pack.js";
 import type { NativeLinkPlan } from "./link-plan.js";
 import { inputsWithoutLld, lldLinkerArgs } from "./lld.js";
+import { hugeTextLinkerArgs, rewriteHugeTextExecutable } from "./huge-text.js";
 import { systemLibraryLinkerArgs } from "./link-plan-core.js";
 import {
   createDarwinDebugSymbols,
@@ -248,11 +249,13 @@ export async function linkNativeExecutable(
     stagedRoot = staged.root;
     // Linux links with lld when the default driver finds it.
     const lld = explicitLinker === undefined ? lldLinkerArgs(plan.target.platform, linker) : [];
+    const zig = explicitLinker === undefined && plan.target.defaultLinker === "zig";
     const linkArgs = (withLld: boolean): string[] => {
       const inputs = plan.inputs.map((input) => staged.replacements.get(input) ?? input);
       return [
         ...(explicitLinker === undefined ? plan.target.defaultLinkerArgs : []),
         ...(withLld ? lld : []),
+        ...hugeTextLinkerArgs(plan.hugeText === true, (withLld && lld.length !== 0) || zig),
         ...plan.driverFlags,
         ...mapArgs,
         ...(withLld ? inputs : inputsWithoutLld(plan.target.platform, inputs)),
@@ -298,6 +301,7 @@ export async function linkNativeExecutable(
     });
     const output = await stat(privateOut);
     if (!output.isFile() || output.size === 0) throw new Error("linker produced no executable");
+    if (plan.hugeText === true) rewriteHugeTextExecutable(privateOut);
     if (mapArgs.length !== 0) {
       const refusal = staticExecutableRefusal(plan.target, true, await readFile(linkMap, "utf8"));
       if (refusal !== null) throw new NativeCodegenError("SC3002", refusal);

packages/compiler/src/native/compiler.ts

@@ -75,6 +75,11 @@ import { splitLlvmLibraryProgram } from "../backend/llvm/split.js";
 import { prepareNativeExecutable, prepareNativeLibrary } from "./prepare.js";
 import { buildSanitizedRuntime, sanitizerDriver, sanitizerFlags } from "./sanitizer.js";
 import { inputsWithoutLld, lldLinkerArgs } from "../backend/lld.js";
+import {
+  hugeTextLinkerArgs,
+  hugeTextProgram,
+  rewriteHugeTextExecutable,
+} from "../backend/huge-text.js";
 import { openNativeExecutableCache } from "./executable-cache.js";
 import { LinkerJobs } from "./linker-job.js";
 import { nativeFileIdentity } from "./file-identity.js";
@@ -637,11 +642,15 @@ export class NativeCompiler {
         });
         // Linux links with lld when the driver finds it.
         const lld = options.sanitize ? [] : lldLinkerArgs(target.platform, toolchain.linker);
+        const hugeText =
+          !options.sanitize && hugeTextProgram(target, [object, ...programObjects.slice(1)]);
+        const zig = process.env["SCRIPTC_LINKER"] === undefined && target.defaultLinker === "zig";
         const linkMap = join(stage, "link.map");
         const mapArgs = staticLinkMapArgs(target, options.staticLink === true, linkMap);
         const linkArgsWith = (withLld: boolean): string[] => [
           ...(options.sanitize ? ["-fsanitize=address"] : toolchain.linkerArgs),
           ...(withLld ? lld : []),
+          ...hugeTextLinkerArgs(hugeText, (withLld && lld.length !== 0) || zig),
           ...plan.driverFlags,
           ...mapArgs,
           ...(withLld ? plan.inputs : inputsWithoutLld(target.platform, plan.inputs)),
@@ -679,6 +688,7 @@ export class NativeCompiler {
         }
         timing("native-link");
         requireNativeArtifact(stagedOutput);
+        if (hugeText) rewriteHugeTextExecutable(stagedOutput);
         if (mapArgs.length !== 0) {
           const refusal = staticExecutableRefusal(target, true, readFileSync(linkMap, "utf8"));
           if (refusal !== null) throw new NativeCodegenError("SC3002", refusal);

packages/runtime/src/scr_lib.c

@@ -2066,6 +2066,48 @@ bool scr_process_has_exit_code(void) {
   return scr_exit_code_assigned || scr_exit_code_hint_get() != 0;
 }
 
+#if defined(__linux__) && !defined(SCR_LIB)
+#include <link.h>
+#include <sys/mman.h>
+/* Text on huge pages (huge-text.ts). An executable with enough program code
+ * is linked with 2 MiB segment alignment, so each 2 MiB window inside its
+ * text segment starts at a 2 MiB file offset, and the kernel maps a window
+ * with one huge page when the page cache holds it as one 2 MiB folio.
+ * MADV_HUGEPAGE on those windows makes the kernel read a window that is not
+ * cached as a 2 MiB folio. glibc runs it from .preinit_array, before _init
+ * and the constructors fault in any of the text; musl, which has no
+ * .preinit_array, runs it as a constructor. Executables with 4 KiB alignment
+ * make no call. */
+#define SCR_TEXT_HUGE_PAGE ((uintptr_t)2 << 20)
+
+static int scr_text_segment(struct dl_phdr_info *info, size_t size, void *data) {
+  (void)size;
+  uintptr_t *range = data;
+  for (size_t i = 0; i < info->dlpi_phnum; i++) {
+    const ElfW(Phdr) *ph = &info->dlpi_phdr[i];
+    if (ph->p_type != PT_LOAD || !(ph->p_flags & PF_X) || ph->p_align < SCR_TEXT_HUGE_PAGE) continue;
+    range[0] = (uintptr_t)info->dlpi_addr + (uintptr_t)ph->p_vaddr;
+    range[1] = range[0] + (uintptr_t)ph->p_memsz;
+    break;
+  }
+  return 1; /* the first object is the executable */
+}
+
+static bool scr_text_huge_pages_done;
+
+__attribute__((constructor)) static void scr_text_huge_pages(void) {
+  if (scr_text_huge_pages_done) return;
+  scr_text_huge_pages_done = true;
+  uintptr_t range[2] = {0, 0};
+  (void)dl_iterate_phdr(scr_text_segment, range);
+  uintptr_t lo = (range[0] + SCR_TEXT_HUGE_PAGE - 1) & ~(SCR_TEXT_HUGE_PAGE - 1);
+  uintptr_t hi = range[1] & ~(SCR_TEXT_HUGE_PAGE - 1);
+  if (hi > lo) (void)madvise((void *)lo, hi - lo, MADV_HUGEPAGE);
+}
+__attribute__((section(".preinit_array"), used)) static void (*const scr_text_huge_pages_entry)(void) =
+    scr_text_huge_pages;
+#endif
+
 #if defined(__linux__)
 #include <sched.h>
 #include <signal.h>