Localize merged program shards in process on Linux (#2310)

This commit is contained in:
Chris Tate
2026-10-26 00:31:57 -05:00
committed by GitHub
7 changed files with 215 additions and 39 deletions
@@ -166,6 +166,7 @@ export async function compilePackedLibrary(
options.programPublicSymbols,
stem,
target.platform,
{ inProcessElf: true },
);
} else {
program = await emit(
@@ -1519,6 +1519,8 @@ export async function compileCInternal(
[],
programPublicSymbols,
`${stem}.program`,
undefined,
{ inProcessElf: true },
);
const publishable = entries.filter((entry) => entry.missed);
if (publishable.length > 0) {
@@ -517,6 +517,8 @@ export async function compileLibArchive(opts: LibArchiveOptions): Promise<void>
[],
programPublicSymbols!,
`${stem}.program`,
undefined,
{ inProcessElf: true },
);
// Keep the canonical archive member spelling. The fact that native
// compilation used shards is an implementation detail; consumers and
@@ -5,6 +5,17 @@ import { localizeElfObject, mergeAndLocalizeCoffObjects } from "../object-locali
import { type CcDriver, targetPlatform } from "./driver.js";
import { execFileAsync, subprocessFailureDetail } from "./process.js";
/* Demotes in process (localizeElfObject), rewriting the object in place. */
async function localizeInProcess(object: string, keepSymbols: readonly string[]): Promise<void> {
try {
await writeFile(object, localizeElfObject(await readFile(object), new Set(keepSymbols)));
} catch (err) {
throw new Error(
`ELF symbol localization failed while localizing the library archive's runtime symbols (abi.localize_runtime).\n\n${(err as Error).message}`,
);
}
}
/* Multi-instance library mode's localization step: combine the program
* object with exactly the runtime/vendor members it reaches into ONE
* relocatable object, then demote every scriptc external definition except
@@ -46,7 +57,11 @@ import { execFileAsync, subprocessFailureDetail } from "./process.js";
* multi-archive link from discarding one archive's copies),
* then binutils objcopy --keep-global-symbols localizes every
* other DEFINED global (objcopy leaves undefined symbols
* global by its own rule). The exact historical recipe.
* global by its own rule). The exact historical recipe. With
* `inProcessElf` (program-shard merges), localizeElfObject
* demotes instead of objcopy: a sanitized dev program merges
* into an object with hundreds of thousands of sections, which
* objcopy takes minutes over and localizeElfObject seconds.
* ELF, cross triples from any host — `zig cc -target <triple> -r`
* merges (zig is already the cross driver's hard requirement),
* then localizeElfObject demotes in process and resolves
@@ -68,6 +83,7 @@ export async function localizeLibraryObjects(
keepSymbols: readonly string[],
stem: string,
platform = targetPlatform(driver),
options: { inProcessElf?: boolean } = {},
): Promise<string> {
const combined = join(buildDir, `${stem}.localized.o`);
const staging = join(buildDir, `${stem}.localize-staging.a`);
@@ -79,7 +95,7 @@ export async function localizeLibraryObjects(
const stderr = subprocessFailureDetail(err);
throw new Error(
`${argv[0]} failed while localizing the library archive's runtime symbols (abi.localize_runtime).\n` +
`Runtime symbol localization needs the ${platform === "darwin" ? "host toolchain's ld" : driver.target === null ? "host toolchain's ld and objcopy" : "cross driver's relocatable link"} beside the C compiler.\n\n${stderr}`,
`Runtime symbol localization needs the ${platform === "darwin" || (driver.target === null && options.inProcessElf === true) ? "host toolchain's ld" : driver.target === null ? "host toolchain's ld and objcopy" : "cross driver's relocatable link"} beside the C compiler.\n\n${stderr}`,
);
}
};
@@ -122,7 +138,6 @@ export async function localizeLibraryObjects(
keepFile,
]);
} else if (platform === "linux" && driver.target === null) {
await writeFile(keepFile, keepSymbols.map((s) => `${s}\n`).join(""));
await run([
"ld",
"-r",
@@ -132,7 +147,12 @@ export async function localizeLibraryObjects(
"-o",
combined,
]);
await run(["objcopy", `--keep-global-symbols=${keepFile}`, combined]);
if (options.inProcessElf === true) {
await localizeInProcess(combined, keepSymbols);
} else {
await writeFile(keepFile, keepSymbols.map((s) => `${s}\n`).join(""));
await run(["objcopy", `--keep-global-symbols=${keepFile}`, combined]);
}
} else if (platform === "linux") {
// Cross ELF: the cross driver's own lld performs the relocatable merge
// (with the staging archive's member semantics); demotion and section-
@@ -152,13 +172,7 @@ export async function localizeLibraryObjects(
"-o",
combined,
]);
try {
await writeFile(combined, localizeElfObject(await readFile(combined), new Set(keepSymbols)));
} catch (err) {
throw new Error(
`ELF symbol localization failed while localizing the library archive's runtime symbols (abi.localize_runtime).\n\n${(err as Error).message}`,
);
}
await localizeInProcess(combined, keepSymbols);
} else {
throw new InternalCompilerError(
`runtime symbol localization (abi.localize_runtime) has no ${platform} arm; compileLibrary admits darwin, linux, and win32 builds only`,
@@ -52,13 +52,13 @@ export async function resolveProgramShardMergeIdentity(driver: CcDriver): Promis
}
if (platform === "linux") {
if (driver.target === null) {
const [ld, objcopy] = await Promise.all([resolvedTool("ld"), resolvedTool("objcopy")]);
return ld === null || objcopy === null
// ld -r merges; localizeElfObject demotes in process.
const ld = await resolvedTool("ld");
return ld === null
? null
: rememberFingerprintDependencies(
`program-shard-merge-linux-v1\0${ld.cacheIdentity}\0${objcopy.cacheIdentity}`,
[ld.canonicalPath, objcopy.canonicalPath],
);
: rememberFingerprintDependencies(`program-shard-merge-linux-v2\0${ld.cacheIdentity}`, [
ld.canonicalPath,
]);
}
const arch = driver.target.split("-", 1)[0];
if (arch !== "x86_64" && arch !== "aarch64") return null;
@@ -28,7 +28,10 @@ const SHF_EXECINSTR = 0x4n;
const SHF_GROUP = 0x200n;
const STB_GLOBAL = 1;
const STB_WEAK = 2;
const SHT_SYMTAB_SHNDX = 18;
const SHN_LORESERVE = 0xff00;
const SHN_COMMON = 0xfff2;
const SHN_XINDEX = 0xffff;
interface ElfSymbolSpec {
name: string;
@@ -53,7 +56,11 @@ function buildStrtab(names: string[]): { data: Uint8Array; offsets: Map<string,
/** Assemble an ET_REL ELF64LE: caller supplies content sections; the
* builder appends .symtab, .strtab, .shstrtab (their indices are
* sections.length, +1, +2). Symbols must list locals first. */
* sections.length, +1, +2). Symbols must list locals first. With
* SHN_LORESERVE or more sections the builder uses extended section
* numbering, appending .symtab_shndx (index sections.length + 3) for symbol
* section indices from SHN_LORESERVE up; reserved symbol indices are then
* limited to SHN_COMMON. */
function buildElf(
sections: ElfSectionSpec[],
symbols: ElfSymbolSpec[],
@@ -62,11 +69,19 @@ function buildElf(
const strtab = buildStrtab(symbols.map((s) => s.name).filter((n) => n !== ""));
const symData = new Uint8Array((symbols.length + 1) * 24);
const symView = new DataView(symData.buffer);
const shndxData = new Uint8Array((symbols.length + 1) * 4);
let usesShndx = false;
symbols.forEach((sym, i) => {
const at = (i + 1) * 24;
symView.setUint32(at, sym.name === "" ? 0 : strtab.offsets.get(sym.name)!, true);
symView.setUint8(at + 4, (sym.binding << 4) | 0);
symView.setUint16(at + 6, sym.shndx, true);
if (sym.shndx >= SHN_LORESERVE && sym.shndx !== SHN_COMMON) {
symView.setUint16(at + 6, SHN_XINDEX, true);
new DataView(shndxData.buffer).setUint32((i + 1) * 4, sym.shndx, true);
usesShndx = true;
} else {
symView.setUint16(at + 6, sym.shndx, true);
}
symView.setBigUint64(at + 8, sym.value ?? 0n, true);
});
const all: ElfSectionSpec[] = [
@@ -82,9 +97,22 @@ function buildElf(
},
{ name: ".strtab", type: SHT_STRTAB, data: strtab.data },
{ name: ".shstrtab", type: SHT_STRTAB },
...(usesShndx
? [
{
name: ".symtab_shndx",
type: SHT_SYMTAB_SHNDX,
data: shndxData,
link: sections.length + 1,
entsize: 4n,
},
]
: []),
];
const shstrndx = sections.length + 3;
const extended = all.length >= SHN_LORESERVE;
const shstr = buildStrtab(all.map((s) => s.name).filter((n) => n !== ""));
all[all.length - 1]!.data = shstr.data;
all[shstrndx]!.data = shstr.data;
let offset = 64;
const offsets = all.map((s) => {
const data = s.data ?? new Uint8Array(0);
@@ -106,8 +134,8 @@ function buildElf(
view.setBigUint64(40, BigInt(shoff), true);
view.setUint16(52, 64, true);
view.setUint16(58, 64, true);
view.setUint16(60, all.length, true);
view.setUint16(62, all.length - 1, true);
view.setUint16(60, extended ? 0 : all.length, true);
view.setUint16(62, shstrndx >= SHN_LORESERVE ? SHN_XINDEX : shstrndx, true);
all.forEach((s, i) => {
const at = shoff + i * 64;
const data = s.data ?? new Uint8Array(0);
@@ -116,8 +144,8 @@ function buildElf(
view.setUint32(at + 4, s.type, true);
view.setBigUint64(at + 8, s.flags ?? 0n, true);
view.setBigUint64(at + 24, BigInt(s.type === 0 ? 0 : offsets[i]!), true);
view.setBigUint64(at + 32, BigInt(data.length), true);
view.setUint32(at + 40, s.link ?? 0, true);
view.setBigUint64(at + 32, BigInt(i === 0 && extended ? all.length : data.length), true);
view.setUint32(at + 40, i === 0 && shstrndx >= SHN_LORESERVE ? shstrndx : (s.link ?? 0), true);
view.setUint32(at + 44, s.info ?? 0, true);
view.setBigUint64(at + 48, s.addralign ?? 1n, true);
view.setBigUint64(at + 56, s.entsize ?? 0n, true);
@@ -142,8 +170,10 @@ interface ReadElf {
function readElf(bytes: Uint8Array): ReadElf {
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
const shoff = Number(view.getBigUint64(40, true));
const shnum = view.getUint16(60, true);
const shstrndx = view.getUint16(62, true);
let shnum = view.getUint16(60, true);
let shstrndx = view.getUint16(62, true);
if (shnum === 0) shnum = Number(view.getBigUint64(shoff + 32, true));
if (shstrndx === SHN_XINDEX) shstrndx = view.getUint32(shoff + 40, true);
const raw = [] as {
nameOff: number;
type: number;
@@ -182,12 +212,16 @@ function readElf(bytes: Uint8Array): ReadElf {
const symtabIdx = raw.findIndex((s) => s.type === SHT_SYMTAB);
const symtab = raw[symtabIdx]!;
const strtab = raw[symtab.link]!;
const shndxTable = raw.find((s) => s.type === SHT_SYMTAB_SHNDX && s.link === symtabIdx);
const symbols = [] as ReadElf["symbols"];
for (let at = symtab.offset; at < symtab.offset + symtab.size; at += 24) {
let shndx = view.getUint16(at + 6, true);
if (shndx === SHN_XINDEX)
shndx = view.getUint32(shndxTable!.offset + ((at - symtab.offset) / 24) * 4, true);
symbols.push({
name: name(strtab, view.getUint32(at, true)),
binding: view.getUint8(at + 4) >> 4,
shndx: view.getUint16(at + 6, true),
shndx,
});
}
return { sections, symbols, symtabInfo: symtab.info };
@@ -241,6 +275,65 @@ describe("localizeElfObject", () => {
expect(info & 0xffffffffn).toBe(2n);
});
test("keeps extended section numbering when the object has SHN_LORESERVE or more sections", () => {
// A sanitized dev program merges into an object with more sections than
// e_shnum and st_shndx can index: section 0 carries the count and the
// section-name table's index, and .symtab_shndx the symbols' indices.
const count = SHN_LORESERVE + 400;
const texts: ElfSectionSpec[] = Array.from({ length: count }, (_, i) => ({
name: `.text.f${i}`,
type: SHT_PROGBITS,
flags: SHF_ALLOC | SHF_EXECINSTR,
data: new Uint8Array(4),
}));
const rela = new Uint8Array(24);
new DataView(rela.buffer).setBigUint64(8, (2n << 32n) | 2n, true); // sym 2 (high)
const object = buildElf(
[
...texts,
{
name: ".rela.text.last",
type: SHT_RELA,
data: rela,
link: count + 2,
info: count,
entsize: 24n,
},
],
[
{ name: "low", binding: STB_GLOBAL, shndx: 1 },
{ name: "high", binding: STB_GLOBAL, shndx: count },
{ name: "main", binding: STB_GLOBAL, shndx: count - 1 },
{ name: "libc_ref", binding: STB_GLOBAL, shndx: 0 },
{ name: "shared_common", binding: STB_GLOBAL, shndx: SHN_COMMON, value: 8n },
],
0,
);
const output = localizeElfObject(object, new Set(["main"]));
const header = new DataView(output.buffer, output.byteOffset, output.byteLength);
expect(header.getUint16(60, true)).toBe(0);
expect(header.getUint16(62, true)).toBe(SHN_XINDEX);
const localized = readElf(output);
expect(localized.sections.length).toBe(count + 6);
expect(localized.sections.filter((s) => s.type === SHT_SYMTAB_SHNDX)).toHaveLength(1);
const byName = new Map(localized.symbols.map((s) => [s.name, s]));
expect(byName.get("low")!.binding).toBe(0);
expect(byName.get("high")!.binding).toBe(0);
expect(localized.sections[byName.get("high")!.shndx]!.name).toBe(`.text.f${count - 1}`);
expect(byName.get("main")!.binding).toBe(STB_GLOBAL);
expect(localized.sections[byName.get("main")!.shndx]!.name).toBe(`.text.f${count - 2}`);
expect(byName.get("libc_ref")!.shndx).toBe(0);
expect(byName.get("shared_common")!.shndx).toBe(SHN_COMMON);
const relaOut = localized.sections.find((s) => s.name === ".rela.text.last")!;
expect(localized.sections[relaOut.info]!.name).toBe(`.text.f${count - 1}`);
const info = new DataView(relaOut.data.buffer, relaOut.data.byteOffset).getBigUint64(8, true);
expect(localized.symbols[Number(info >> 32n)]!.name).toBe("high");
// The output reads back as the same object.
expect(readElf(localizeElfObject(output, new Set(["main"]))).symbols).toEqual(
localized.symbols,
);
});
test("resolves section groups: SHT_GROUP dropped, SHF_GROUP cleared, members and indices intact", () => {
const group = new Uint8Array(8);
const groupView = new DataView(group.buffer);
@@ -84,9 +84,13 @@ interface ElfSymbol {
name: string;
info: number;
other: number;
/** Resolved section index (SHT_SYMTAB_SHNDX applied), or a reserved
* SHN_* value below/at SHN_LORESERVE semantics. */
/** Section index (SHT_SYMTAB_SHNDX applied), or the reserved SHN_* value
* when `reserved`. */
shndx: number;
/** st_shndx holds a reserved value (SHN_ABS, SHN_COMMON, ...), not a
* section index. An index from SHT_SYMTAB_SHNDX can itself be at or above
* SHN_LORESERVE. */
reserved: boolean;
}
/** Demote every defined global/weak symbol not named in `keep` to a local
@@ -143,8 +147,8 @@ export function localizeElfObject(object: Uint8Array, keep: ReadonlySet<string>)
if (strtab === undefined) fail("ELF symbol table names an invalid string table");
// SHT_SYMTAB_SHNDX (only present when section indices overflow st_shndx):
// resolve the real indices up front and drop the section — the rebuilt
// object's section count only shrinks, so inline indices always fit.
// resolve the real indices up front and drop the section. The rebuilt
// object writes a new one when its own section count still overflows.
const shndxSection = sections.find(
(s) => s.type === SHT_SYMTAB_SHNDX && sections[s.link] === symtab,
);
@@ -170,9 +174,11 @@ export function localizeElfObject(object: Uint8Array, keep: ReadonlySet<string>)
for (let i = 0; i < symCount; i++) {
const at = i * 24;
let shndx: number = symView.getUint16(at + 6, true);
let reserved = shndx >= SHN_LORESERVE;
if (shndx === SHN_XINDEX) {
if (shndxView === null) fail("ELF symbol uses SHN_XINDEX without a SHT_SYMTAB_SHNDX table");
shndx = shndxView.getUint32(i * 4, true);
reserved = false;
}
symbols.push({
nameOffset: symView.getUint32(at, true),
@@ -180,6 +186,7 @@ export function localizeElfObject(object: Uint8Array, keep: ReadonlySet<string>)
info: symView.getUint8(at + 4),
other: symView.getUint8(at + 5),
shndx,
reserved,
});
}
@@ -212,11 +219,11 @@ export function localizeElfObject(object: Uint8Array, keep: ReadonlySet<string>)
const symbolKept: boolean[] = [];
for (let i = 0; i < symCount; i++) {
const sym = symbols[i]!;
const regular = sym.shndx > SHN_UNDEF && sym.shndx < SHN_LORESERVE;
const regular = !sym.reserved && sym.shndx !== SHN_UNDEF;
symbolKept.push(!(regular && sectionKept[sym.shndx] !== true));
if (i === 0) continue;
const binding = sym.info >> 4;
const defined = sym.shndx !== SHN_UNDEF && sym.shndx !== SHN_COMMON;
const defined = sym.shndx !== SHN_UNDEF && !(sym.reserved && sym.shndx === SHN_COMMON);
if (binding !== STB_LOCAL && defined && !keep.has(sym.name)) {
sym.info = (STB_LOCAL << 4) | (sym.info & 0xf);
}
@@ -236,6 +243,16 @@ export function localizeElfObject(object: Uint8Array, keep: ReadonlySet<string>)
symbolMap[oldIndex] = newIndex;
});
// Extended section numbering: with SHN_LORESERVE or more sections, an
// index from SHN_LORESERVE up does not fit st_shndx, e_shnum or
// e_shstrndx. Such a symbol's st_shndx is SHN_XINDEX and its index goes in
// a SHT_SYMTAB_SHNDX section (one entry per symbol, appended after the
// kept sections); the counts go in section 0 (sh_size, sh_link).
let keptCount = 0;
for (const keptSection of sectionKept) if (keptSection) keptCount++;
const extended = keptCount >= SHN_LORESERVE;
const shndxOut = extended ? new Uint8Array(symbolOrder.length * 4) : null;
const shndxOutView = shndxOut === null ? null : new DataView(shndxOut.buffer);
const newSymtabData = new Uint8Array(symbolOrder.length * 24);
const newSymtabView = new DataView(newSymtabData.buffer);
symbolOrder.forEach((oldIndex, newIndex) => {
@@ -244,9 +261,14 @@ export function localizeElfObject(object: Uint8Array, keep: ReadonlySet<string>)
newSymtabView.setUint32(at, sym.nameOffset, true);
newSymtabView.setUint8(at + 4, sym.info);
newSymtabView.setUint8(at + 5, sym.other);
const shndx =
sym.shndx > SHN_UNDEF && sym.shndx < SHN_LORESERVE ? sectionMap[sym.shndx]! : sym.shndx;
newSymtabView.setUint16(at + 6, shndx, true);
const regular = !sym.reserved && sym.shndx !== SHN_UNDEF;
const shndx = regular ? sectionMap[sym.shndx]! : sym.shndx;
if (regular && shndx >= SHN_LORESERVE) {
newSymtabView.setUint16(at + 6, SHN_XINDEX, true);
shndxOutView!.setUint32(newIndex * 4, shndx, true);
} else {
newSymtabView.setUint16(at + 6, shndx, true);
}
newSymtabView.setBigUint64(at + 8, symView.getBigUint64(oldIndex * 24 + 8, true), true);
newSymtabView.setBigUint64(at + 16, symView.getBigUint64(oldIndex * 24 + 16, true), true);
});
@@ -278,6 +300,33 @@ export function localizeElfObject(object: Uint8Array, keep: ReadonlySet<string>)
// Rebuild. sh_link is a section index in every use this object can carry;
// sh_info is one only on relocation sections.
const kept = sections.filter((_, i) => sectionKept[i] === true);
const newSymtabIndex = sectionMap[symtabIndex]!;
if (shndxOut !== null) {
// The extended index table goes last, named in the section-name table.
const shstrtab = sections[shstrndx]!;
let nameOffset = shndxSection?.nameOffset;
if (nameOffset === undefined) {
const name = textEncoder.encode(".symtab_shndx\0");
const grown = new Uint8Array(shstrtab.data.length + name.length);
grown.set(shstrtab.data);
grown.set(name, shstrtab.data.length);
nameOffset = shstrtab.data.length;
shstrtab.data = grown;
}
kept.push({
nameOffset,
type: SHT_SYMTAB_SHNDX,
flags: 0n,
addr: 0n,
size: BigInt(shndxOut.length),
link: symtabIndex,
info: 0,
addralign: 4n,
entsize: 4n,
data: shndxOut,
});
}
const newShstrndx = sectionMap[shstrndx]!;
let offset = 64;
const offsets: number[] = [];
for (const section of kept) {
@@ -301,8 +350,8 @@ export function localizeElfObject(object: Uint8Array, keep: ReadonlySet<string>)
outView.setBigUint64(40, BigInt(shoffOut), true);
outView.setUint16(54, 0, true); // e_phentsize
outView.setUint16(56, 0, true); // e_phnum
outView.setUint16(60, kept.length, true);
outView.setUint16(62, sectionMap[shstrndx]!, true);
outView.setUint16(60, extended ? 0 : kept.length, true);
outView.setUint16(62, newShstrndx >= SHN_LORESERVE ? SHN_XINDEX : newShstrndx, true);
kept.forEach((section, i) => {
if (section.type !== SHT_NULL && section.type !== SHT_NOBITS) {
out.set(section.data, offsets[i]!);
@@ -313,12 +362,27 @@ export function localizeElfObject(object: Uint8Array, keep: ReadonlySet<string>)
outView.setBigUint64(at + 8, section.flags, true);
outView.setBigUint64(at + 16, section.addr, true);
outView.setBigUint64(at + 24, BigInt(offsets[i]!), true);
if (i === 0) {
// Section 0 carries the counts that overflow the ELF header.
outView.setBigUint64(at + 32, extended ? BigInt(kept.length) : 0n, true);
outView.setUint32(at + 40, newShstrndx >= SHN_LORESERVE ? newShstrndx : 0, true);
outView.setUint32(at + 44, 0, true);
return;
}
outView.setBigUint64(
at + 32,
section.type === SHT_NOBITS ? section.size : BigInt(section.data.length),
true,
);
outView.setUint32(at + 40, section.link === 0 ? 0 : (sectionMap[section.link] ?? 0), true);
outView.setUint32(
at + 40,
section.type === SHT_SYMTAB_SHNDX
? newSymtabIndex
: section.link === 0
? 0
: (sectionMap[section.link] ?? 0),
true,
);
outView.setUint32(
at + 44,
section.type === SHT_RELA || section.type === SHT_REL