From 6b2d31ce173618bcbeb5b6da386702cc2bc41b1e Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Wed, 19 Aug 2026 14:59:25 -0500 Subject: [PATCH] Preserve library identity during localization --- .../compiler/src/backend/cc-cache.test.ts | 63 +++++++++++++++++ packages/compiler/src/backend/cc.ts | 28 ++++---- .../src/backend/object-localize.test.ts | 67 +++++++++++++++---- .../compiler/src/backend/object-localize.ts | 35 +++++----- 4 files changed, 150 insertions(+), 43 deletions(-) diff --git a/packages/compiler/src/backend/cc-cache.test.ts b/packages/compiler/src/backend/cc-cache.test.ts index 1478bc914..ca6701cc3 100644 --- a/packages/compiler/src/backend/cc-cache.test.ts +++ b/packages/compiler/src/backend/cc-cache.test.ts @@ -2991,3 +2991,66 @@ test("library identity edits reuse the cached large program object", async () => else process.env["SCRIPTC_NO_CACHE"] = oldNoCache; } }); + +test.skipIf(process.platform === "win32" || zigExecutable === undefined)( + "cross-ELF localized archives retain unreferenced identity roots", + async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-lib-localized-identity-")); + scratch.push(dir); + const cPath = join(dir, "program.c"); + const identityCPath = join(dir, "identity.c"); + const probePath = join(dir, "probe.c"); + const archivePath = join(dir, "program.lib.a"); + const probeOutput = join(dir, "probe"); + const cacheRoot = join(dir, "cache"); + const target = "x86_64-linux-gnu.2.36"; + const oldCacheDir = process.env["SCRIPTC_CACHE_DIR"]; + const oldNoCache = process.env["SCRIPTC_NO_CACHE"]; + const oldCc = process.env["SCRIPTC_CC"]; + const oldTarget = process.env["SCRIPTC_TARGET"]; + try { + process.env["SCRIPTC_CACHE_DIR"] = cacheRoot; + delete process.env["SCRIPTC_NO_CACHE"]; + process.env["SCRIPTC_CC"] = "zigcc"; + process.env["SCRIPTC_TARGET"] = target; + await writeFile(cPath, "int scriptc_large_program_value(void) { return 7; }\n"); + await writeFile(identityCPath, [ + "unsigned long long scriptc_build_id(void) { return 2; }", + "unsigned scriptc_abi_version(void) { return 1; }", + "", + ].join("\n")); + await compileLibArchive({ + cPath, + identityCPath, + outPath: archivePath, + cacheIdentity: TEST_CACHE_IDENTITY, + localizeSymbols: [ + "scriptc_large_program_value", + "scriptc_build_id", + "scriptc_abi_version", + ], + }); + await writeFile(probePath, [ + "int scriptc_large_program_value(void);", + "unsigned long long scriptc_build_id(void);", + "unsigned scriptc_abi_version(void);", + "int main(void) {", + " return scriptc_large_program_value() != 7 || scriptc_build_id() != 2 || scriptc_abi_version() != 1;", + "}", + "", + ].join("\n")); + execFileSync(zigExecutable!, [ + "cc", "-target", target, probePath, archivePath, "-lm", "-o", probeOutput, + ]); + } finally { + if (oldCacheDir === undefined) delete process.env["SCRIPTC_CACHE_DIR"]; + else process.env["SCRIPTC_CACHE_DIR"] = oldCacheDir; + if (oldNoCache === undefined) delete process.env["SCRIPTC_NO_CACHE"]; + else process.env["SCRIPTC_NO_CACHE"] = oldNoCache; + if (oldCc === undefined) delete process.env["SCRIPTC_CC"]; + else process.env["SCRIPTC_CC"] = oldCc; + if (oldTarget === undefined) delete process.env["SCRIPTC_TARGET"]; + else process.env["SCRIPTC_TARGET"] = oldTarget; + } + }, +); diff --git a/packages/compiler/src/backend/cc.ts b/packages/compiler/src/backend/cc.ts index 5c027dfac..c67d000ed 100644 --- a/packages/compiler/src/backend/cc.ts +++ b/packages/compiler/src/backend/cc.ts @@ -1844,8 +1844,8 @@ export async function compileLibArchive(opts: LibArchiveOptions): Promise driver, arArgv, buildDir, - programObject, - [...(identityObject === null ? [] : [identityObject]), ...runtimeObjects, ...lreObjects, ...zlibObjects], + [programObject, ...(identityObject === null ? [] : [identityObject])], + [...runtimeObjects, ...lreObjects, ...zlibObjects], opts.localizeSymbols, stem, ), @@ -1941,7 +1941,8 @@ export async function compileLibArchive(opts: LibArchiveOptions): Promise * shared by design. Windows embedders additionally link advapi32, iphlpapi, * and ws2_32. * - * Member selection matters: a classic archive's unused members (and their + * The generated program and optional identity objects are mandatory roots; + * support-member selection still matters. A classic archive's unused members (and their * undefined references to units library mode excludes, like the * fs-promises unit's fiber symbols) never reach an embedder's link. A * blind merge of every object would carry those references into the one @@ -1950,7 +1951,7 @@ export async function compileLibArchive(opts: LibArchiveOptions): Promise * members the program object transitively needs (the COFF arm implements * the same member semantics in process). * - * Mach-O — one host-ld64 invocation: -r merges program + needed members, + * Mach-O — one host-ld64 invocation: -r merges roots + needed members, * -exported_symbols_list demotes every unlisted global to * private extern, and -r without -keep_private_externs writes * private externs out as non-external symbols. Apple ASan's @@ -1985,7 +1986,7 @@ async function localizeLibraryObjects( driver: CcDriver, arArgv: readonly string[], buildDir: string, - programObject: string, + rootObjects: readonly string[], supportObjects: readonly string[], keepSymbols: readonly string[], stem: string, @@ -2008,14 +2009,15 @@ async function localizeLibraryObjects( if (platform === "win32") { // COFF has no relocatable-link tool to stage through; the member // selection and combine+demote happen in process over the object bytes. - const [program, ...support] = await Promise.all( - [programObject, ...supportObjects].map((path) => readFile(path)), - ); + const [roots, support] = await Promise.all([ + Promise.all(rootObjects.map((path) => readFile(path))), + Promise.all(supportObjects.map((path) => readFile(path))), + ]); try { await writeFile( combined, - mergeAndLocalizeCoffObjects(program!, support, new Set(keepSymbols), { - program: basename(programObject), + mergeAndLocalizeCoffObjects(roots, support, new Set(keepSymbols), { + roots: rootObjects.map((path) => basename(path)), support: supportObjects.map((path) => basename(path)), }), ); @@ -2029,10 +2031,10 @@ async function localizeLibraryObjects( await run([arArgv[0] ?? "ar", ...arArgv.slice(1), "rcs", staging, ...supportObjects]); if (platform === "darwin") { await writeFile(keepFile, keepSymbols.map((s) => `_${s}\n`).join("")); - await run(["ld", "-r", programObject, staging, "-o", combined, "-exported_symbols_list", keepFile]); + await run(["ld", "-r", ...rootObjects, staging, "-o", combined, "-exported_symbols_list", keepFile]); } else if (platform === "linux" && driver.target === null) { await writeFile(keepFile, keepSymbols.map((s) => `${s}\n`).join("")); - await run(["ld", "-r", "--force-group-allocation", programObject, staging, "-o", combined]); + await run(["ld", "-r", "--force-group-allocation", ...rootObjects, staging, "-o", combined]); await run(["objcopy", `--keep-global-symbols=${keepFile}`, combined]); } else if (platform === "linux") { // Cross ELF: the cross driver's own lld performs the relocatable merge @@ -2046,7 +2048,7 @@ async function localizeLibraryObjects( ...driver.argv.slice(1), "-target", driver.zigTarget ?? driver.target!, "-nostdlib", - "-r", programObject, staging, + "-r", ...rootObjects, staging, "-o", combined, ]); try { diff --git a/packages/compiler/src/backend/object-localize.test.ts b/packages/compiler/src/backend/object-localize.test.ts index 7ec74fd8b..e5321bee0 100644 --- a/packages/compiler/src/backend/object-localize.test.ts +++ b/packages/compiler/src/backend/object-localize.test.ts @@ -431,7 +431,7 @@ describe("mergeAndLocalizeCoffObjects", () => { ], ); const merged = readCoff( - mergeAndLocalizeCoffObjects(program, [needed, unneeded], new Set(["keep_me"])), + mergeAndLocalizeCoffObjects([program], [needed, unneeded], new Set(["keep_me"])), ); const byName = new Map(merged.symbols.map((s) => [s.name, s])); expect(byName.get("keep_me")!.storageClass).toBe(IMAGE_SYM_CLASS_EXTERNAL); @@ -446,6 +446,47 @@ describe("mergeAndLocalizeCoffObjects", () => { expect(merged.sections[0]!.relocs[0]!.sym).toBe(byName.get("helper")!.index); }); + test("includes every root even when no other object references its exports", () => { + const program = buildCoff( + [text()], + [ + { name: ".text", section: 1, storageClass: IMAGE_SYM_CLASS_STATIC, sectionDef: true }, + { name: "keep_me", section: 1 }, + ], + ); + const identity = buildCoff( + [text()], + [ + { name: ".text", section: 1, storageClass: IMAGE_SYM_CLASS_STATIC, sectionDef: true }, + { name: "build_id", section: 1 }, + { name: "abi_version", section: 1, value: 4 }, + ], + ); + const unneeded = buildCoff( + [text()], + [ + { name: ".text", section: 1, storageClass: IMAGE_SYM_CLASS_STATIC, sectionDef: true }, + { name: "lonely", section: 1 }, + { name: "excluded_unit_ref", section: 0 }, + ], + ); + + const merged = readCoff( + mergeAndLocalizeCoffObjects( + [program, identity], + [unneeded], + new Set(["keep_me", "build_id", "abi_version"]), + ), + ); + const byName = new Map(merged.symbols.map((symbol) => [symbol.name, symbol])); + for (const name of ["keep_me", "build_id", "abi_version"]) { + expect(byName.get(name)?.storageClass).toBe(IMAGE_SYM_CLASS_EXTERNAL); + expect(byName.get(name)?.section).toBeGreaterThan(0); + } + expect(byName.has("lonely")).toBe(false); + expect(byName.has("excluded_unit_ref")).toBe(false); + }); + test("does not pull an alternate definition when a selected object already satisfies the reference", () => { // program defines foo and needs bar; the first support member defines // bar and calls back into foo. A real archive link stops there. The @@ -477,7 +518,7 @@ describe("mergeAndLocalizeCoffObjects", () => { ); const merged = readCoff( - mergeAndLocalizeCoffObjects(program, [needed, alternate], new Set()), + mergeAndLocalizeCoffObjects([program], [needed, alternate], new Set()), ); const byName = new Map(merged.symbols.map((s) => [s.name, s])); expect(byName.has("alternate_only")).toBe(false); @@ -528,7 +569,7 @@ describe("mergeAndLocalizeCoffObjects", () => { { name: ".refptr.shared", section: 2 }, ], ); - const merged = readCoff(mergeAndLocalizeCoffObjects(program, [support], new Set(["keep_me"]))); + const merged = readCoff(mergeAndLocalizeCoffObjects([program], [support], new Set(["keep_me"]))); // One survivor section carries the stub; no COMDAT flag remains anywhere. const rdata = merged.sections.filter((s) => s.name === ".rdata"); expect(rdata.length).toBe(1); @@ -578,7 +619,7 @@ describe("mergeAndLocalizeCoffObjects", () => { test("COMDAT LARGEST retains the largest selected definition", () => { const pair = comdatPair(6, new Uint8Array([1, 2, 3, 4]), new Uint8Array([5, 6, 7, 8, 9])); - const merged = readCoff(mergeAndLocalizeCoffObjects(pair.program, pair.support, new Set(["entry"]))); + const merged = readCoff(mergeAndLocalizeCoffObjects([pair.program], pair.support, new Set(["entry"]))); const rdata = merged.sections.filter((section) => section.name === ".rdata"); expect(rdata).toHaveLength(1); expect([...rdata[0]!.data]).toEqual([5, 6, 7, 8, 9]); @@ -620,7 +661,7 @@ describe("mergeAndLocalizeCoffObjects", () => { }; const merged = readCoff( mergeAndLocalizeCoffObjects( - pair.program, + [pair.program], [withAssociate(1), withAssociate(2)], new Set(["entry"]), ), @@ -633,8 +674,8 @@ describe("mergeAndLocalizeCoffObjects", () => { test("COMDAT SAME_SIZE refuses definitions with different sizes", () => { const pair = comdatPair(3, new Uint8Array(4), new Uint8Array(8)); expect(() => - mergeAndLocalizeCoffObjects(pair.program, pair.support, new Set(), { - program: "program.o", + mergeAndLocalizeCoffObjects([pair.program], pair.support, new Set(), { + roots: ["program.o"], support: ["one.o", "two.o"], }), ).toThrow(/SAME_SIZE mismatch.*one\.o.*two\.o/); @@ -643,8 +684,8 @@ describe("mergeAndLocalizeCoffObjects", () => { test("COMDAT EXACT_MATCH refuses equal-size definitions with different contents", () => { const pair = comdatPair(4, new Uint8Array([1, 2, 3, 4]), new Uint8Array([1, 2, 3, 5])); expect(() => - mergeAndLocalizeCoffObjects(pair.program, pair.support, new Set(), { - program: "program.o", + mergeAndLocalizeCoffObjects([pair.program], pair.support, new Set(), { + roots: ["program.o"], support: ["one.o", "two.o"], }), ).toThrow(/EXACT_MATCH mismatch.*one\.o.*two\.o/); @@ -653,7 +694,7 @@ describe("mergeAndLocalizeCoffObjects", () => { test("COMDAT duplicates refuse conflicting selection kinds", () => { const one = comdatPair(2, new Uint8Array(4), new Uint8Array(4)); const two = comdatPair(3, new Uint8Array(4), new Uint8Array(4)); - expect(() => mergeAndLocalizeCoffObjects(one.program, [one.support[0]!, two.support[1]!], new Set())) + expect(() => mergeAndLocalizeCoffObjects([one.program], [one.support[0]!, two.support[1]!], new Set())) .toThrow(/conflicting COMDAT selections/); }); @@ -676,8 +717,8 @@ describe("mergeAndLocalizeCoffObjects", () => { ], ); expect(() => - mergeAndLocalizeCoffObjects(one, [two], new Set(["keep_me"]), { - program: "one.o", + mergeAndLocalizeCoffObjects([one], [two], new Set(["keep_me"]), { + roots: ["one.o"], support: ["two.o"], }), ).toThrow(/duplicate external symbol dup.*one\.o.*two\.o/); @@ -686,6 +727,6 @@ describe("mergeAndLocalizeCoffObjects", () => { test("refuses non-AMD64 machines", () => { const object = buildCoff([text()], [{ name: "x", section: 1 }]); new DataView(object.buffer, object.byteOffset).setUint16(0, 0xaa64, true); - expect(() => mergeAndLocalizeCoffObjects(object, [], new Set())).toThrow(/machine/); + expect(() => mergeAndLocalizeCoffObjects([object], [], new Set())).toThrow(/machine/); }); }); diff --git a/packages/compiler/src/backend/object-localize.ts b/packages/compiler/src/backend/object-localize.ts index 979bac009..5c8bde8bb 100644 --- a/packages/compiler/src/backend/object-localize.ts +++ b/packages/compiler/src/backend/object-localize.ts @@ -19,13 +19,13 @@ * another archive's copy at the embedder's link. * * mergeAndLocalizeCoffObjects - * the COFF combine+demote in one pass: pull - * support objects on undefined-symbol demand (the - * staging-archive member semantics `ld -r` gives - * the other formats), concatenate the selected - * objects' sections, resolve cross-object symbol - * references by index, then demote every defined - * external outside the keep set to a static + * the COFF combine+demote in one pass: include every + * root object, pull support objects on undefined- + * symbol demand (the staging-archive member semantics + * `ld -r` gives the other formats), concatenate the + * selected objects' sections, resolve cross-object + * symbol references by index, then demote every + * defined external outside the keep set to a static * symbol. * * Shared demotion rule (GNU objcopy --keep-global-symbols semantics): @@ -544,20 +544,21 @@ function coffSectionContentsEqual(a: CoffSection, b: CoffSection): boolean { return true; } -/** Combine a program object with the support objects it (transitively) - * needs into ONE COFF object, then demote every defined external outside - * `keep` to a static symbol. Support objects join on undefined-symbol +/** Combine mandatory root objects with the support objects they + * (transitively) need into ONE COFF object, then demote every defined external + * outside `keep` to a static symbol. Support objects join on undefined-symbol * demand — the staging-archive member semantics the other formats get from * `ld -r` — so an unused member's undefined references never reach the * embedder's link. x86_64 only (the one COFF target scriptc produces). */ export function mergeAndLocalizeCoffObjects( - program: Uint8Array, + roots: readonly Uint8Array[], support: readonly Uint8Array[], keep: ReadonlySet, - labels?: { program?: string; support?: readonly string[] }, + labels?: { roots?: readonly string[]; support?: readonly string[] }, ): Uint8Array { + if (roots.length === 0) fail("COFF localization requires at least one root object"); const objects = [ - parseCoff(program, labels?.program ?? "program object"), + ...roots.map((bytes, i) => parseCoff(bytes, labels?.roots?.[i] ?? `root object ${i}`)), ...support.map((bytes, i) => parseCoff(bytes, labels?.support?.[i] ?? `support object ${i}`)), ]; for (const object of objects) { @@ -571,7 +572,7 @@ export function mergeAndLocalizeCoffObjects( // list order wins the pull, matching `ar` member order. const definers = new Map(); objects.forEach((object, index) => { - if (index === 0) return; + if (index < roots.length) return; for (const sym of object.symbols) { if (!coffDefines(sym)) continue; const list = definers.get(sym.name); @@ -579,7 +580,7 @@ export function mergeAndLocalizeCoffObjects( else list.push(index); } }); - const included: boolean[] = objects.map((_, i) => i === 0); + const included: boolean[] = objects.map((_, i) => i < roots.length); // Archive extraction consults the linker's CURRENT symbol state: an // undefined in a newly pulled member is already satisfied when the // program object (or an earlier member) defines it. Record every @@ -593,8 +594,8 @@ export function mergeAndLocalizeCoffObjects( if (coffDefines(sym)) selectedDefinitions.add(sym.name); } }; - addDefinitions(objects[0]!); - const queue = [0]; + for (let i = 0; i < roots.length; i++) addDefinitions(objects[i]!); + const queue = roots.map((_, i) => i); while (queue.length > 0) { const object = objects[queue.shift()!]!; for (const sym of object.symbols) {