Preserve library identity during localization

packages/compiler/src/backend/cc-cache.test.ts

@@ -2991,3 +2991,66 @@ test("library identity edits reuse the cached large program object", async () =>
     else process.env["SCRIPTC_NO_CACHE"] = oldNoCache;
   }
 });
+
+test.skipIf(process.platform === "win32" || zigExecutable === undefined)(
+  "cross-ELF localized archives retain unreferenced identity roots",
+  async () => {
+    const dir = await mkdtemp(join(tmpdir(), "scriptc-lib-localized-identity-"));
+    scratch.push(dir);
+    const cPath = join(dir, "program.c");
+    const identityCPath = join(dir, "identity.c");
+    const probePath = join(dir, "probe.c");
+    const archivePath = join(dir, "program.lib.a");
+    const probeOutput = join(dir, "probe");
+    const cacheRoot = join(dir, "cache");
+    const target = "x86_64-linux-gnu.2.36";
+    const oldCacheDir = process.env["SCRIPTC_CACHE_DIR"];
+    const oldNoCache = process.env["SCRIPTC_NO_CACHE"];
+    const oldCc = process.env["SCRIPTC_CC"];
+    const oldTarget = process.env["SCRIPTC_TARGET"];
+    try {
+      process.env["SCRIPTC_CACHE_DIR"] = cacheRoot;
+      delete process.env["SCRIPTC_NO_CACHE"];
+      process.env["SCRIPTC_CC"] = "zigcc";
+      process.env["SCRIPTC_TARGET"] = target;
+      await writeFile(cPath, "int scriptc_large_program_value(void) { return 7; }\n");
+      await writeFile(identityCPath, [
+        "unsigned long long scriptc_build_id(void) { return 2; }",
+        "unsigned scriptc_abi_version(void) { return 1; }",
+        "",
+      ].join("\n"));
+      await compileLibArchive({
+        cPath,
+        identityCPath,
+        outPath: archivePath,
+        cacheIdentity: TEST_CACHE_IDENTITY,
+        localizeSymbols: [
+          "scriptc_large_program_value",
+          "scriptc_build_id",
+          "scriptc_abi_version",
+        ],
+      });
+      await writeFile(probePath, [
+        "int scriptc_large_program_value(void);",
+        "unsigned long long scriptc_build_id(void);",
+        "unsigned scriptc_abi_version(void);",
+        "int main(void) {",
+        "  return scriptc_large_program_value() != 7 || scriptc_build_id() != 2 || scriptc_abi_version() != 1;",
+        "}",
+        "",
+      ].join("\n"));
+      execFileSync(zigExecutable!, [
+        "cc", "-target", target, probePath, archivePath, "-lm", "-o", probeOutput,
+      ]);
+    } finally {
+      if (oldCacheDir === undefined) delete process.env["SCRIPTC_CACHE_DIR"];
+      else process.env["SCRIPTC_CACHE_DIR"] = oldCacheDir;
+      if (oldNoCache === undefined) delete process.env["SCRIPTC_NO_CACHE"];
+      else process.env["SCRIPTC_NO_CACHE"] = oldNoCache;
+      if (oldCc === undefined) delete process.env["SCRIPTC_CC"];
+      else process.env["SCRIPTC_CC"] = oldCc;
+      if (oldTarget === undefined) delete process.env["SCRIPTC_TARGET"];
+      else process.env["SCRIPTC_TARGET"] = oldTarget;
+    }
+  },
+);

packages/compiler/src/backend/cc.ts

@@ -1844,8 +1844,8 @@ export async function compileLibArchive(opts: LibArchiveOptions): Promise<void>
                 driver,
                 arArgv,
                 buildDir,
-                programObject,
-                [...(identityObject === null ? [] : [identityObject]), ...runtimeObjects, ...lreObjects, ...zlibObjects],
+                [programObject, ...(identityObject === null ? [] : [identityObject])],
+                [...runtimeObjects, ...lreObjects, ...zlibObjects],
                 opts.localizeSymbols,
                 stem,
               ),
@@ -1941,7 +1941,8 @@ export async function compileLibArchive(opts: LibArchiveOptions): Promise<void>
  * shared by design. Windows embedders additionally link advapi32, iphlpapi,
  * and ws2_32.
  *
- * Member selection matters: a classic archive's unused members (and their
+ * The generated program and optional identity objects are mandatory roots;
+ * support-member selection still matters. A classic archive's unused members (and their
  * undefined references to units library mode excludes, like the
  * fs-promises unit's fiber symbols) never reach an embedder's link. A
  * blind merge of every object would carry those references into the one
@@ -1950,7 +1951,7 @@ export async function compileLibArchive(opts: LibArchiveOptions): Promise<void>
  * members the program object transitively needs (the COFF arm implements
  * the same member semantics in process).
  *
- *   Mach-O — one host-ld64 invocation: -r merges program + needed members,
+ *   Mach-O — one host-ld64 invocation: -r merges roots + needed members,
  *            -exported_symbols_list demotes every unlisted global to
  *            private extern, and -r without -keep_private_externs writes
  *            private externs out as non-external symbols. Apple ASan's
@@ -1985,7 +1986,7 @@ async function localizeLibraryObjects(
   driver: CcDriver,
   arArgv: readonly string[],
   buildDir: string,
-  programObject: string,
+  rootObjects: readonly string[],
   supportObjects: readonly string[],
   keepSymbols: readonly string[],
   stem: string,
@@ -2008,14 +2009,15 @@ async function localizeLibraryObjects(
   if (platform === "win32") {
     // COFF has no relocatable-link tool to stage through; the member
     // selection and combine+demote happen in process over the object bytes.
-    const [program, ...support] = await Promise.all(
-      [programObject, ...supportObjects].map((path) => readFile(path)),
-    );
+    const [roots, support] = await Promise.all([
+      Promise.all(rootObjects.map((path) => readFile(path))),
+      Promise.all(supportObjects.map((path) => readFile(path))),
+    ]);
     try {
       await writeFile(
         combined,
-        mergeAndLocalizeCoffObjects(program!, support, new Set(keepSymbols), {
-          program: basename(programObject),
+        mergeAndLocalizeCoffObjects(roots, support, new Set(keepSymbols), {
+          roots: rootObjects.map((path) => basename(path)),
           support: supportObjects.map((path) => basename(path)),
         }),
       );
@@ -2029,10 +2031,10 @@ async function localizeLibraryObjects(
   await run([arArgv[0] ?? "ar", ...arArgv.slice(1), "rcs", staging, ...supportObjects]);
   if (platform === "darwin") {
     await writeFile(keepFile, keepSymbols.map((s) => `_${s}\n`).join(""));
-    await run(["ld", "-r", programObject, staging, "-o", combined, "-exported_symbols_list", keepFile]);
+    await run(["ld", "-r", ...rootObjects, staging, "-o", combined, "-exported_symbols_list", keepFile]);
   } else if (platform === "linux" && driver.target === null) {
     await writeFile(keepFile, keepSymbols.map((s) => `${s}\n`).join(""));
-    await run(["ld", "-r", "--force-group-allocation", programObject, staging, "-o", combined]);
+    await run(["ld", "-r", "--force-group-allocation", ...rootObjects, staging, "-o", combined]);
     await run(["objcopy", `--keep-global-symbols=${keepFile}`, combined]);
   } else if (platform === "linux") {
     // Cross ELF: the cross driver's own lld performs the relocatable merge
@@ -2046,7 +2048,7 @@ async function localizeLibraryObjects(
       ...driver.argv.slice(1),
       "-target", driver.zigTarget ?? driver.target!,
       "-nostdlib",
-      "-r", programObject, staging,
+      "-r", ...rootObjects, staging,
       "-o", combined,
     ]);
     try {

packages/compiler/src/backend/object-localize.test.ts

@@ -431,7 +431,7 @@ describe("mergeAndLocalizeCoffObjects", () => {
       ],
     );
     const merged = readCoff(
-      mergeAndLocalizeCoffObjects(program, [needed, unneeded], new Set(["keep_me"])),
+      mergeAndLocalizeCoffObjects([program], [needed, unneeded], new Set(["keep_me"])),
     );
     const byName = new Map(merged.symbols.map((s) => [s.name, s]));
     expect(byName.get("keep_me")!.storageClass).toBe(IMAGE_SYM_CLASS_EXTERNAL);
@@ -446,6 +446,47 @@ describe("mergeAndLocalizeCoffObjects", () => {
     expect(merged.sections[0]!.relocs[0]!.sym).toBe(byName.get("helper")!.index);
   });
 
+  test("includes every root even when no other object references its exports", () => {
+    const program = buildCoff(
+      [text()],
+      [
+        { name: ".text", section: 1, storageClass: IMAGE_SYM_CLASS_STATIC, sectionDef: true },
+        { name: "keep_me", section: 1 },
+      ],
+    );
+    const identity = buildCoff(
+      [text()],
+      [
+        { name: ".text", section: 1, storageClass: IMAGE_SYM_CLASS_STATIC, sectionDef: true },
+        { name: "build_id", section: 1 },
+        { name: "abi_version", section: 1, value: 4 },
+      ],
+    );
+    const unneeded = buildCoff(
+      [text()],
+      [
+        { name: ".text", section: 1, storageClass: IMAGE_SYM_CLASS_STATIC, sectionDef: true },
+        { name: "lonely", section: 1 },
+        { name: "excluded_unit_ref", section: 0 },
+      ],
+    );
+
+    const merged = readCoff(
+      mergeAndLocalizeCoffObjects(
+        [program, identity],
+        [unneeded],
+        new Set(["keep_me", "build_id", "abi_version"]),
+      ),
+    );
+    const byName = new Map(merged.symbols.map((symbol) => [symbol.name, symbol]));
+    for (const name of ["keep_me", "build_id", "abi_version"]) {
+      expect(byName.get(name)?.storageClass).toBe(IMAGE_SYM_CLASS_EXTERNAL);
+      expect(byName.get(name)?.section).toBeGreaterThan(0);
+    }
+    expect(byName.has("lonely")).toBe(false);
+    expect(byName.has("excluded_unit_ref")).toBe(false);
+  });
+
   test("does not pull an alternate definition when a selected object already satisfies the reference", () => {
     // program defines foo and needs bar; the first support member defines
     // bar and calls back into foo. A real archive link stops there. The
@@ -477,7 +518,7 @@ describe("mergeAndLocalizeCoffObjects", () => {
     );
 
     const merged = readCoff(
-      mergeAndLocalizeCoffObjects(program, [needed, alternate], new Set()),
+      mergeAndLocalizeCoffObjects([program], [needed, alternate], new Set()),
     );
     const byName = new Map(merged.symbols.map((s) => [s.name, s]));
     expect(byName.has("alternate_only")).toBe(false);
@@ -528,7 +569,7 @@ describe("mergeAndLocalizeCoffObjects", () => {
         { name: ".refptr.shared", section: 2 },
       ],
     );
-    const merged = readCoff(mergeAndLocalizeCoffObjects(program, [support], new Set(["keep_me"])));
+    const merged = readCoff(mergeAndLocalizeCoffObjects([program], [support], new Set(["keep_me"])));
     // One survivor section carries the stub; no COMDAT flag remains anywhere.
     const rdata = merged.sections.filter((s) => s.name === ".rdata");
     expect(rdata.length).toBe(1);
@@ -578,7 +619,7 @@ describe("mergeAndLocalizeCoffObjects", () => {
 
   test("COMDAT LARGEST retains the largest selected definition", () => {
     const pair = comdatPair(6, new Uint8Array([1, 2, 3, 4]), new Uint8Array([5, 6, 7, 8, 9]));
-    const merged = readCoff(mergeAndLocalizeCoffObjects(pair.program, pair.support, new Set(["entry"])));
+    const merged = readCoff(mergeAndLocalizeCoffObjects([pair.program], pair.support, new Set(["entry"])));
     const rdata = merged.sections.filter((section) => section.name === ".rdata");
     expect(rdata).toHaveLength(1);
     expect([...rdata[0]!.data]).toEqual([5, 6, 7, 8, 9]);
@@ -620,7 +661,7 @@ describe("mergeAndLocalizeCoffObjects", () => {
     };
     const merged = readCoff(
       mergeAndLocalizeCoffObjects(
-        pair.program,
+        [pair.program],
         [withAssociate(1), withAssociate(2)],
         new Set(["entry"]),
       ),
@@ -633,8 +674,8 @@ describe("mergeAndLocalizeCoffObjects", () => {
   test("COMDAT SAME_SIZE refuses definitions with different sizes", () => {
     const pair = comdatPair(3, new Uint8Array(4), new Uint8Array(8));
     expect(() =>
-      mergeAndLocalizeCoffObjects(pair.program, pair.support, new Set(), {
-        program: "program.o",
+      mergeAndLocalizeCoffObjects([pair.program], pair.support, new Set(), {
+        roots: ["program.o"],
         support: ["one.o", "two.o"],
       }),
     ).toThrow(/SAME_SIZE mismatch.*one\.o.*two\.o/);
@@ -643,8 +684,8 @@ describe("mergeAndLocalizeCoffObjects", () => {
   test("COMDAT EXACT_MATCH refuses equal-size definitions with different contents", () => {
     const pair = comdatPair(4, new Uint8Array([1, 2, 3, 4]), new Uint8Array([1, 2, 3, 5]));
     expect(() =>
-      mergeAndLocalizeCoffObjects(pair.program, pair.support, new Set(), {
-        program: "program.o",
+      mergeAndLocalizeCoffObjects([pair.program], pair.support, new Set(), {
+        roots: ["program.o"],
         support: ["one.o", "two.o"],
       }),
     ).toThrow(/EXACT_MATCH mismatch.*one\.o.*two\.o/);
@@ -653,7 +694,7 @@ describe("mergeAndLocalizeCoffObjects", () => {
   test("COMDAT duplicates refuse conflicting selection kinds", () => {
     const one = comdatPair(2, new Uint8Array(4), new Uint8Array(4));
     const two = comdatPair(3, new Uint8Array(4), new Uint8Array(4));
-    expect(() => mergeAndLocalizeCoffObjects(one.program, [one.support[0]!, two.support[1]!], new Set()))
+    expect(() => mergeAndLocalizeCoffObjects([one.program], [one.support[0]!, two.support[1]!], new Set()))
       .toThrow(/conflicting COMDAT selections/);
   });
 
@@ -676,8 +717,8 @@ describe("mergeAndLocalizeCoffObjects", () => {
       ],
     );
     expect(() =>
-      mergeAndLocalizeCoffObjects(one, [two], new Set(["keep_me"]), {
-        program: "one.o",
+      mergeAndLocalizeCoffObjects([one], [two], new Set(["keep_me"]), {
+        roots: ["one.o"],
         support: ["two.o"],
       }),
     ).toThrow(/duplicate external symbol dup.*one\.o.*two\.o/);
@@ -686,6 +727,6 @@ describe("mergeAndLocalizeCoffObjects", () => {
   test("refuses non-AMD64 machines", () => {
     const object = buildCoff([text()], [{ name: "x", section: 1 }]);
     new DataView(object.buffer, object.byteOffset).setUint16(0, 0xaa64, true);
-    expect(() => mergeAndLocalizeCoffObjects(object, [], new Set())).toThrow(/machine/);
+    expect(() => mergeAndLocalizeCoffObjects([object], [], new Set())).toThrow(/machine/);
   });
 });

packages/compiler/src/backend/object-localize.ts

@@ -19,13 +19,13 @@
  *                          another archive's copy at the embedder's link.
  *
  *   mergeAndLocalizeCoffObjects
- *                          the COFF combine+demote in one pass: pull
- *                          support objects on undefined-symbol demand (the
- *                          staging-archive member semantics `ld -r` gives
- *                          the other formats), concatenate the selected
- *                          objects' sections, resolve cross-object symbol
- *                          references by index, then demote every defined
- *                          external outside the keep set to a static
+ *                          the COFF combine+demote in one pass: include every
+ *                          root object, pull support objects on undefined-
+ *                          symbol demand (the staging-archive member semantics
+ *                          `ld -r` gives the other formats), concatenate the
+ *                          selected objects' sections, resolve cross-object
+ *                          symbol references by index, then demote every
+ *                          defined external outside the keep set to a static
  *                          symbol.
  *
  * Shared demotion rule (GNU objcopy --keep-global-symbols semantics):
@@ -544,20 +544,21 @@ function coffSectionContentsEqual(a: CoffSection, b: CoffSection): boolean {
   return true;
 }
 
-/** Combine a program object with the support objects it (transitively)
- * needs into ONE COFF object, then demote every defined external outside
- * `keep` to a static symbol. Support objects join on undefined-symbol
+/** Combine mandatory root objects with the support objects they
+ * (transitively) need into ONE COFF object, then demote every defined external
+ * outside `keep` to a static symbol. Support objects join on undefined-symbol
  * demand — the staging-archive member semantics the other formats get from
  * `ld -r` — so an unused member's undefined references never reach the
  * embedder's link. x86_64 only (the one COFF target scriptc produces). */
 export function mergeAndLocalizeCoffObjects(
-  program: Uint8Array,
+  roots: readonly Uint8Array[],
   support: readonly Uint8Array[],
   keep: ReadonlySet<string>,
-  labels?: { program?: string; support?: readonly string[] },
+  labels?: { roots?: readonly string[]; support?: readonly string[] },
 ): Uint8Array {
+  if (roots.length === 0) fail("COFF localization requires at least one root object");
   const objects = [
-    parseCoff(program, labels?.program ?? "program object"),
+    ...roots.map((bytes, i) => parseCoff(bytes, labels?.roots?.[i] ?? `root object ${i}`)),
     ...support.map((bytes, i) => parseCoff(bytes, labels?.support?.[i] ?? `support object ${i}`)),
   ];
   for (const object of objects) {
@@ -571,7 +572,7 @@ export function mergeAndLocalizeCoffObjects(
   // list order wins the pull, matching `ar` member order.
   const definers = new Map<string, number[]>();
   objects.forEach((object, index) => {
-    if (index === 0) return;
+    if (index < roots.length) return;
     for (const sym of object.symbols) {
       if (!coffDefines(sym)) continue;
       const list = definers.get(sym.name);
@@ -579,7 +580,7 @@ export function mergeAndLocalizeCoffObjects(
       else list.push(index);
     }
   });
-  const included: boolean[] = objects.map((_, i) => i === 0);
+  const included: boolean[] = objects.map((_, i) => i < roots.length);
   // Archive extraction consults the linker's CURRENT symbol state: an
   // undefined in a newly pulled member is already satisfied when the
   // program object (or an earlier member) defines it. Record every
@@ -593,8 +594,8 @@ export function mergeAndLocalizeCoffObjects(
       if (coffDefines(sym)) selectedDefinitions.add(sym.name);
     }
   };
-  addDefinitions(objects[0]!);
-  const queue = [0];
+  for (let i = 0; i < roots.length; i++) addDefinitions(objects[i]!);
+  const queue = roots.map((_, i) => i);
   while (queue.length > 0) {
     const object = objects[queue.shift()!]!;
     for (const sym of object.symbols) {