Shard large Linux dev executables again (#2430)

packages/compiler/src/backend/llvm/split.test.ts

@@ -324,6 +324,33 @@ test("debug metadata stays on definitions when splitting LLVM modules", async ()
   }
 });
 
+test("personality routines stay on definitions when splitting LLVM modules", async () => {
+  // Functions with landing pads (Linux unwinding) carry a personality; every
+  // other shard declares them, and a declaration must not repeat it.
+  const source =
+    SAMPLE.replaceAll(") #0 {", ") #0 personality ptr @__gcc_personality_v0 {") +
+    "declare i32 @__gcc_personality_v0(...)\n";
+  const split = splitLlvmProgram(source, { minimumBytes: 0 });
+  expect(split).not.toBeNull();
+  const dir = await mkdtemp(join(tmpdir(), "scriptc-personality-split-"));
+  scratch.push(dir);
+  for (const shard of split!.shards) {
+    expect(shard.source()).not.toMatch(/^declare.* personality ptr/m);
+    const path = join(dir, shard.name);
+    await writeFile(path, shard.source());
+    const result = spawnSync("clang", ["-Wno-override-module", "-c", path, "-o", `${path}.o`], {
+      encoding: "utf8",
+    });
+    expect(result.status, result.stderr).toBe(0);
+    expect(result.stderr).toBe("");
+  }
+  expect(
+    split!.shards.flatMap((shard) => [
+      ...shard.source().matchAll(/^define .* personality ptr @__gcc_personality_v0 \{/gm),
+    ]).length,
+  ).toBe(3);
+});
+
 test("dev libraries split at the measured 2MB crossover while executables retain 4MB", () => {
   const body = Array.from(
     { length: 700 },

packages/compiler/src/backend/llvm/split.ts

@@ -104,7 +104,12 @@ function functionHeader(
   // attributes, zeroext, and varargs stay byte-exact.
   // A definition's DISubprogram cannot be attached to a declaration in a
   // different shard. Its owning definition retains the original metadata.
-  const attrs = match[5]!.replace(/\s+!dbg !\d+/g, "");
+  // A personality routine belongs to a definition too: LLVM rejects it on a
+  // declaration, which would fail every shard that calls a function with
+  // landing pads (Linux unwinding) and fall back to the single-TU compile.
+  const attrs = match[5]!
+    .replace(/\s+!dbg !\d+/g, "")
+    .replace(/\s+personality\s+ptr\s+@[-$._A-Za-z][-$.\w]*/g, "");
   const declaration = `declare ${linkage}${match[2]} ${match[3]}(${match[4]})${attrs}`.trimEnd();
   return { line, declaration, symbol, promoted };
 }

packages/compiler/src/backend/native/executable.ts

@@ -1485,12 +1485,18 @@ export async function compileCInternal(
             missed: false,
           };
         });
-        const shardWidth = Math.min(8, availableParallelism());
-        for (let i = 0; i < entries.length; i += shardWidth) {
-          await Promise.all(
-            entries.slice(i, i + shardWidth).map(async (entry) => {
+        // Up to eight clangs (one per available core), each taking the next
+        // shard as it finishes: shards differ in size, and a fixed batch
+        // would wait for its slowest member before starting the next. A
+        // failed shard stops the others from taking more work.
+        let nextShard = 0;
+        let failed = false;
+        await Promise.all(
+          Array.from({ length: Math.min(entries.length, 8, availableParallelism()) }, async () => {
+            while (!failed && nextShard < entries.length) {
+              const entry = entries[nextShard++]!;
               await writeFile(entry.sourcePath, entry.source());
-              if (await copyValidCachedFile(entry.cachePath, entry.staged)) return;
+              if (await copyValidCachedFile(entry.cachePath, entry.staged)) continue;
               entry.missed = true;
               await runClang([
                 ...programCompilerArgs,
@@ -1499,10 +1505,13 @@ export async function compileCInternal(
                 entry.sourcePath,
                 "-o",
                 entry.staged,
-              ]);
-            }),
-          );
-        }
+              ]).catch((error: unknown) => {
+                failed = true;
+                throw error;
+              });
+            }
+          }),
+        );
         const arArgv = isZigDriver(driver) ? [driver.argv[0]!, "ar"] : ["ar"];
         const merged = await localizeLibraryObjects(
           driver,