Fix Sandbox command transport and pinned native setup (#295)

Build the Linux LLVM helper and runtime pack before running both test lanes, and align native cache contracts with the current target policy. Full managed Sandbox gate passed.

Dockerfile.sandbox

@@ -7,18 +7,30 @@ ARG PNPM_VERSION=11.1.3
 ENV DEBIAN_FRONTEND=noninteractive
 
 RUN apt-get update \
+    && apt-get install --yes --no-install-recommends \
+        ca-certificates curl gnupg \
+    && curl --fail --silent --show-error --location https://apt.llvm.org/llvm-snapshot.gpg.key \
+        | gpg --dearmor --output /usr/share/keyrings/llvm-snapshot.gpg \
+    && echo "deb [signed-by=/usr/share/keyrings/llvm-snapshot.gpg] https://apt.llvm.org/noble/ llvm-toolchain-noble-22 main" \
+        > /etc/apt/sources.list.d/llvm22.list \
+    && apt-get update \
     && apt-get install --yes --no-install-recommends \
         build-essential \
         ca-certificates \
         ccache \
-        clang \
+        clang-22 \
         cmake \
         curl \
         git \
-        libclang-rt-dev \
-        llvm \
+        libclang-rt-22-dev \
+        libzstd-dev \
+        llvm-22-dev \
+        ninja-build \
         xz-utils \
         zlib1g-dev \
+    && ln -sf clang-22 /usr/bin/clang \
+    && ln -sf clang++-22 /usr/bin/clang++ \
+    && test "$(llvm-config-22 --version)" = 22.1.8 \
     && rm -rf /var/lib/apt/lists/*
 
 RUN curl --fail --silent --show-error --location \

packages/cli/test/bootstrap.test.ts

@@ -4,12 +4,16 @@ import { release as osRelease, tmpdir } from "node:os";
 import { join } from "node:path";
 import { promisify } from "node:util";
 import { expect, test } from "vitest";
+import { precompiledRuntimePackTarget } from "../../compiler/src/startup-cache.js";
+import { platformLinkerSupportsPersistentCache } from "../../compiler/src/backend/linker.js";
 
 const execFileAsync = promisify(execFile);
 const repoRoot = join(import.meta.dirname, "../../..");
 const bootstrap = join(repoRoot, "packages/cli/dist/bootstrap.js");
 const runtimePackHost = process.platform === "darwin" && process.arch === "arm64" &&
   Number.parseInt(osRelease().split(".", 1)[0] ?? "", 10) >= 24;
+const helperTarget = precompiledRuntimePackTarget();
+const persistentExecutable = helperTarget === null || platformLinkerSupportsPersistentCache(process.env, helperTarget);
 
 test("bootstrap serves version and help without loading the compiler graph", async () => {
   const preloadDir = await mkdtemp(join(tmpdir(), "scriptc-bootstrap-preload-"));
@@ -83,7 +87,15 @@ test("bootstrap exact builds use the routed cache and source edits fall through"
       rm(join(cacheRoot, "early-exe-implementation"), { recursive: true, force: true }),
     ]);
     expect((await build()).stderr).not.toContain("scriptc lowering");
-    await expect(build(true)).resolves.toMatchObject({ stderr: "" });
+    if (persistentExecutable) {
+      await expect(build(true)).resolves.toMatchObject({ stderr: "" });
+    } else {
+      // This target intentionally lacks a persistent native dependency proof.
+      // It must enter the full compiler to relink, even on a frontend hit.
+      await expect(build(true)).rejects.toMatchObject({ stderr: expect.stringContaining("compiler graph loaded") });
+      expect((await build()).stderr).not.toContain("scriptc lowering");
+      expect((await execFileAsync(outPath)).stdout).toBe("one\n");
+    }
 
     await writeFile(entry, 'console.log("two");\n');
     expect((await build()).stderr).toContain("scriptc lowering");

packages/cli/test/executable-cache.test.ts

@@ -5,6 +5,8 @@ import { release as osRelease, tmpdir } from "node:os";
 import { delimiter, dirname, join } from "node:path";
 import { promisify } from "node:util";
 import { expect, test } from "vitest";
+import { precompiledRuntimePackTarget } from "../../compiler/src/startup-cache.js";
+import { platformLinkerSupportsPersistentCache } from "../../compiler/src/backend/linker.js";
 
 const execFileAsync = promisify(execFile);
 const require = createRequire(import.meta.url);
@@ -13,6 +15,8 @@ const cliEntry = join(repoRoot, "packages/cli/src/main.ts");
 const tsxLoader = join(dirname(require.resolve("tsx/package.json")), "dist/loader.mjs");
 const runtimePackHost = process.platform === "darwin" && process.arch === "arm64" &&
   Number.parseInt(osRelease().split(".", 1)[0] ?? "", 10) >= 24;
+const helperTarget = precompiledRuntimePackTarget();
+const persistentExecutable = helperTarget === null || platformLinkerSupportsPersistentCache(process.env, helperTarget);
 
 test("exact executable repeats skip lowering while edits and damaged outputs stay correct", async () => {
   const dir = await mkdtemp(join(tmpdir(), "scriptc-cli-executable-cache-"));
@@ -50,10 +54,12 @@ test("exact executable repeats skip lowering while edits and damaged outputs sta
     const originalBinaryTime = (await stat(outPath)).mtimeMs;
     const originalTuTime = (await stat(tuPath)).mtimeMs;
 
-    // The exact repeat returns from the whole-program cache before tsgo and
-    // leaves already-correct caller outputs on their existing inodes.
+    // Every route reuses the frontend. Only routes with a complete native
+    // dependency proof can also retain the executable inode; other targets
+    // deliberately relink their cached translation unit.
     expect((await build()).stderr).not.toContain("scriptc lowering");
-    expect((await stat(outPath)).mtimeMs).toBe(originalBinaryTime);
+    if (persistentExecutable) expect((await stat(outPath)).mtimeMs).toBe(originalBinaryTime);
+    expect(await run()).toBe("one\n");
     expect((await stat(tuPath)).mtimeMs).toBe(originalTuTime);
 
     // Output-kind cleanup is honest even on an exact early hit: an IR file

packages/cli/test/native-output.test.ts

@@ -35,11 +35,13 @@ function cli(args: string[], env: NodeJS.ProcessEnv = process.env) {
   });
 }
 
-test("unsupported hosts fail with SC3002 before creating an artifact", async () => {
-  if (supported) return;
+test("unsupported targets fail with SC3002 before creating an artifact", async () => {
   const { dir, entry } = await fixture();
   const output = join(dir, "hello.o");
-  await expect(cli(["build", entry, "--emit=obj", "-o", output])).rejects.toMatchObject({
+  await expect(cli(["build", entry, "--emit=obj", "-o", output], {
+    ...process.env,
+    SCRIPTC_TARGET: "scriptc-unsupported-target",
+  })).rejects.toMatchObject({
     code: 1,
     stderr: expect.stringContaining("SC3002"),
   });

packages/compiler/src/backend/native-codegen.test.ts

@@ -71,6 +71,7 @@ function request(root: string, packageJson: string, output = join(root, "program
     outputKind: "obj" as const,
     sourcePath: "/source/app.ts",
     target,
+    helperHost: { platform: "darwin" as const, arch: "arm64" },
     resolvePackageJson: () => packageJson,
     cacheRoot: join(root, "cache"),
   };

packages/compiler/src/backend/native-toolchain.test.ts

@@ -4178,7 +4178,11 @@ test.skipIf(
       probeSource,
       "long long public_value(void);\nint main(void) { return public_value() != 7; }\n",
     );
-    execFileSync(clangExecutable!, [probeSource, outPath, "-lm", "-o", probe]);
+    // The failed linker belongs only to the shard-merge probe. Link the
+    // resulting archive with the real host tools to verify its fallback.
+    execFileSync(clangExecutable!, [probeSource, outPath, "-lm", "-o", probe], {
+      env: { ...process.env, PATH: oldPath },
+    });
     expect(execFileSync(probe, { encoding: "utf8" })).toBe("");
     // The canonical retry is valid for this invocation, but must not occupy a
     // key describing merged shard bytes. A repaired merge tool retries the

scripts/sandbox-bootstrap.sh

@@ -9,30 +9,35 @@ if [ -z "$node_version" ] || [ -z "$pnpm_version" ]; then
 fi
 
 # The managed image intentionally floats its distro and language runtimes.
-# Install the same LLVM major and exact Node/pnpm tuple as the custom image so
+# Install the same LLVM and exact Node/pnpm tuple as the custom image so
 # differential oracles do not change when the managed image is refreshed.
+. /etc/os-release
+llvm_distro=${VERSION_CODENAME:?the Sandbox image must identify its Ubuntu release}
 curl --fail --silent --show-error --location \
   https://apt.llvm.org/llvm-snapshot.gpg.key \
   --output /tmp/llvm-snapshot.gpg.key
 gpg --dearmor --yes --output /tmp/llvm-snapshot.gpg /tmp/llvm-snapshot.gpg.key
 sudo install -D -m 0644 /tmp/llvm-snapshot.gpg /etc/apt/keyrings/llvm-snapshot.gpg
-echo "deb [signed-by=/etc/apt/keyrings/llvm-snapshot.gpg] https://apt.llvm.org/noble/ llvm-toolchain-noble-18 main" \
-  | sudo tee /etc/apt/sources.list.d/llvm18.list >/dev/null
+echo "deb [signed-by=/etc/apt/keyrings/llvm-snapshot.gpg] https://apt.llvm.org/${llvm_distro}/ llvm-toolchain-${llvm_distro}-22 main" \
+  | sudo tee /etc/apt/sources.list.d/llvm22.list >/dev/null
 
 sudo apt-get update --quiet=2
 sudo env DEBIAN_FRONTEND=noninteractive apt-get install --quiet=2 --yes --no-install-recommends \
   build-essential \
   ca-certificates \
   ccache \
-  clang-18 \
+  clang-22 \
   cmake \
-  libclang-rt-18-dev \
-  llvm-18 \
+  libclang-rt-22-dev \
+  libzstd-dev \
+  llvm-22-dev \
+  ninja-build \
   xz-utils \
   zlib1g-dev
 sudo rm -f /usr/local/bin/clang /usr/local/bin/clang++
-sudo ln -sf clang-18 /usr/bin/clang
-sudo ln -sf clang++-18 /usr/bin/clang++
+sudo ln -sf clang-22 /usr/bin/clang
+sudo ln -sf clang++-22 /usr/bin/clang++
+test "$(llvm-config-22 --version)" = 22.1.8
 
 node_archive="node-v${node_version}-linux-x64.tar.xz"
 curl --fail --silent --show-error --location \

scripts/sandbox-command.mjs

@@ -0,0 +1,22 @@
+export const MAX_INLINE_SANDBOX_COMMAND_BYTES = 768;
+
+export const shellQuote = (value) => `'${value.replaceAll("'", `'"'"'`)}'`;
+
+/** Keep long shell programs out of the local CLI's argument vector. The
+ * same script records the remote exit status for either transport. */
+export function sandboxCommand(command, args, exitMarker) {
+  const statusPath = `/tmp/${exitMarker}.status`;
+  const script =
+    `${[command, ...args].map(shellQuote).join(" ")}; scriptc_status=$?; ` +
+    `printf '%s\\n' "$scriptc_status" > ${shellQuote(statusPath)}; ` +
+    `printf '\\n${exitMarker}%s\\n' "$scriptc_status"`;
+  const scriptPath = `/tmp/${exitMarker}.sh`;
+  const file = Buffer.byteLength(script, "utf8") > MAX_INLINE_SANDBOX_COMMAND_BYTES;
+  return {
+    script,
+    scriptPath,
+    statusPath,
+    file,
+    argv: file ? ["sh", scriptPath] : ["sh", "-c", script],
+  };
+}

scripts/sandbox-test.mjs

@@ -1,7 +1,7 @@
 #!/usr/bin/env node
 import { spawn, spawnSync } from "node:child_process";
 import { randomBytes } from "node:crypto";
-import { mkdtemp, rm } from "node:fs/promises";
+import { mkdtemp, rm, writeFile } from "node:fs/promises";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
 import { pipeline } from "node:stream/promises";
@@ -23,6 +23,7 @@ import {
   filterExistingWorktreePaths,
   workspaceResetCommand,
 } from "./worktree-files.mjs";
+import { sandboxCommand, shellQuote } from "./sandbox-command.mjs";
 
 const root = fileURLToPath(new URL("../", import.meta.url));
 const laneCaseShardedFiles = [
@@ -371,7 +372,6 @@ const vercel = ([group, command, ...args], options) =>
   });
 // `vercel sandbox exec` does not propagate the remote process's exit code.
 // Print a per-command nonce after it finishes and enforce that status here.
-const shellQuote = (value) => `'${value.replaceAll("'", `'\"'\"'`)}'`;
 const execIn = async (
   worker,
   command,
@@ -384,12 +384,22 @@ const execIn = async (
 ) => {
   const envArgs = Object.entries(env).flatMap(([key, value]) => ["--env", `${key}=${value}`]);
   const exitMarker = `__SCRIPTC_REMOTE_EXIT_${randomBytes(12).toString("hex")}__`;
-  const statusPath = `/tmp/${exitMarker}.status`;
-  const script =
-    `${[command, ...args].map(shellQuote).join(" ")}; scriptc_status=$?; ` +
-    `printf '%s\\n' "$scriptc_status" > ${shellQuote(statusPath)}; ` +
-    `printf '\\n${exitMarker}%s\\n' "$scriptc_status"`;
+  const prepared = sandboxCommand(command, args, exitMarker);
+  const { statusPath } = prepared;
   const label = task ? `${worker.label} ${task}` : worker.label;
+  if (prepared.file) {
+    const localScript = join(temp, `${exitMarker}.sh`);
+    await writeFile(localScript, prepared.script, { mode: 0o600 });
+    try {
+      await vercel(["sandbox", "copy", localScript, `${worker.name}:${prepared.scriptPath}`], {
+        idleTimeoutMs: 60_000,
+        label: `${label} command`,
+        timeoutMs: 2 * 60_000,
+      });
+    } finally {
+      await rm(localScript, { force: true });
+    }
+  }
   const commandArgs = [
     "sandbox",
     "exec",
@@ -399,9 +409,7 @@ const execIn = async (
     workdir,
     ...envArgs,
     worker.name,
-    "sh",
-    "-c",
-    script,
+    ...prepared.argv,
   ];
   try {
     await vercel(commandArgs, {
@@ -411,7 +419,7 @@ const execIn = async (
       timeoutMs: wallTimeoutMs,
     });
   } catch (error) {
-    console.warn(`[${label}] CLI completion was not confirmed; checking the remote command status...`);
+    console.warn(`[${label}] CLI completion was not confirmed (${error.message}); checking the remote command status...`);
     const probeMarker = `__SCRIPTC_REMOTE_PROBE_${randomBytes(12).toString("hex")}__`;
     const probeScript =
       `scriptc_status=125; test ! -f ${shellQuote(statusPath)} || ` +
@@ -714,6 +722,10 @@ try {
       }
       await execIn(worker, "pnpm", ["install", "--frozen-lockfile"], {}, "", 2 * 60_000);
       await execIn(worker, "pnpm", ["build"], {}, "", 2 * 60_000);
+      // Workspace builds deliberately do not rebuild packaged native artifacts.
+      // Every remote lane needs the Linux helper and runtime from this worktree.
+      await execIn(worker, "pnpm", ["--filter", "@scriptc/llvm-linux-x64-gnu", "build:native"], {}, "LLVM helper", 5 * 60_000);
+      await execIn(worker, "pnpm", ["--filter", "@scriptc/runtime-linux-x64-gnu", "build:native"], { CC: "clang-22", AR: "llvm-ar-22" }, "runtime pack", 5 * 60_000);
     }, imageConfig.custom ? workers.length : 8);
 
     await allWorkers("Testing", async (worker) => {

tests/harness/sandbox-command.test.ts

@@ -0,0 +1,50 @@
+import { execFileSync } from "node:child_process";
+import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import { afterEach, expect, test } from "vitest";
+import { sandboxCommand } from "../../scripts/sandbox-command.mjs";
+
+const roots: string[] = [];
+const statuses: string[] = [];
+
+afterEach(async () => {
+  await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true })));
+  await Promise.all(statuses.splice(0).map((path) => rm(path, { force: true })));
+});
+
+test("short commands stay inline and retain the remote exit contract", async () => {
+  const marker = `__SCRIPTC_COMMAND_TEST_${process.pid}_SHORT__`;
+  const prepared = sandboxCommand("sh", ["-c", "exit 7"], marker);
+  statuses.push(prepared.statusPath);
+
+  expect(prepared.file).toBe(false);
+  const output = execFileSync(prepared.argv[0], prepared.argv.slice(1), { encoding: "utf8" });
+  expect(output).toBe(`\n${marker}7\n`);
+  expect(await readFile(prepared.statusPath, "utf8")).toBe("7\n");
+});
+
+test("uploaded scripts preserve long and shell-sensitive argument bytes", async () => {
+  const root = await mkdtemp("/tmp/scriptc-sandbox-command-test-");
+  roots.push(root);
+  const sentinel = join(root, "unexpected");
+  const args = [
+    "x".repeat(2048),
+    "quote' and spaces",
+    `$(touch ${sentinel})`,
+    "backtick` dollar$ slash\\ newline\nend",
+    "é".repeat(512),
+  ];
+  const marker = `__SCRIPTC_COMMAND_TEST_${process.pid}_LONG__`;
+  const prepared = sandboxCommand("printf", ["<%s>\n", ...args], marker);
+  statuses.push(prepared.statusPath);
+  expect(prepared.file).toBe(true);
+  expect(prepared.argv).toEqual(["sh", prepared.scriptPath]);
+  expect(prepared.argv.every((arg) => arg.length < 128)).toBe(true);
+
+  const localScript = join(root, "command.sh");
+  await writeFile(localScript, prepared.script);
+  const output = execFileSync("sh", [localScript], { encoding: "utf8" });
+  expect(output).toBe(args.map((arg) => `<${arg}>\n`).join("") + `\n${marker}0\n`);
+  await expect(readFile(sentinel)).rejects.toMatchObject({ code: "ENOENT" });
+  expect(await readFile(prepared.statusPath, "utf8")).toBe("0\n");
+});

tests/harness/sandbox-config.test.ts

@@ -66,11 +66,12 @@ test("the managed-image bootstrap pins the custom image's Node, pnpm, and LLVM",
   const bootstrap = readFileSync(new URL("../../scripts/sandbox-bootstrap.sh", import.meta.url), "utf8");
   expect(bootstrap).toContain("< .node-version");
   expect(bootstrap).toContain("ARG PNPM_VERSION=");
-  expect(bootstrap).toContain("llvm-toolchain-noble-18");
+  expect(bootstrap).toContain("llvm-toolchain-${llvm_distro}-22");
   expect(bootstrap).toContain("install -D -m 0644");
-  expect(bootstrap).toContain("clang-18");
-  expect(bootstrap).toContain("libclang-rt-18-dev");
-  expect(bootstrap).toContain("llvm-18");
+  expect(bootstrap).toContain("clang-22");
+  expect(bootstrap).toContain("libclang-rt-22-dev");
+  expect(bootstrap).toContain("llvm-22-dev");
+  expect(bootstrap).toContain("ninja-build");
 });
 
 test("a custom image does not select the Sandbox team or project", () => {