Name the reason in the CompileError for untranslated WebAssembly modules

A WebAssembly module the build finds but does not translate (one that uses
a proposal outside the translated feature set, an invalid one, or one the
translator rejects) now rejects with a CompileError that says the module
was found and names the reason, for example "it uses simd, outside the
translated WebAssembly feature set". Bytes the build did not see keep the
existing message.

The build emits the hash, length and reason of each such module in a
second table, scr_wasm_untranslated, next to scr_wasm_registry; the runtime
reads it when a lookup misses. WebAssembly.validate still answers false for
these modules, and the error stays a catchable CompileError.

Fixes #1617

Co-authored-by: jlucaso1 <55464917+jlucaso1@users.noreply.github.com>

docs/content/limitations/webassembly-modules.mdx

@@ -5,7 +5,8 @@ Bytes the build did not see, including bytes produced at run time, reject with a
 `scriptc coverage` lists the modules a build finds, with their size, whether they validate, and the WebAssembly proposals they use.
 
 Translated modules can use the WebAssembly 2.0 feature set: the MVP plus sign-extension operators, non-trapping float-to-int conversions, bulk memory, multi-value, mutable globals, reference types, and extended constant expressions.
-Modules that use SIMD, threads, exception handling, tail calls, typed function references or GC, memory64, or multiple memories are not translated, so loading them rejects with the `CompileError` above; packages that fall back to a scalar build when that happens, such as undici and es-module-lexer, take their fallback.
+Modules that use SIMD, threads, exception handling, tail calls, typed function references or GC, memory64, or multiple memories are not translated, so loading them rejects with a `CompileError` that names the proposal the module uses, and `WebAssembly.validate` answers `false` for them; packages that fall back to a scalar build when that happens, such as undici and es-module-lexer, take their fallback.
+Invalid modules the build finds reject with a `CompileError` that includes the validation error.
 
 Supported: `WebAssembly.compile`, `instantiate` (with bytes or a `Module`), `validate`, `new WebAssembly.Module`, `new WebAssembly.Instance`, `new WebAssembly.Memory`, `new WebAssembly.Table`, `new WebAssembly.Global`, `WebAssembly.Module.imports`, `exports` and `customSections`, and `instanceof` for every class; the error classes `CompileError`, `LinkError` and `RuntimeError`, constructed with or without `new`; `globalThis.WebAssembly` and its `compile`, `instantiate` and `validate` functions as values.
 Instances expose exported functions, memories (`buffer`, `grow`), tables (`get`, `set`, `grow`, `length`) and globals (`value`, `valueOf`).

packages/compiler/src/backend/llvm/wasm-modules.ts

@@ -1,12 +1,18 @@
 /* The WebAssembly modules translated into a program (IrModule.wasmModules):
- * each module's translation (wasm/translate.ts) and the registry the
- * runtime looks modules up in (scr_wasm_registry, scr_wasm_abi.h). The
+ * each module's translation (wasm/translate.ts), the registry the runtime
+ * looks modules up in (scr_wasm_registry, scr_wasm_abi.h), and the table of
+ * modules the build saw but does not translate (scr_wasm_untranslated). The
  * translations join the program module, so the native helper optimizes,
  * partitions and caches them with it. Programs that do not use the
  * WebAssembly API carry no wasmModules and emit nothing here. */
-import type { IrModule } from "../../ir/ir.js";
+import type { IrModule, IrWasmUntranslated } from "../../ir/ir.js";
 import { decodeBase64 } from "../../wasm/base64.js";
-import { translateWasmModule, WasmTranslateError } from "../../wasm/translate.js";
+import {
+  cBytes,
+  translateWasmModule,
+  utf8Bytes,
+  WasmTranslateError,
+} from "../../wasm/translate.js";
 
 function hexBytes(hex: string): Uint8Array {
   const out = new Uint8Array(hex.length / 2);
@@ -28,11 +34,38 @@ export function wasmBoundsChecks(options: {
   return /windows|mingw|cygwin|win32|msvc/.test(target);
 }
 
-/** The top-level lines of every translated module plus the registry.
- * `declared` holds the declarations the module already carries; runtime
- * entry points it lacks are declared here. A module the translator
- * cannot translate is left out of the registry, so loading it at run
- * time rejects with the CompileError for untranslated bytes. */
+/** scr_wasm_untranslated: hash, byte length and reason per module. */
+function emitUntranslated(untranslated: readonly IrWasmUntranslated[]): string[] {
+  if (untranslated.length === 0)
+    return [`@scr_wasm_untranslated = constant { i64, ptr } { i64 0, ptr null }`];
+  const out: string[] = [];
+  const entries: string[] = [];
+  for (let index = 0; index < untranslated.length; index++) {
+    const module = untranslated[index]!;
+    const reason = new Uint8Array(utf8Bytes(module.reason));
+    out.push(
+      `@sc_wasm_untranslated_reason${index} = private unnamed_addr constant [${reason.length} x i8] c"${cBytes(reason, 0, reason.length)}"`,
+    );
+    const digest = hexBytes(module.sha256);
+    const sha: string[] = [];
+    for (let i = 0; i < 32; i++) sha.push(`i8 ${digest[i]!}`);
+    entries.push(
+      `{ [32 x i8], i64, ptr, i64 } { [32 x i8] [${sha.join(", ")}], i64 ${module.size}, ptr @sc_wasm_untranslated_reason${index}, i64 ${reason.length} }`,
+    );
+  }
+  out.push(
+    `@sc_wasm_untranslated_modules = private constant [${entries.length} x { [32 x i8], i64, ptr, i64 }] [${entries.join(", ")}]`,
+    `@scr_wasm_untranslated = constant { i64, ptr } { i64 ${entries.length}, ptr @sc_wasm_untranslated_modules }`,
+  );
+  return out;
+}
+
+/** The top-level lines of every translated module plus the registry and
+ * the untranslated table. `declared` holds the declarations the module
+ * already carries; runtime entry points it lacks are declared here. A
+ * module the translator cannot translate joins the untranslated table, so
+ * loading it at run time rejects with a CompileError that names the
+ * translator's reason. */
 export function emitWasmModules(
   mod: IrModule,
   declared: ReadonlySet<string>,
@@ -43,6 +76,7 @@ export function emitWasmModules(
   const out: string[] = [];
   const decls = new Set(declared);
   const descriptors: string[] = [];
+  const untranslated = [...(mod.wasmUntranslated ?? [])];
   for (const module of modules) {
     const bytes = decodeBase64(module.bytes);
     if (bytes === null) continue;
@@ -53,8 +87,9 @@ export function emitWasmModules(
         customSections: mod.wasmCustomSections === true,
       });
     } catch (error) {
-      if (error instanceof WasmTranslateError) continue;
-      throw error;
+      if (!(error instanceof WasmTranslateError)) throw error;
+      untranslated.push({ sha256: module.sha256, size: bytes.length, reason: error.message });
+      continue;
     }
     for (const decl of translation.decls) {
       if (decls.has(decl)) continue;
@@ -72,6 +107,7 @@ export function emitWasmModules(
       `@scr_wasm_registry = constant { i64, ptr } { i64 ${descriptors.length}, ptr @sc_wasm_registry_modules }`,
     );
   }
+  out.push(...emitUntranslated(untranslated));
   out.push("");
   return out;
 }

packages/compiler/src/frontend/lowering/lowerer.ts

@@ -210,6 +210,7 @@ import type {
   IrLocal,
   IrModule,
   IrWasmModule,
+  IrWasmUntranslated,
   IrParam,
   IrRecordShape,
   IrStmt,
@@ -1210,17 +1211,28 @@ export function lowerToIr(
     });
   // A program that uses the WebAssembly API carries every module the build
   // can see and translate (wasm/translate.ts); the rest fail at run time
-  // with the CompileError for untranslated bytes.
+  // with a CompileError, which names the reason for the modules the build
+  // saw but does not translate.
   if (result.module !== null && resultLowerer.usesWebAssembly) {
     const bytesByHash = new Map<string, Uint8Array>();
     const reports = discoverModules((sha256, bytes) => bytesByHash.set(sha256, bytes));
     const wasmModules: IrWasmModule[] = [];
+    const wasmUntranslated: IrWasmUntranslated[] = [];
     for (const report of reports) {
       const bytes = bytesByHash.get(report.sha256);
-      if (report.status !== "valid" || bytes === undefined) continue;
-      wasmModules.push({ sha256: report.sha256, bytes: encodeBase64(bytes) });
+      if (bytes === undefined) continue;
+      if (report.status === "valid") {
+        wasmModules.push({ sha256: report.sha256, bytes: encodeBase64(bytes) });
+        continue;
+      }
+      const reason =
+        report.status === "unsupported"
+          ? `it uses ${report.unsupported.join(", ")}, outside the translated WebAssembly feature set`
+          : `it is not a valid module: ${report.error ?? report.status}`;
+      wasmUntranslated.push({ sha256: report.sha256, size: bytes.length, reason });
     }
     result.module.wasmModules = wasmModules;
+    result.module.wasmUntranslated = wasmUntranslated;
     if (resultLowerer.usesWasmCustomSections) result.module.wasmCustomSections = true;
     timing("wasm-discovery", { modules: wasmModules.length });
   }

packages/compiler/src/ir/function-references.test.ts

@@ -214,6 +214,8 @@ const MODULE_FIELDS: Record<StringFields<Omit<IrModule, "functions">>, Role> = {
   "contracts.programLifetime[]": "other",
   "wasmModules[].sha256": "other",
   "wasmModules[].bytes": "other",
+  "wasmUntranslated[].sha256": "other",
+  "wasmUntranslated[].reason": "other",
   "threadSlotKeys[]": "other",
 };
 

packages/compiler/src/ir/ir.ts

@@ -1052,6 +1052,10 @@ export interface IrModule {
    * API (possibly empty). The backend appends each module's translation
    * and emits the registry the runtime looks modules up in. */
   wasmModules?: IrWasmModule[];
+  /** The modules the build saw but does not translate, with the reason,
+   * present with wasmModules. The backend emits them in a second table, so
+   * loading one rejects with a CompileError that names the reason. */
+  wasmUntranslated?: IrWasmUntranslated[];
   /** The program calls WebAssembly.Module.customSections, so translated
    * modules keep their custom sections (otherwise they carry none, and
    * debug sections cost nothing). */
@@ -1067,6 +1071,16 @@ export interface IrWasmModule {
   bytes: string;
 }
 
+/** A WebAssembly module whose bytes the build saw but does not translate. */
+export interface IrWasmUntranslated {
+  /** Hex SHA-256 of the bytes. */
+  sha256: string;
+  /** The byte length. */
+  size: number;
+  /** Why the module is not translated, as a clause ("it uses simd, ..."). */
+  reason: string;
+}
+
 /** Verified contract facts, keyed by IR names. */
 export interface IrContracts {
   /** Int32 class field families: `${topmost declaring class}\0${field}`. */

packages/compiler/src/wasm/translate.ts

@@ -211,7 +211,7 @@ export function signatureHash(sig: string): string {
 }
 
 /** UTF-8 bytes of a string. */
-function utf8Bytes(text: string): number[] {
+export function utf8Bytes(text: string): number[] {
   const out: number[] = [];
   for (let i = 0; i < text.length; i++) {
     const cp = text.codePointAt(i)!;
@@ -233,7 +233,7 @@ function utf8Bytes(text: string): number[] {
 }
 
 /** An LLVM c"..." string body for bytes. */
-function cBytes(bytes: Uint8Array, start: number, end: number): string {
+export function cBytes(bytes: Uint8Array, start: number, end: number): string {
   let s = "";
   for (let i = start; i < end; i++) {
     const b = bytes[i]!;

packages/runtime/src/scr_wasm.c

@@ -5,7 +5,9 @@
  * WebAssembly.compile, instantiate, validate and new Module hash the bytes
  * they are given and look the module up by SHA-256 and length in
  * scr_wasm_registry, which the compiler emits; bytes that were not
- * translated at build time reject with a CompileError that says so.
+ * translated at build time reject with a CompileError that says so, and
+ * that names the reason for modules the build saw but does not translate
+ * (scr_wasm_untranslated).
  * Nothing is interpreted or compiled at run time, and a translated
  * module's bytes were validated when the program was built.
  *
@@ -1668,12 +1670,35 @@ static const ScrWasmModuleDesc *scr_wasm_lookup(const uint8_t *data, size_t len)
   return NULL;
 }
 
-static void scr_wasm_not_translated(const char *what) {
-  char msg[400];
-  snprintf(msg, sizeof msg,
-           "%s: this module was not translated when the program was built (scriptc compiles WebAssembly ahead of "
-           "time and cannot compile modules whose bytes it did not see; see https://scriptc.dev/limitations)",
-           what);
+/* The CompileError for bytes the registry does not hold: the reason the
+ * build recorded for a module it saw but does not translate, or the
+ * message for bytes it did not see. */
+static void scr_wasm_not_translated(const char *what, const uint8_t *data, size_t len) {
+  const ScrWasmUntranslatedDesc *found = NULL;
+  uint8_t digest[32];
+  bool hashed = false;
+  for (uint64_t i = 0; i < scr_wasm_untranslated.count && !found; i++) {
+    const ScrWasmUntranslatedDesc *d = &scr_wasm_untranslated.modules[i];
+    if (d->byte_length != len) continue;
+    if (!hashed) {
+      scr_wasm_sha256(data, len, digest);
+      hashed = true;
+    }
+    if (!memcmp(d->sha256, digest, 32)) found = d;
+  }
+  char msg[800];
+  if (found) {
+    int reason_len = found->reason_len > 600 ? 600 : (int)found->reason_len;
+    snprintf(msg, sizeof msg,
+             "%s: this module was found when the program was built but not translated (%.*s; see "
+             "https://scriptc.dev/limitations)",
+             what, reason_len, found->reason);
+  } else {
+    snprintf(msg, sizeof msg,
+             "%s: this module was not translated when the program was built (scriptc compiles WebAssembly ahead of "
+             "time and cannot compile modules whose bytes it did not see; see https://scriptc.dev/limitations)",
+             what);
+  }
   scr_wasm_throw_kind(SCR_WASM_COMPILE_ERROR, msg);
 }
 
@@ -1683,7 +1708,7 @@ static ScrDyn *scr_wasm_module_from(ScrDyn *bytes, const char *what, const char
   if (!scr_wasm_bytes_of(bytes, what, expected, &data, &len)) return NULL;
   const ScrWasmModuleDesc *desc = scr_wasm_lookup(data, len);
   if (!desc) {
-    scr_wasm_not_translated(what);
+    scr_wasm_not_translated(what, data, len);
     return NULL;
   }
   ScrWasmObj *o = scr_wasm_obj_new(SCR_WASM_K_MODULE);

packages/runtime/src/scr_wasm_abi.h

@@ -174,6 +174,22 @@ typedef struct {
 } ScrWasmRegistry;
 extern const ScrWasmRegistry scr_wasm_registry;
 
+/* The modules the build saw but does not translate, emitted next to the
+ * registry: unsupported proposals, invalid modules and translator
+ * limits. `reason` is a clause naming why (not NUL-terminated). */
+typedef struct {
+  uint8_t sha256[32];
+  uint64_t byte_length;
+  const char *reason;
+  uint64_t reason_len;
+} ScrWasmUntranslatedDesc;
+
+typedef struct {
+  uint64_t count;
+  const ScrWasmUntranslatedDesc *modules;
+} ScrWasmUntranslatedTable;
+extern const ScrWasmUntranslatedTable scr_wasm_untranslated;
+
 /* Runtime objects the translated code reads at fixed offsets. */
 typedef struct ScrWasmMemory {
   uint8_t *base;      /* 0 */

tests/harness/webassembly-untranslated.test.ts

@@ -0,0 +1,97 @@
+/* The CompileError for WebAssembly bytes that have no translation —
+ * scriptc-only.
+ *
+ * A module the build finds but does not translate (one that uses a
+ * proposal outside the translated feature set, or an invalid one) rejects
+ * with a CompileError that names the reason the build recorded. Bytes the
+ * build did not see reject with the message for unseen bytes. Both stay
+ * catchable CompileErrors, and WebAssembly.validate answers false for
+ * them.
+ */
+import { execFile } from "node:child_process";
+import { createHash } from "node:crypto";
+import { mkdirSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { promisify } from "node:util";
+import { describe, expect, test } from "vitest";
+import { compile } from "@scriptc/compiler";
+
+const execFileAsync = promisify(execFile);
+const repoRoot = join(import.meta.dirname, "../..");
+const cacheDir = join(repoRoot, "node_modules/.cache/scriptc-tests");
+const sanitize = process.env["SCRIPTC_SAN"] === "1";
+
+// (module (func i32.const 0 i8x16.splat drop))
+const simdModule = "AGFzbQEAAAABBAFgAAADAgEACgkBBwBBAP0PGgs=";
+
+const source = `const simd = Buffer.from("${simdModule}", "base64");
+// A type section whose entry is not a function type, so the module is
+// malformed.
+const invalid = new Uint8Array([0, 97, 115, 109, 1, 0, 0, 0, 1, 2, 1, 64]);
+// A custom section whose last byte is computed when the program runs, so
+// the build does not see these bytes.
+const unseen = Uint8Array.from([0, 97, 115, 109, 1, 0, 0, 0, 0, 4, 2, 97, 98, process.argv.length]);
+function load(label, bytes) {
+  console.log(label, WebAssembly.validate(bytes));
+  try {
+    new WebAssembly.Module(bytes);
+  } catch (e) {
+    console.log(label, e instanceof WebAssembly.CompileError, e.message);
+  }
+}
+load("simd", simd);
+load("invalid", invalid);
+load("unseen", unseen);
+WebAssembly.compile(simd).catch((e) => console.log("compile", e instanceof WebAssembly.CompileError, e.message));
+`;
+
+async function run(): Promise<string> {
+  const key = createHash("sha256")
+    .update(source)
+    .update(sanitize ? "san" : "plain")
+    .digest("hex")
+    .slice(0, 16);
+  const outDir = join(cacheDir, `wasm-untranslated-${key}`);
+  mkdirSync(outDir, { recursive: true });
+  const file = join(outDir, "untranslated.mjs");
+  writeFileSync(file, source);
+  const result = await compile(file, {
+    outPath: join(outDir, "untranslated"),
+    outDir,
+    sanitize,
+    backend: "llvm",
+  });
+  if (!result.ok) {
+    throw new Error(
+      "program failed to compile:\n" +
+        result.diagnostics.map((d) => `${d.code}: ${d.message}`).join("\n"),
+    );
+  }
+  const { stdout } = await execFileAsync(result.binaryPath, [], { encoding: "utf8" });
+  return stdout;
+}
+
+describe(`WebAssembly modules without a translation${sanitize ? " (sanitized)" : ""}`, () => {
+  test("the CompileError names why a module the build found is not translated", async () => {
+    const lines = (await run()).trimEnd().split("\n");
+    const found = (what: string, reason: string) =>
+      `${what}: this module was found when the program was built but not translated (${reason}; see https://scriptc.dev/limitations)`;
+    const unseen =
+      "WebAssembly.Module(): this module was not translated when the program was built (scriptc compiles " +
+      "WebAssembly ahead of time and cannot compile modules whose bytes it did not see; see https://scriptc.dev/limitations)";
+    expect(lines[0]).toBe("simd false");
+    expect(lines[1]).toBe(
+      `simd true ${found("WebAssembly.Module()", "it uses simd, outside the translated WebAssembly feature set")}`,
+    );
+    expect(lines[2]).toBe("invalid false");
+    expect(lines[3]).toBe(
+      `invalid true ${found("WebAssembly.Module()", "it is not a valid module: malformed function type (at byte 12)")}`,
+    );
+    expect(lines[4]).toBe("unseen false");
+    expect(lines[5]).toBe(`unseen true ${unseen}`);
+    expect(lines[6]).toBe(
+      `compile true ${found("WebAssembly.compile()", "it uses simd, outside the translated WebAssembly feature set")}`,
+    );
+    expect(lines).toHaveLength(7);
+  }, 180_000);
+});