diff --git a/docs/content/limitations/webassembly-modules.mdx b/docs/content/limitations/webassembly-modules.mdx index 43fa55e48..b74f2094d 100644 --- a/docs/content/limitations/webassembly-modules.mdx +++ b/docs/content/limitations/webassembly-modules.mdx @@ -5,7 +5,8 @@ Bytes the build did not see, including bytes produced at run time, reject with a `scriptc coverage` lists the modules a build finds, with their size, whether they validate, and the WebAssembly proposals they use. Translated modules can use the WebAssembly 2.0 feature set: the MVP plus sign-extension operators, non-trapping float-to-int conversions, bulk memory, multi-value, mutable globals, reference types, and extended constant expressions. -Modules that use SIMD, threads, exception handling, tail calls, typed function references or GC, memory64, or multiple memories are not translated, so loading them rejects with the `CompileError` above; packages that fall back to a scalar build when that happens, such as undici and es-module-lexer, take their fallback. +Modules that use SIMD, threads, exception handling, tail calls, typed function references or GC, memory64, or multiple memories are not translated, so loading them rejects with a `CompileError` that names the proposal the module uses, and `WebAssembly.validate` answers `false` for them; packages that fall back to a scalar build when that happens, such as undici and es-module-lexer, take their fallback. +Invalid modules the build finds reject with a `CompileError` that includes the validation error. Supported: `WebAssembly.compile`, `instantiate` (with bytes or a `Module`), `validate`, `new WebAssembly.Module`, `new WebAssembly.Instance`, `new WebAssembly.Memory`, `new WebAssembly.Table`, `new WebAssembly.Global`, `WebAssembly.Module.imports`, `exports` and `customSections`, and `instanceof` for every class; the error classes `CompileError`, `LinkError` and `RuntimeError`, constructed with or without `new`; `globalThis.WebAssembly` and its `compile`, `instantiate` and `validate` functions as values. Instances expose exported functions, memories (`buffer`, `grow`), tables (`get`, `set`, `grow`, `length`) and globals (`value`, `valueOf`). diff --git a/packages/compiler/src/backend/llvm/wasm-modules.ts b/packages/compiler/src/backend/llvm/wasm-modules.ts index 8a7045687..d08497214 100644 --- a/packages/compiler/src/backend/llvm/wasm-modules.ts +++ b/packages/compiler/src/backend/llvm/wasm-modules.ts @@ -1,12 +1,18 @@ /* The WebAssembly modules translated into a program (IrModule.wasmModules): - * each module's translation (wasm/translate.ts) and the registry the - * runtime looks modules up in (scr_wasm_registry, scr_wasm_abi.h). The + * each module's translation (wasm/translate.ts), the registry the runtime + * looks modules up in (scr_wasm_registry, scr_wasm_abi.h), and the table of + * modules the build saw but does not translate (scr_wasm_untranslated). The * translations join the program module, so the native helper optimizes, * partitions and caches them with it. Programs that do not use the * WebAssembly API carry no wasmModules and emit nothing here. */ -import type { IrModule } from "../../ir/ir.js"; +import type { IrModule, IrWasmUntranslated } from "../../ir/ir.js"; import { decodeBase64 } from "../../wasm/base64.js"; -import { translateWasmModule, WasmTranslateError } from "../../wasm/translate.js"; +import { + cBytes, + translateWasmModule, + utf8Bytes, + WasmTranslateError, +} from "../../wasm/translate.js"; function hexBytes(hex: string): Uint8Array { const out = new Uint8Array(hex.length / 2); @@ -28,11 +34,38 @@ export function wasmBoundsChecks(options: { return /windows|mingw|cygwin|win32|msvc/.test(target); } -/** The top-level lines of every translated module plus the registry. - * `declared` holds the declarations the module already carries; runtime - * entry points it lacks are declared here. A module the translator - * cannot translate is left out of the registry, so loading it at run - * time rejects with the CompileError for untranslated bytes. */ +/** scr_wasm_untranslated: hash, byte length and reason per module. */ +function emitUntranslated(untranslated: readonly IrWasmUntranslated[]): string[] { + if (untranslated.length === 0) + return [`@scr_wasm_untranslated = constant { i64, ptr } { i64 0, ptr null }`]; + const out: string[] = []; + const entries: string[] = []; + for (let index = 0; index < untranslated.length; index++) { + const module = untranslated[index]!; + const reason = new Uint8Array(utf8Bytes(module.reason)); + out.push( + `@sc_wasm_untranslated_reason${index} = private unnamed_addr constant [${reason.length} x i8] c"${cBytes(reason, 0, reason.length)}"`, + ); + const digest = hexBytes(module.sha256); + const sha: string[] = []; + for (let i = 0; i < 32; i++) sha.push(`i8 ${digest[i]!}`); + entries.push( + `{ [32 x i8], i64, ptr, i64 } { [32 x i8] [${sha.join(", ")}], i64 ${module.size}, ptr @sc_wasm_untranslated_reason${index}, i64 ${reason.length} }`, + ); + } + out.push( + `@sc_wasm_untranslated_modules = private constant [${entries.length} x { [32 x i8], i64, ptr, i64 }] [${entries.join(", ")}]`, + `@scr_wasm_untranslated = constant { i64, ptr } { i64 ${entries.length}, ptr @sc_wasm_untranslated_modules }`, + ); + return out; +} + +/** The top-level lines of every translated module plus the registry and + * the untranslated table. `declared` holds the declarations the module + * already carries; runtime entry points it lacks are declared here. A + * module the translator cannot translate joins the untranslated table, so + * loading it at run time rejects with a CompileError that names the + * translator's reason. */ export function emitWasmModules( mod: IrModule, declared: ReadonlySet, @@ -43,6 +76,7 @@ export function emitWasmModules( const out: string[] = []; const decls = new Set(declared); const descriptors: string[] = []; + const untranslated = [...(mod.wasmUntranslated ?? [])]; for (const module of modules) { const bytes = decodeBase64(module.bytes); if (bytes === null) continue; @@ -53,8 +87,9 @@ export function emitWasmModules( customSections: mod.wasmCustomSections === true, }); } catch (error) { - if (error instanceof WasmTranslateError) continue; - throw error; + if (!(error instanceof WasmTranslateError)) throw error; + untranslated.push({ sha256: module.sha256, size: bytes.length, reason: error.message }); + continue; } for (const decl of translation.decls) { if (decls.has(decl)) continue; @@ -72,6 +107,7 @@ export function emitWasmModules( `@scr_wasm_registry = constant { i64, ptr } { i64 ${descriptors.length}, ptr @sc_wasm_registry_modules }`, ); } + out.push(...emitUntranslated(untranslated)); out.push(""); return out; } diff --git a/packages/compiler/src/frontend/lowering/lowerer.ts b/packages/compiler/src/frontend/lowering/lowerer.ts index 543149f83..896f158a1 100644 --- a/packages/compiler/src/frontend/lowering/lowerer.ts +++ b/packages/compiler/src/frontend/lowering/lowerer.ts @@ -210,6 +210,7 @@ import type { IrLocal, IrModule, IrWasmModule, + IrWasmUntranslated, IrParam, IrRecordShape, IrStmt, @@ -1210,17 +1211,28 @@ export function lowerToIr( }); // A program that uses the WebAssembly API carries every module the build // can see and translate (wasm/translate.ts); the rest fail at run time - // with the CompileError for untranslated bytes. + // with a CompileError, which names the reason for the modules the build + // saw but does not translate. if (result.module !== null && resultLowerer.usesWebAssembly) { const bytesByHash = new Map(); const reports = discoverModules((sha256, bytes) => bytesByHash.set(sha256, bytes)); const wasmModules: IrWasmModule[] = []; + const wasmUntranslated: IrWasmUntranslated[] = []; for (const report of reports) { const bytes = bytesByHash.get(report.sha256); - if (report.status !== "valid" || bytes === undefined) continue; - wasmModules.push({ sha256: report.sha256, bytes: encodeBase64(bytes) }); + if (bytes === undefined) continue; + if (report.status === "valid") { + wasmModules.push({ sha256: report.sha256, bytes: encodeBase64(bytes) }); + continue; + } + const reason = + report.status === "unsupported" + ? `it uses ${report.unsupported.join(", ")}, outside the translated WebAssembly feature set` + : `it is not a valid module: ${report.error ?? report.status}`; + wasmUntranslated.push({ sha256: report.sha256, size: bytes.length, reason }); } result.module.wasmModules = wasmModules; + result.module.wasmUntranslated = wasmUntranslated; if (resultLowerer.usesWasmCustomSections) result.module.wasmCustomSections = true; timing("wasm-discovery", { modules: wasmModules.length }); } diff --git a/packages/compiler/src/ir/function-references.test.ts b/packages/compiler/src/ir/function-references.test.ts index 4ecc60a6f..7bef40aa6 100644 --- a/packages/compiler/src/ir/function-references.test.ts +++ b/packages/compiler/src/ir/function-references.test.ts @@ -214,6 +214,8 @@ const MODULE_FIELDS: Record>, Role> = { "contracts.programLifetime[]": "other", "wasmModules[].sha256": "other", "wasmModules[].bytes": "other", + "wasmUntranslated[].sha256": "other", + "wasmUntranslated[].reason": "other", "threadSlotKeys[]": "other", }; diff --git a/packages/compiler/src/ir/ir.ts b/packages/compiler/src/ir/ir.ts index 359d45ee7..603b9d0ae 100644 --- a/packages/compiler/src/ir/ir.ts +++ b/packages/compiler/src/ir/ir.ts @@ -1052,6 +1052,10 @@ export interface IrModule { * API (possibly empty). The backend appends each module's translation * and emits the registry the runtime looks modules up in. */ wasmModules?: IrWasmModule[]; + /** The modules the build saw but does not translate, with the reason, + * present with wasmModules. The backend emits them in a second table, so + * loading one rejects with a CompileError that names the reason. */ + wasmUntranslated?: IrWasmUntranslated[]; /** The program calls WebAssembly.Module.customSections, so translated * modules keep their custom sections (otherwise they carry none, and * debug sections cost nothing). */ @@ -1067,6 +1071,16 @@ export interface IrWasmModule { bytes: string; } +/** A WebAssembly module whose bytes the build saw but does not translate. */ +export interface IrWasmUntranslated { + /** Hex SHA-256 of the bytes. */ + sha256: string; + /** The byte length. */ + size: number; + /** Why the module is not translated, as a clause ("it uses simd, ..."). */ + reason: string; +} + /** Verified contract facts, keyed by IR names. */ export interface IrContracts { /** Int32 class field families: `${topmost declaring class}\0${field}`. */ diff --git a/packages/compiler/src/wasm/translate.ts b/packages/compiler/src/wasm/translate.ts index 4c02499ca..549b51a43 100644 --- a/packages/compiler/src/wasm/translate.ts +++ b/packages/compiler/src/wasm/translate.ts @@ -211,7 +211,7 @@ export function signatureHash(sig: string): string { } /** UTF-8 bytes of a string. */ -function utf8Bytes(text: string): number[] { +export function utf8Bytes(text: string): number[] { const out: number[] = []; for (let i = 0; i < text.length; i++) { const cp = text.codePointAt(i)!; @@ -233,7 +233,7 @@ function utf8Bytes(text: string): number[] { } /** An LLVM c"..." string body for bytes. */ -function cBytes(bytes: Uint8Array, start: number, end: number): string { +export function cBytes(bytes: Uint8Array, start: number, end: number): string { let s = ""; for (let i = start; i < end; i++) { const b = bytes[i]!; diff --git a/packages/runtime/src/scr_wasm.c b/packages/runtime/src/scr_wasm.c index 121acacbb..885f6adfe 100644 --- a/packages/runtime/src/scr_wasm.c +++ b/packages/runtime/src/scr_wasm.c @@ -5,7 +5,9 @@ * WebAssembly.compile, instantiate, validate and new Module hash the bytes * they are given and look the module up by SHA-256 and length in * scr_wasm_registry, which the compiler emits; bytes that were not - * translated at build time reject with a CompileError that says so. + * translated at build time reject with a CompileError that says so, and + * that names the reason for modules the build saw but does not translate + * (scr_wasm_untranslated). * Nothing is interpreted or compiled at run time, and a translated * module's bytes were validated when the program was built. * @@ -1668,12 +1670,35 @@ static const ScrWasmModuleDesc *scr_wasm_lookup(const uint8_t *data, size_t len) return NULL; } -static void scr_wasm_not_translated(const char *what) { - char msg[400]; - snprintf(msg, sizeof msg, - "%s: this module was not translated when the program was built (scriptc compiles WebAssembly ahead of " - "time and cannot compile modules whose bytes it did not see; see https://scriptc.dev/limitations)", - what); +/* The CompileError for bytes the registry does not hold: the reason the + * build recorded for a module it saw but does not translate, or the + * message for bytes it did not see. */ +static void scr_wasm_not_translated(const char *what, const uint8_t *data, size_t len) { + const ScrWasmUntranslatedDesc *found = NULL; + uint8_t digest[32]; + bool hashed = false; + for (uint64_t i = 0; i < scr_wasm_untranslated.count && !found; i++) { + const ScrWasmUntranslatedDesc *d = &scr_wasm_untranslated.modules[i]; + if (d->byte_length != len) continue; + if (!hashed) { + scr_wasm_sha256(data, len, digest); + hashed = true; + } + if (!memcmp(d->sha256, digest, 32)) found = d; + } + char msg[800]; + if (found) { + int reason_len = found->reason_len > 600 ? 600 : (int)found->reason_len; + snprintf(msg, sizeof msg, + "%s: this module was found when the program was built but not translated (%.*s; see " + "https://scriptc.dev/limitations)", + what, reason_len, found->reason); + } else { + snprintf(msg, sizeof msg, + "%s: this module was not translated when the program was built (scriptc compiles WebAssembly ahead of " + "time and cannot compile modules whose bytes it did not see; see https://scriptc.dev/limitations)", + what); + } scr_wasm_throw_kind(SCR_WASM_COMPILE_ERROR, msg); } @@ -1683,7 +1708,7 @@ static ScrDyn *scr_wasm_module_from(ScrDyn *bytes, const char *what, const char if (!scr_wasm_bytes_of(bytes, what, expected, &data, &len)) return NULL; const ScrWasmModuleDesc *desc = scr_wasm_lookup(data, len); if (!desc) { - scr_wasm_not_translated(what); + scr_wasm_not_translated(what, data, len); return NULL; } ScrWasmObj *o = scr_wasm_obj_new(SCR_WASM_K_MODULE); diff --git a/packages/runtime/src/scr_wasm_abi.h b/packages/runtime/src/scr_wasm_abi.h index 083a2f77a..d90deabc3 100644 --- a/packages/runtime/src/scr_wasm_abi.h +++ b/packages/runtime/src/scr_wasm_abi.h @@ -174,6 +174,22 @@ typedef struct { } ScrWasmRegistry; extern const ScrWasmRegistry scr_wasm_registry; +/* The modules the build saw but does not translate, emitted next to the + * registry: unsupported proposals, invalid modules and translator + * limits. `reason` is a clause naming why (not NUL-terminated). */ +typedef struct { + uint8_t sha256[32]; + uint64_t byte_length; + const char *reason; + uint64_t reason_len; +} ScrWasmUntranslatedDesc; + +typedef struct { + uint64_t count; + const ScrWasmUntranslatedDesc *modules; +} ScrWasmUntranslatedTable; +extern const ScrWasmUntranslatedTable scr_wasm_untranslated; + /* Runtime objects the translated code reads at fixed offsets. */ typedef struct ScrWasmMemory { uint8_t *base; /* 0 */ diff --git a/tests/harness/webassembly-untranslated.test.ts b/tests/harness/webassembly-untranslated.test.ts new file mode 100644 index 000000000..a12c75a19 --- /dev/null +++ b/tests/harness/webassembly-untranslated.test.ts @@ -0,0 +1,97 @@ +/* The CompileError for WebAssembly bytes that have no translation — + * scriptc-only. + * + * A module the build finds but does not translate (one that uses a + * proposal outside the translated feature set, or an invalid one) rejects + * with a CompileError that names the reason the build recorded. Bytes the + * build did not see reject with the message for unseen bytes. Both stay + * catchable CompileErrors, and WebAssembly.validate answers false for + * them. + */ +import { execFile } from "node:child_process"; +import { createHash } from "node:crypto"; +import { mkdirSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { promisify } from "node:util"; +import { describe, expect, test } from "vitest"; +import { compile } from "@scriptc/compiler"; + +const execFileAsync = promisify(execFile); +const repoRoot = join(import.meta.dirname, "../.."); +const cacheDir = join(repoRoot, "node_modules/.cache/scriptc-tests"); +const sanitize = process.env["SCRIPTC_SAN"] === "1"; + +// (module (func i32.const 0 i8x16.splat drop)) +const simdModule = "AGFzbQEAAAABBAFgAAADAgEACgkBBwBBAP0PGgs="; + +const source = `const simd = Buffer.from("${simdModule}", "base64"); +// A type section whose entry is not a function type, so the module is +// malformed. +const invalid = new Uint8Array([0, 97, 115, 109, 1, 0, 0, 0, 1, 2, 1, 64]); +// A custom section whose last byte is computed when the program runs, so +// the build does not see these bytes. +const unseen = Uint8Array.from([0, 97, 115, 109, 1, 0, 0, 0, 0, 4, 2, 97, 98, process.argv.length]); +function load(label, bytes) { + console.log(label, WebAssembly.validate(bytes)); + try { + new WebAssembly.Module(bytes); + } catch (e) { + console.log(label, e instanceof WebAssembly.CompileError, e.message); + } +} +load("simd", simd); +load("invalid", invalid); +load("unseen", unseen); +WebAssembly.compile(simd).catch((e) => console.log("compile", e instanceof WebAssembly.CompileError, e.message)); +`; + +async function run(): Promise { + const key = createHash("sha256") + .update(source) + .update(sanitize ? "san" : "plain") + .digest("hex") + .slice(0, 16); + const outDir = join(cacheDir, `wasm-untranslated-${key}`); + mkdirSync(outDir, { recursive: true }); + const file = join(outDir, "untranslated.mjs"); + writeFileSync(file, source); + const result = await compile(file, { + outPath: join(outDir, "untranslated"), + outDir, + sanitize, + backend: "llvm", + }); + if (!result.ok) { + throw new Error( + "program failed to compile:\n" + + result.diagnostics.map((d) => `${d.code}: ${d.message}`).join("\n"), + ); + } + const { stdout } = await execFileAsync(result.binaryPath, [], { encoding: "utf8" }); + return stdout; +} + +describe(`WebAssembly modules without a translation${sanitize ? " (sanitized)" : ""}`, () => { + test("the CompileError names why a module the build found is not translated", async () => { + const lines = (await run()).trimEnd().split("\n"); + const found = (what: string, reason: string) => + `${what}: this module was found when the program was built but not translated (${reason}; see https://scriptc.dev/limitations)`; + const unseen = + "WebAssembly.Module(): this module was not translated when the program was built (scriptc compiles " + + "WebAssembly ahead of time and cannot compile modules whose bytes it did not see; see https://scriptc.dev/limitations)"; + expect(lines[0]).toBe("simd false"); + expect(lines[1]).toBe( + `simd true ${found("WebAssembly.Module()", "it uses simd, outside the translated WebAssembly feature set")}`, + ); + expect(lines[2]).toBe("invalid false"); + expect(lines[3]).toBe( + `invalid true ${found("WebAssembly.Module()", "it is not a valid module: malformed function type (at byte 12)")}`, + ); + expect(lines[4]).toBe("unseen false"); + expect(lines[5]).toBe(`unseen true ${unseen}`); + expect(lines[6]).toBe( + `compile true ${found("WebAssembly.compile()", "it uses simd, outside the translated WebAssembly feature set")}`, + ); + expect(lines).toHaveLength(7); + }, 180_000); +});