Merge main

benchmarks/runtime/README.md

@@ -30,6 +30,7 @@ Application-shaped workloads for measuring the speed of compiled executables. Ea
 | `validate-errors` | cpu | throw/catch on a fraction of inputs, small call chains |
 | `csv-numbers` | cpu | `split`, `Number`, `parseInt`, `toFixed` |
 | `log-lines` | cpu | 200k immediate `console.log`/`process.stdout.write` lines into the runner's pipe |
+| `unicode-lexer` | cpu | a hand-written lexer over a 2.7M-unit bundled module whose comments and strings are in several languages: `charCodeAt` scans, line starts |
 | `log-summary`, `inventory-report` | cpu | the multi-module applications from `benchmarks/builds`, on 300k-line inputs |
 | `cli-config` | startup | a `deploy-plan` CLI: read a ~110 KB JSON manifest, validate, interpolate env values, order 80 services into dependency waves, print a summary |
 | `http-api` | server | a `node:http` JSON API: routing, query parsing, a seeded catalog, per-cart state, typed JSON bodies, `ETag`/`If-None-Match`, auth and request-id headers |

benchmarks/runtime/workloads.json

@@ -231,6 +231,13 @@
         "4"
       ]
     },
+    {
+      "name": "unicode-lexer",
+      "entry": "workloads/unicode-lexer.ts",
+      "args": [
+        "1"
+      ]
+    },
     {
       "name": "log-summary",
       "entry": "../builds/log-summary/main.ts",

benchmarks/runtime/workloads/unicode-lexer.ts

@@ -0,0 +1,116 @@
+// A lexer over a large bundled module whose comments and strings are in
+// several languages: line starts, tokens by kind and comment text, read
+// with charCodeAt as hand-written scanners do. Each pass relexes the whole
+// text, as a bundler or an editor does after a change.
+const comments = [
+  "// Berechnet die Größe des Puffers für übergroße Einträge",
+  "// Calcule la taille maximale avant la prochaine itération",
+  "// バッファのサイズを計算します",
+  "/* 计算缓冲区大小，然后返回结果 */",
+  "// Returns the cached value (fast path) — see the note above",
+  "/** Résumé: the état of the café's ñandú registry */",
+];
+const strings = ['"Grüße"', '"naïve façade"', '"東京"', '"ok ✓"', '"plain ascii"', "'mañana'"];
+const names = ["buffer", "größe", "index", "resultat", "count", "value", "entry", "table"];
+
+let seed = 11;
+function random(): number {
+  seed = (Math.imul(seed, 1103515245) + 12345) >>> 0;
+  return seed / 4294967296;
+}
+function pick(list: readonly string[]): string {
+  return list[Math.floor(random() * list.length)]!;
+}
+
+function makeSource(lines: number): string {
+  const out: string[] = [];
+  for (let i = 0; i < lines; i++) {
+    const r = random();
+    if (r < 0.2) out.push(pick(comments));
+    else if (r < 0.6) out.push(`  const ${pick(names)}${i} = ${pick(names)}(${Math.floor(random() * 1000)}, ${pick(strings)});`);
+    else if (r < 0.85) out.push(`  if (${pick(names)} >= ${i % 97}) return ${pick(names)}.${pick(names)} + 1;`);
+    else out.push(`function ${pick(names)}${i}(a, b) { return a * b - ${i}; }`);
+  }
+  return out.join("\n");
+}
+
+interface Counts {
+  lines: number;
+  identifiers: number;
+  numbers: number;
+  strings: number;
+  comments: number;
+  commentChars: number;
+  punctuation: number;
+}
+
+function isIdentifierStart(ch: number): boolean {
+  return (ch >= 97 && ch <= 122) || (ch >= 65 && ch <= 90) || ch === 95 || ch === 36 || ch > 127;
+}
+
+function isIdentifierPart(ch: number): boolean {
+  return isIdentifierStart(ch) || (ch >= 48 && ch <= 57);
+}
+
+function lex(text: string): Counts {
+  const counts: Counts = { lines: 1, identifiers: 0, numbers: 0, strings: 0, comments: 0, commentChars: 0, punctuation: 0 };
+  let pos = 0;
+  const end = text.length;
+  while (pos < end) {
+    const ch = text.charCodeAt(pos);
+    if (ch === 10) {
+      counts.lines++;
+      pos++;
+    } else if (ch === 32 || ch === 9 || ch === 13) {
+      pos++;
+    } else if (ch === 47 && text.charCodeAt(pos + 1) === 47) {
+      const start = pos;
+      while (pos < end && text.charCodeAt(pos) !== 10) pos++;
+      counts.comments++;
+      counts.commentChars += pos - start;
+    } else if (ch === 47 && text.charCodeAt(pos + 1) === 42) {
+      const start = pos;
+      pos += 2;
+      while (pos < end && !(text.charCodeAt(pos) === 42 && text.charCodeAt(pos + 1) === 47)) {
+        if (text.charCodeAt(pos) === 10) counts.lines++;
+        pos++;
+      }
+      pos += 2;
+      counts.comments++;
+      counts.commentChars += pos - start;
+    } else if (ch === 34 || ch === 39) {
+      pos++;
+      while (pos < end && text.charCodeAt(pos) !== ch) pos++;
+      pos++;
+      counts.strings++;
+    } else if (ch >= 48 && ch <= 57) {
+      while (pos < end && text.charCodeAt(pos) >= 48 && text.charCodeAt(pos) <= 57) pos++;
+      counts.numbers++;
+    } else if (isIdentifierStart(ch)) {
+      while (pos < end && isIdentifierPart(text.charCodeAt(pos))) pos++;
+      counts.identifiers++;
+    } else {
+      counts.punctuation++;
+      pos++;
+    }
+  }
+  return counts;
+}
+
+function lineStarts(text: string): number[] {
+  const starts = [0];
+  for (let pos = 0; pos < text.length; pos++) if (text.charCodeAt(pos) === 10) starts.push(pos + 1);
+  return starts;
+}
+
+const scale = Number(process.argv[2] ?? "1");
+const source = makeSource(60000);
+const passes = Math.round(6 * scale);
+let last = lex(source);
+let total = last.identifiers + lineStarts(source).length;
+for (let i = 1; i < passes; i++) {
+  last = lex(source);
+  total += last.identifiers + lineStarts(source).length;
+}
+console.log("units", source.length, "passes", passes, "total", total);
+console.log(JSON.stringify(last));

packages/compiler/src/backend/llvm/dyn.ts

@@ -39,6 +39,7 @@ import {
   isDirentShape,
   isDynTypedRefType,
   isRefCounted,
+  isUnitType,
   typeKey,
 } from "../../ir/ir.js";
 import { dynDesc, streamTypedRefEligible } from "../../ir/analysis.js";
@@ -697,6 +698,46 @@ export class LlDyn {
 
   /** Emit the standard in-helper pending check: on a pending exception
    * run `cleanup` and return `dummy`. */
+  /** The failure block of a union dynCheck whose value matched no arm. A
+   * value of the one container kind exactly one arm can take (an object
+   * for the union's only record arm, an array for its only array arm,
+   * beside unit and scalar arms) runs that arm's builder, which throws the
+   * deepest failing path and the reason there (`expected string at
+   * $.order.owner.id, got undefined`) instead of the union's own outer
+   * path. Other values, and an arm builder that unexpectedly succeeds,
+   * fall through to the union's message. Only the failure path changes. */
+  private unionArmFailure(B: BlockBuilder, def: IrUnionDef, kind: string): void {
+    const scalar = (arm: IrType): boolean =>
+      isUnitType(arm) ||
+      arm.kind === "f64" ||
+      arm.kind === "bool" ||
+      arm.kind === "string" ||
+      arm.kind === "bigint";
+    const containers = def.arms.filter((arm) => !scalar(arm));
+    const arm = containers.length === 1 ? containers[0]! : null;
+    const container =
+      arm?.kind === "array"
+        ? DYN_KIND.ARR
+        : arm?.kind === "record" && !this.host.recordsById.get(arm.shapeId)?.tuple
+          ? DYN_KIND.OBJ
+          : null;
+    if (!arm || container === null) return;
+    const host = this.host;
+    const deep = B.newLabel("dcu.deep");
+    const outer = B.newLabel("dcu.outer");
+    const same = B.tmp();
+    B.line(`${same} = icmp eq i32 ${kind}, ${container}`);
+    B.condBr(same, deep, outer);
+    B.startBlock(deep);
+    const built = B.tmp();
+    B.line(`${built} = call ptr @${this.dynCheckHelper(arm)}(ptr %d, ptr %path)`);
+    this.pendingBail(B, "dcu.deep", () => {}, "ptr null");
+    const rc = vAdapters(host, arm);
+    B.line(`call void ${rc.release}(ptr ${built})`);
+    B.br(outer);
+    B.startBlock(outer);
+  }
+
   private pendingBail(B: BlockBuilder, hint: string, cleanup: () => void, dummy: string): void {
     const p = B.tmp();
     for (const line of this.host.pendingTestLines(p)) B.line(line.trimStart());
@@ -2291,6 +2332,7 @@ export class LlDyn {
           B.line(`call void @scr_dyn_release_v(ptr ${materialized})`);
           B.terminate(`ret ptr ${checked}`);
           B.startBlock(lFail);
+          this.unionArmFailure(B, def, kind);
           B.line(`call void @scr_dyn_check_fail(ptr %path, ptr ${want}, ptr %d)`);
           B.terminate(`ret ptr null`);
           break;
@@ -2359,6 +2401,7 @@ export class LlDyn {
         );
         B.terminate(`ret ptr ${checked}`);
         B.startBlock(lFail);
+        this.unionArmFailure(B, def, kind);
         B.line(`call void @scr_dyn_check_fail(ptr %path, ptr ${want}, ptr %d)`);
         B.terminate(`ret ptr null`);
         break;

packages/compiler/src/backend/llvm/emit-module.ts

@@ -1333,14 +1333,17 @@ export function helperDefs(emitter: LlEmitter): string[] {
   // of the capacity word (SCR_STR_ASCII_BIT, the low half's sign bit on
   // little-endian targets) makes UTF-16 units bytes. Unproven strings and
   // anything but an in-range index take the runtime's UTF-16 mapping.
-  // A scan's receiver reads inline too where runtime state is ordinary
-  // global storage (ScrStrScan; SCR_TL makes it thread-local otherwise).
-  const scanInline = emitter.mod.workers !== true && emitter.mod.lib?.threadInstances !== true;
+  // A scan's receiver reads inline too (ScrStrScan): from ordinary global
+  // storage, or, in worker executables, from the thread's static TLS block,
+  // one thread-pointer-relative load (see executableTls).
+  const scanInline =
+    emitter.executableTls ||
+    (emitter.mod.workers !== true && emitter.mod.lib?.threadInstances !== true);
   let scanDeclared = false;
   for (const helper of STRING_READ_HELPERS) {
     if (!emitter.decls.has(helper.decl)) continue;
     if (scanInline && !scanDeclared) {
-      defs.push(stringScanDecl(emitter.sizeType), ``);
+      defs.push(stringScanDecl(emitter.sizeType, emitter.executableTls), ``);
       scanDeclared = true;
     }
     defs.push(...helper.define(emitter.sizeType, FN_ATTRS, scanInline), ``);

packages/compiler/src/backend/llvm/expr-primitives.test.ts

@@ -306,3 +306,44 @@ test("sums that only feed a bitwise conversion of int32 operands add in 32 bits"
   }
   expect(body("number_add")).not.toMatch(/ = add i32 |fcmp ord/);
 });
+
+test("string reads of a scan receiver load it inline where emitted code can address it", () => {
+  const receiver: IrExpr = { kind: "varRef", localId: "text", type: STRING, loc };
+  const mod = fixture();
+  mod.functions = [
+    { name: "__main", params: [], returnType: VOID, locals: [], body: [], loc },
+    {
+      name: "units",
+      params: [{ name: "text", localId: "text", type: STRING }],
+      returnType: F64,
+      locals: [{ id: "text", name: "text", type: STRING, mutable: false }],
+      loc,
+      body: [
+        {
+          kind: "return",
+          loc,
+          value: { kind: "strIntrinsic", method: "length", receiver, args: [], type: F64, loc },
+        },
+      ],
+    },
+  ];
+  const helper = (llvm: string): string =>
+    /^define internal double @sc_str_utf16_len\([^]*?^}/m.exec(llvm)![0];
+  const plain = emitLlvmModule(mod);
+  expect(plain).toContain("@scr_str_scan = external global");
+  expect(helper(plain)).toContain("%scan.same");
+  // ELF worker executables keep the receiver in the thread's static TLS block.
+  const elf = emitLlvmModule(
+    { ...mod, workers: true },
+    { targetTriple: "x86_64-unknown-linux-gnu" },
+  );
+  expect(elf).toContain("@scr_str_scan = external thread_local(localexec) global");
+  expect(helper(elf)).toContain("%scan.same");
+  // Darwin worker executables leave every non-ASCII read to the runtime.
+  const darwin = emitLlvmModule(
+    { ...mod, workers: true },
+    { targetTriple: "arm64-apple-macosx14.0.0" },
+  );
+  expect(darwin).not.toContain("@scr_str_scan");
+  expect(helper(darwin)).not.toContain("%scan.same");
+});

packages/compiler/src/backend/llvm/string-reads.ts

@@ -13,8 +13,8 @@
  * With `scan`, a string that is not proven ASCII is first compared with
  * the runtime's published scan receiver (ScrStrScan, scr_runtime.h): the
  * string an indexed scan converted to UTF-16 units, whose length and units
- * then read inline. Only targets whose runtime state is not thread-local
- * publish it to emitted code.
+ * then read inline. Where the runtime keeps it per thread, the declaration
+ * is thread-local.
  */
 
 import { exactIndexLines, sizeToDouble } from "./common.js";
@@ -53,8 +53,8 @@ export function stringScanType(sz: string): string {
   return `{ ptr, ptr, ${sz} }`;
 }
 
-export function stringScanDecl(sz: string): string {
-  return `@scr_str_scan = external global ${stringScanType(sz)}`;
+export function stringScanDecl(sz: string, threadLocal: boolean): string {
+  return `@scr_str_scan = external ${threadLocal ? "thread_local " : ""}global ${stringScanType(sz)}`;
 }
 
 /** The scan receiver test of `%s`: branches to `hit` with `%scan.n` (its

packages/compiler/src/frontend/lowering/builtins/process.ts

@@ -295,7 +295,7 @@ export function isOptionalProcessVersionsRead(lowerer: Lowerer, node: ts.Node):
   if (!programMentionsVersions(lowerer)) return false;
   const symbol = lowerer.typeOf(node.expression).getSymbol();
   return (
-    symbol?.name === "ProcessVersions" &&
+    symbol !== undefined && ts.symbolName(symbol) === "ProcessVersions" &&
     symbol.declarations.some(
       (d) => ts.isInterfaceDeclaration(d) && lowerer.isStdlibFile(d.getSourceFile()),
     ) &&

packages/runtime/src/scr_string.c

@@ -1846,9 +1846,10 @@ static uint16_t *scr_str_units(const ScrStr *s, size_t *n) {
  * UTF-16 units to bytes through the cursor and checkpoints. Once its
  * reads would have paid for converting it (one read per 16 bytes), a
  * receiver up to SCR_SIDX_UNITS_MAX units keeps its code units in its
- * entry and reads become loads. The array is freed with the entry, and an
- * append drops it (scr_sidx_concat_append). */
-#define SCR_SIDX_UNITS_MAX ((size_t)1 << 20)
+ * entry and reads become loads. The bound admits source texts of several
+ * megabytes, which scanners read in full. The array is freed with the
+ * entry, and an append drops it (scr_sidx_concat_append). */
+#define SCR_SIDX_UNITS_MAX ((size_t)1 << 24)
 static bool scr_sidx_units(const ScrStr *s, ScrSidx *e) {
   if (!e->units) {
     if (++e->reads < 32 || e->reads * 16 < s->len || s->len > SCR_SIDX_UNITS_MAX) return false;

packages/runtime/src/scr_string_methods.h

@@ -27,7 +27,8 @@ double scr_str_utf16_len(ScrStr *s);
  * holds (scr_string.c), published so emitted length and charCodeAt reads of
  * that string are a compare and a load. Cleared whenever its entry drops
  * the units (release, append, eviction); all fields zero otherwise. Emitted
- * code reads it only where SCR_TL is empty. */
+ * code reads it where SCR_TL is empty, and in worker executables on ELF
+ * targets, whose thread-locals live in the executable's static TLS block. */
 typedef struct ScrStrScan {
   const ScrStr *s;
   const uint16_t *units;

tests/harness/dyncheck.test.ts

@@ -305,6 +305,46 @@ console.log("unreachable", c.host);
     );
   });
 
+  test("optional record member: the failure names the deepest path inside it", async () => {
+    // `Partial<T>` makes every member an `X | undefined` union. A present
+    // object that fails X used to report only the union's own path
+    // ("expected object | undefined at $.order, got object"); the record
+    // arm's builder now reports where inside it the value fails.
+    const r = await compileAndRun(
+      "optional-record-deep",
+      `interface Order {
+  owner: { id: string; plan: { tier: string } };
+  total: number;
+}
+interface Envelope { order: Order; tags?: string[] }
+const texts = [
+  '{"order":{"owner":{"id":"u1"},"total":3}}',
+  '{"order":{"owner":{"id":7,"plan":{"tier":"pro"}},"total":3}}',
+  '{"order":{"owner":{"id":"u1","plan":{"tier":"pro"}},"total":3},"tags":["a",2]}',
+  '{"order":5}',
+];
+for (const text of texts) {
+  try {
+    const e = JSON.parse(text) as Partial<Envelope>;
+    console.log("ok", e.order?.owner.id);
+  } catch (err) {
+    if (err instanceof TypeError) console.log(err.message);
+  }
+}
+`,
+    );
+    expect(r.exitCode).toBe(0);
+    expect(r.stdout).toBe(
+      [
+        "expected object at $.order.owner.plan, got undefined",
+        "expected string at $.order.owner.id, got number",
+        "expected string at $.tags[1], got number",
+        "expected object | undefined at $.order, got number",
+        "",
+      ].join("\n"),
+    );
+  });
+
   test("JSON null matches nothing", async () => {
     const r = await compileAndRun(
       "null-value",