Merge branch 'ni/i10-js-island-requires' into ni/i11-cross-heap-gc

.github/workflows/ci.yml

@@ -17,6 +17,11 @@ on:
         type: choice
         options: [full, darwin]
         default: full
+      scope:
+        description: Every lane, or only the shape overlay validation (shape_overlay)
+        type: choice
+        options: [all, shape_overlay]
+        default: all
 
 concurrency:
   # A new push to a pull request cancels its previous run. Every push to main runs to completion in a group of
@@ -59,7 +64,10 @@ jobs:
             sameRepository && trustedAuthor && typeof stackBase === 'string' && stackBase.length > 0 &&
             typeof prBase === 'string' && prBase.length > 0 && stackBase !== prBase &&
             prBase !== defaultBranch;
-          const full = !upperLayer;
+          // A manual run of the shape overlay validation alone skips the other lanes.
+          const shapeOnly = process.env.GITHUB_EVENT_NAME === 'workflow_dispatch' &&
+            event.inputs?.scope === 'shape_overlay';
+          const full = !upperLayer && !shapeOnly;
           appendFileSync(process.env.GITHUB_OUTPUT, `full=${full}\n`);
           console.log(full ? 'Run full CI' : 'Defer full CI until this PR becomes the lowest unmerged layer');
           NODE
@@ -390,6 +398,81 @@ jobs:
       - run: tar -xzf "$RUNNER_TEMP/native-linux-x64-gnu.tar.gz"
       - run: pnpm test tests/harness/three.test.ts -t "'nodes' runs statically through LLVM on 'native'"
 
+  # Shape inference's overlay (opt-in, SCRIPTC_SHAPE_OVERLAY=1) over the
+  # suites of real packages: nightly and on manual runs only (scope
+  # shape_overlay runs it alone), outside the `test` roll-up. Every case
+  # must build and run as in the default build. It builds its own native
+  # packages, so it does not depend on the other lanes.
+  shape_overlay_native:
+    if: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }}
+    name: build (Linux x64 native packages, shape overlay)
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@v4
+      - uses: pnpm/action-setup@v4
+        with:
+          version: 11.1.3
+      - uses: actions/setup-node@v4
+        with:
+          node-version-file: .node-version
+          cache: pnpm
+      - uses: ./.github/actions/llvm-linux
+      - run: pnpm install --frozen-lockfile
+      - uses: ./.github/actions/zig
+      - uses: ./.github/actions/native-packages
+        with:
+          target: linux-x64-gnu
+      - name: Preserve executable permissions in shared packages
+        run: >-
+          tar -czf "$RUNNER_TEMP/native-linux-x64-gnu.tar.gz"
+          packages/llvm-linux-x64-gnu/bin
+          packages/runtime-linux-x64-gnu/artifacts
+          packages/runtime-linux-x64-gnu/runtime-pack.json
+      - uses: actions/upload-artifact@v4
+        with:
+          name: native-linux-x64-gnu-shape-overlay
+          path: ${{ runner.temp }}/native-linux-x64-gnu.tar.gz
+          retention-days: 1
+
+  shape_overlay:
+    needs: [shape_overlay_native]
+    if: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }}
+    name: test (Linux, shape overlay, ${{ matrix.suite }} ${{ matrix.shard }}/8)
+    runs-on: ubuntu-24.04
+    strategy:
+      fail-fast: false
+      matrix:
+        suite: [npm, npm-static, three, effect]
+        shard: [1, 2, 3, 4, 5, 6, 7, 8]
+    env:
+      SCRIPTC_SHAPE_OVERLAY: "1"
+      SCRIPTC_TEST_WORKERS: "1"
+      SCRIPTC_EFFECT_TEST_CONCURRENCY: "1"
+      SCRIPTC_TEST_SHARD: ${{ matrix.shard }}/8
+    steps:
+      - uses: actions/checkout@v4
+      - uses: pnpm/action-setup@v4
+        with:
+          version: 11.1.3
+      - uses: actions/setup-node@v4
+        with:
+          node-version-file: .node-version
+          cache: pnpm
+      - run: pnpm install --frozen-lockfile
+      - uses: actions/download-artifact@v4
+        with:
+          name: native-linux-x64-gnu-shape-overlay
+          path: ${{ runner.temp }}
+      - run: tar -xzf "$RUNNER_TEMP/native-linux-x64-gnu.tar.gz"
+      - name: Run the suite with the overlay on
+        run: |
+          case "${{ matrix.suite }}" in
+            npm) pnpm test tests/harness/npm-shape-overlay.test.ts ;;
+            npm-static) pnpm test tests/harness/npm-static.test.ts ;;
+            three) pnpm test tests/harness/three.test.ts ;;
+            effect) pnpm test tests/harness/effect.test.ts tests/harness/effect4.test.ts ;;
+          esac
+
   # Build each instrumented seed once. Downstream contracts retain the same
   # compiler and source artifacts without repeating seed construction.
   bootstrap_seed:

.gitignore

@@ -1,4 +1,6 @@
 node_modules/
+# A symlinked node_modules is a file to git, which node_modules/ does not match.
+/packages/*/node_modules
 .pnpm-store/
 # test-fixture packages are TEST DATA, committed on purpose
 !tests/diagnostics/npm-imports/node_modules/

RELEASING.md

@@ -10,7 +10,8 @@ To prepare a release:
 4. Wrap the new entry in `<!-- release:start -->` and `<!-- release:end -->` markers; this marked block is also the GitHub release body
 5. Remove the `<!-- release:start -->` and `<!-- release:end -->` markers from the previous release entry; only the latest release should have markers
 6. With Zig on `PATH`, run `SCRIPTC_CROSS=1 pnpm exec vitest run tests/harness/library-cross.test.ts` and require the cross-target library conformance lane to pass
-7. Commit to `main`
+7. Refresh the runtime profile on a quiet Linux x64 host: `node scripts/refresh-runtime-profile.mjs --tsc-ts <tsc-ts checkout> --project <TypeORM>:tsconfig.bench.json --project <T3 server>:tsconfig.bench.json --project <VS Code>/src:tsconfig.bench.json`, and include the new `packages/runtime-pack-common/profiles/linux-x64-gnu.profdata.gz` (see the runtime profile paragraph below)
+8. Commit to `main`
 
 CI (`.github/workflows/release.yml`) compares the version in `packages/cli/package.json` to what `scriptc` has on npm. If it differs, it builds platform packages on matching macOS, Linux, and Windows runners, verifies the version spine, and publishes the runtime, helper, compiler, and CLI packages in dependency order. After the publish succeeds, a separate job creates the git tag `v<version>` and the GitHub release with the marked changelog entry as its body, and attaches `surface-manifest.json` — the machine-readable listing of the surface the static tier compiles at that version (stable per-entry ids, so two releases diff mechanically; see `packages/compiler/src/coverage/surface-manifest.ts` for the schema). The job regenerates the manifest from the tree and fails on any byte difference from the committed file before attaching, so the asset is always the manifest of the code being released. The same file ships inside the `@scriptc/compiler` package as `@scriptc/compiler/surface-manifest.json`.
 
@@ -20,6 +21,8 @@ Release builds of the native CLI optimize the compiler with a profile collected
 
 Builds of the native compiler also order its functions by the committed layout `packages/compiler/native/cli.layout.gz`: the functions that `--version` and rebuilds of unchanged programs run come first, then the ones other builds run, ordered by the share of the layout's training builds that run each, so a run maps fewer pages of the executable. Functions the layout does not list keep their usual order. Function names of closures and generated helpers change with nearby edits, so the `cli-profile` job also collects a layout on the release commit (`scripts/refresh-cli-layout.mjs`), and the CLI builds use it (`SCRIPTC_NATIVE_LAYOUT`). Instrumented builds use the same layout as optimized builds, because a function laid out in a different partition than in the instrumented build no longer matches its profile record. The job therefore collects the layout first and then builds the instrumented compiler for the profile with that layout. Refresh the committed layout the same way, with `node scripts/refresh-cli-layout.mjs`, before the profile, and commit each on its own.
 
+The Linux x64 runtime pack optimizes its speed-flavor units with the committed runtime profile `packages/runtime-pack-common/profiles/linux-x64-gnu.profdata.gz`. Runtime functions whose code changed since the profile was collected are optimized as without a profile. The pack manifest records how the profile matched (`flavors.speed.runtime_profile`): the matched and changed functions, and the share of the profile's counts on changed functions. `packages/runtime-pack-common/test/runtime-profile.test.ts` fails when changed functions hold more than 5% of the counts, and names the functions with the most counts. Refresh the profile in each release commit (step 7), and between releases when that test fails.
+
 Publishing uses npm trusted publishing (OIDC) — there is no npm token secret.
 Each published package must have a GitHub Actions trusted publisher for
 `release.yml` and the `Release` environment. A missing configuration fails

benchmarks/runtime/README.md

@@ -21,6 +21,7 @@ Application-shaped workloads for measuring the speed of compiled executables. Ea
 | `json-records` | cpu | `JSON.stringify`, typed `JSON.parse`, Map aggregation |
 | `json-unions` | cpu | `JSON.parse` into discriminated unions: an event log with eight kinds and nested expression trees with nine, evaluated after the cast |
 | `keyed-option-writes` | cpu | class methods whose names are also keys of option records written through element accesses: `enabled[name] = true` over a union of literal names and `totals["count"] = n` |
+| `layered-errors` | cpu | validation errors thrown below a stack of small functions that hold strings and arrays, and caught per entry in a loop: about half of 640,000 entries fail |
 | `line-reader` | cpu | `readline` over a file stream: 'line' events for a 67 MB access log (every 50th line not ASCII), with a status count, a path tally and byte totals |
 | `log-lines` | cpu | 200k immediate `console.log`/`process.stdout.write` lines into the runner's pipe |
 | `log-summary`, `inventory-report` | cpu | the multi-module applications from `benchmarks/builds`, on 300k-line inputs |

benchmarks/runtime/workloads/layered-errors.json

@@ -0,0 +1,6 @@
+{
+  "entry": "workloads/layered-errors.ts",
+  "args": [
+    "1"
+  ]
+}

benchmarks/runtime/workloads/layered-errors.ts

@@ -0,0 +1,95 @@
+// A configuration loader whose validation errors are thrown at the bottom
+// of a stack of small layers and caught per entry at the top: each layer
+// holds strings and arrays while the error passes through it.
+class ConfigError extends Error {
+  readonly key: string;
+  constructor(key: string, message: string) {
+    super(message);
+    this.key = key;
+  }
+}
+
+let seed = 4242;
+function random(): number {
+  seed = (Math.imul(seed, 1103515245) + 12345) >>> 0;
+  return seed / 4294967296;
+}
+
+function fail(key: string, text: string): never {
+  throw new ConfigError(key, `invalid value "${text}" for ${key}`);
+}
+function digits(key: string, text: string): number {
+  let value = 0;
+  for (let i = 0; i < text.length; i++) {
+    const c = text.charCodeAt(i);
+    if (c < 48 || c > 57) fail(key, text);
+    value = value * 10 + (c - 48);
+  }
+  return value;
+}
+function parseNumber(key: string, text: string): number {
+  return text.length === 0 ? fail(key, text) : digits(key, text);
+}
+function checkRange(key: string, text: string, max: number): number {
+  const value = parseNumber(key, text);
+  return value > max ? fail(key, text) : value;
+}
+function coerce(key: string, raw: string, max: number): number {
+  const trimmed = raw.trim();
+  const parts = [key, trimmed];
+  const value = checkRange(key, trimmed, max);
+  return value + parts.length;
+}
+function resolveValue(key: string, raw: string, defaults: Map<string, string>): number {
+  const text = raw === "default" ? (defaults.get(key) ?? "") : raw;
+  const label = "value:" + key;
+  return coerce(key, text, 100000) + label.length;
+}
+function loadEntry(section: string, entry: string, defaults: Map<string, string>): number {
+  const eq = entry.indexOf("=");
+  const key = section + "." + entry.slice(0, eq);
+  const path = [section, key];
+  return resolveValue(key, entry.slice(eq + 1), defaults) + path.length;
+}
+function loadEntryWithContext(section: string, entry: string, defaults: Map<string, string>): number {
+  const context = `${section}/${entry.length}`;
+  return loadEntry(section, entry, defaults) + context.length;
+}
+
+function makeEntry(i: number): string {
+  const r = random();
+  const name = "k" + (i % 97);
+  if (r < 0.3) return `${name}=x${i}`;
+  if (r < 0.4) return `${name}=`;
+  if (r < 0.5) return `${name}=${200000 + i}`;
+  if (r < 0.55) return `${name}=default`;
+  return `${name}= ${Math.floor(random() * 90000)} `;
+}
+
+const rounds = Number(process.argv[2] ?? "1");
+const defaults = new Map<string, string>();
+for (let i = 0; i < 97; i += 2) defaults.set(`net.k${i}`, String(i * 10));
+const sections = ["net", "db", "cache", "auth"];
+const entries: string[] = [];
+for (let i = 0; i < 4000; i++) entries.push(makeEntry(i));
+
+let loaded = 0;
+let sum = 0;
+let failed = 0;
+const failedKeys = new Map<string, number>();
+for (let round = 0; round < rounds * 40; round++) {
+  for (const section of sections) {
+    for (const entry of entries) {
+      try {
+        sum += loadEntryWithContext(section, entry, defaults);
+        loaded++;
+      } catch (error) {
+        if (!(error instanceof ConfigError)) throw error;
+        failed++;
+        failedKeys.set(error.key, (failedKeys.get(error.key) ?? 0) + 1);
+      }
+    }
+  }
+}
+console.log(`loaded ${loaded} failed ${failed} sum ${sum} distinct failing keys ${failedKeys.size}`);
+console.log(`net.k0 failures ${failedKeys.get("net.k0") ?? 0}`);

docs/content/docs/cli.mdx

@@ -176,7 +176,7 @@ With `--sparse`, the profile leaves out the functions the runs never reached. Th
   <dd>On Linux, build an instrumented executable for profile-guided optimization. Each run writes a raw profile to the path pattern, where <code>%p</code> expands to the process ID; the <code>LLVM_PROFILE_FILE</code> environment variable overrides the pattern at run time. A relative pattern resolves from the current working directory. Instrumented programs run slower. With <code>--profile-use</code>, the build is optimized by that profile and instrumented after inlining, the second phase of a <a href="#context-sensitive-profiles">context-sensitive profile</a>. Valid only for executable builds with <code>--optimization=release</code> or <code>speed</code>.</dd>
 
   <dt><code>--profile-use &lt;file.profdata&gt;</code></dt>
-  <dd>Optimize the executable with a profile from <code>scriptc merge-profiles</code>: code is laid out, inlined, and split by how the profiled runs used it. Code the profile does not describe, or describes from an older version of the program, is optimized as without a profile, so a stale profile only loses part of its gain. A new or changed profile compiles every partition of the program again. <code>scriptc build</code> prints how the profile matched the program's functions on stderr; see <a href="#profile-guided-optimization">Profile-guided optimization</a>. A profile with <a href="#context-sensitive-profiles">context-sensitive counts</a> also orders each function's blocks by them on Linux. Valid only for executable builds with <code>--optimization=release</code> or <code>speed</code>.</dd>
+  <dd>Optimize the executable with a profile from <code>scriptc merge-profiles</code>: code is laid out, inlined, and split by how the profiled runs used it. Code the profile does not describe, or describes from an older version of the program, is optimized as without a profile, so a stale profile only loses part of its gain; a <a href="#block-counts">block-count file</a> beside the profile gives changed functions their counts again. A new or changed profile compiles every partition of the program again. <code>scriptc build</code> prints how the profile matched the program's functions on stderr; see <a href="#profile-guided-optimization">Profile-guided optimization</a>. A profile with <a href="#context-sensitive-profiles">context-sensitive counts</a> also orders each function's blocks by them on Linux. Valid only for executable builds with <code>--optimization=release</code> or <code>speed</code>.</dd>
 
   <dt><code>--npm-static &lt;pkg[,pkg…]|auto&gt;</code></dt>
   <dd>Name npm packages to compile even if they don't ship their own type declarations (repeatable). A trailing <code>...</code>, as in <code>fs-extra...</code>, also names every package that package depends on. <code>auto</code> uses the default automatic selection. Packages that can't compile produce diagnostics. See <a href="/docs/dependencies#explicit-package-selection">npm Dependencies</a>.</dd>
@@ -266,7 +266,7 @@ $ scriptc merge-profiles -o app.profdata profiles/*.profraw
 $ scriptc build app.ts --optimization=speed --profile-use app.profdata
 ```
 
-Train on workloads like the ones the program will run: code the training never reached is optimized as without a profile. The profile can be committed and reused as the program changes. Functions whose control flow changed since the profile was collected lose their profile, and the rest keep theirs, so regenerate it when much of the program has changed. [Build parallelism](/docs/how-it-works#build-parallelism) describes what a profile adds to build time.
+Train on workloads like the ones the program will run: code the training never reached is optimized as without a profile. The profile can be committed and reused as the program changes, and in other checkouts and on other machines: it names functions by the entry's file name, not its directory. Functions whose control flow changed since the profile was collected lose their profile, and the rest keep theirs, so regenerate it when much of the program has changed. [Build parallelism](/docs/how-it-works#build-parallelism) describes what a profile adds to build time.
 
 `scriptc build --profile-use` prints one line on stderr with how the profile matched the program's functions:
 
@@ -304,6 +304,25 @@ $ scriptc build app.ts --optimization=speed --profile-use app-cs.profdata
 
 A profile that already holds context-sensitive counts cannot start a second phase. Merge the second phase's raw profiles into the first-phase profile instead. Both phases run the same workloads.
 
+### Block counts
+
+A profile matches a function only while the function's control flow hashes as it did when the profile was collected, so a change to the program or to scriptc leaves the changed functions without counts. A block-count file beside the profile, with its name and a `.blocks` extension in place of `.profdata` (`app.blocks` beside `app.profdata`), describes each function by the labels of its blocks instead. `--profile-use` reads it when it exists and annotates the functions whose control flow changed (the mismatched ones; a function the profile has no record of keeps no counts): blocks that keep their place in the function take their recorded counts, and the counts of the other blocks follow from the counts of the blocks and edges around them. The report then adds a line, and the warning counts only the functions that remain without counts:
+
+```
+scriptc: profile app-cs.profdata: 0 functions annotated, 1 mismatched, 0 without a record
+scriptc: profile app-cs.profdata: 1 function without profile counts annotated from app-cs.blocks
+```
+
+A `--profile-use` build writes the block counts of the functions its profile annotates when the `SCRIPTC_RECORD_PROFILE_BLOCKS` environment variable names a path; such a build reuses no cached artifacts. For a context-sensitive profile, record them in the second phase, which the first-phase profile annotates:
+
+```console
+$ SCRIPTC_RECORD_PROFILE_BLOCKS=app-cs.blocks scriptc build app.ts --optimization=speed --profile-use app.profdata --profile-generate profiles-cs/%p.profraw -o app-instrumented-cs
+```
+
+A new or changed block-count file compiles every partition of the program again. Delete it when the profile it was recorded with is replaced.
+
+The file holds the counts before inlining. With a context-sensitive profile, a function whose control flow changed, and a function it is inlined into, also lose their context-sensitive counts and keep the counts from before inlining.
+
 ## Target CPU
 
 By default, x86-64 executables run on any x86-64 CPU, so the compiler uses only the baseline instruction set (SSE2). `--target-cpu` compiles the program's code for a higher x86-64 microarchitecture level. Each level includes the ones below it:

docs/content/docs/contracts.mdx

@@ -366,6 +366,10 @@ Static code has no `WeakRef`, a `FinalizationRegistry` never runs its cleanup ca
 
 A long-running program, such as a compiler in watch mode, can free program-lifetime instances one generation at a time with `LifetimeRegion` from [`@scriptc/experimental`](/docs/experimental). Instances created while `region.run(fn)` runs belong to the region. After `region.release()`, scriptc frees them together once nothing outside the region refers to any of them, and removes the `WeakMap` and `WeakSet` entries they are keys of. Instances created outside every region are never freed, as described above.
 
+### Inline array fields
+
+An array field of a program-lifetime class whose elements are program-lifetime instances can keep its elements in the instance itself, after the other fields, with `InlineArray` from [`@scriptc/experimental`](/docs/experimental). The constructor writes the field once from one of its parameters; reads that index the field, read its `length` or iterate it load the elements from the instance, and other reads get an array of them. The array passed to the constructor is one that nothing else keeps, and the arrays fields hold are never written. scriptc reports a field that does not keep these rules (SC7009).
+
 ## Diagnostics
 
 Each failed contract names the declaration and the reason:
@@ -468,5 +472,9 @@ $ scriptc build reject.ts
       <td>SC7008</td>
       <td>A <code>ProgramLifetime</code> class is not the root of its hierarchy, or is part of a library build.</td>
     </tr>
+    <tr>
+      <td>SC7009</td>
+      <td>An <code>InlineArray</code> field is not on a <code>ProgramLifetime</code> class, is written other than once from a constructor parameter, or is read in a way that could tell its reads apart.</td>
+    </tr>
   </tbody>
 </table>

docs/content/docs/experimental.mdx

@@ -334,3 +334,73 @@ scriptc requires:
 Each instance of a key class, and of every class in its hierarchy, carries a 4-byte slot index, 0 until a value is set for it. The first `set` for a key assigns the index; indices are never reused. `get` and `has` read the index, a page of the thread's family and the entry, with no lock, hash or atomic update. A family stores values in pages of 1024 entries, allocated as the thread sets values in them.
 
 A family holds a reference to each of the thread's values. A value is released when it is replaced or deleted, when its key is freed, and when the thread exits. Keys that are freed with a [`LifetimeRegion`](#lifetimeregion) keep their values until the thread exits, a value that belongs to a region keeps the region allocated while the family holds it, and a value that refers to its own key keeps the key alive. Programs that never construct a `ThreadSlots` do not run any of its code, and their classes have no slot index.
+
+## InlineArray
+
+`InlineArray<T>` marks an array field of a [`ProgramLifetime`](/docs/contracts#programlifetime) class whose elements scriptc keeps in the instance itself, after its other fields, instead of in an array object of their own. It suits a program with millions of short arrays that never change once built, such as the children of each node of a syntax tree: an ordinary array costs a separate allocation with a header, and a reference to it in the instance.
+
+```ts:lists.ts
+import type { ProgramLifetime } from "@scriptc/types";
+import type { InlineArray } from "@scriptc/experimental";
+
+class Token implements ProgramLifetime {
+  readonly text: string;
+  constructor(text: string) {
+    this.text = text;
+  }
+}
+
+class TokenList implements ProgramLifetime {
+  readonly pos: number;
+  readonly tokens: InlineArray<Token>;
+  constructor(pos: number, tokens: readonly Token[]) {
+    this.pos = pos;
+    this.tokens = tokens;
+  }
+}
+
+function width(list: TokenList): number {
+  let total = 0;
+  for (const token of list.tokens) total += token.text.length;
+  return total;
+}
+
+const list = new TokenList(0, ["let", "x", "=", "1"].map((text) => new Token(text)));
+console.log(list.tokens.length, width(list), list.tokens[0]!.text);
+console.log(list.tokens.map((token) => token.text).join(" "));
+```
+
+Both runtimes print the same:
+
+```console
+$ node lists.ts
+4 6 let
+let x = 1
+$ scriptc build lists.ts -o lists >/dev/null
+$ ./lists
+4 6 let
+let x = 1
+```
+
+### Field rules
+
+scriptc requires:
+
+- `T` to be a `ProgramLifetime` class, and the field to be an instance field of a `ProgramLifetime` class;
+- the constructor of the class that declares the field to write it once, from one of its parameters. A subclass constructor passes one of its own parameters there through `super(...)`. Nothing else writes the field;
+- the array passed to the constructor to be one that nothing else keeps: a new array (an array literal, a copy such as `slice()`, `[...items]` or `Array.from(items)`, or the result of a function that returns only such arrays or its own argument), a local array at its last use (calls that only read it or add to it may come first, and a loop may pass it and then assign a new array to the local), a parameter whose callers each pass such an array, an array that an InlineArray field already holds, or a `const` array that is never written, such as a shared empty array;
+- the arrays fields hold never to be written, through a read of the field (which the `readonly` type allows only through a cast) or through any other reference. scriptc follows these arrays through locals, parameters, return values, fields, the elements of arrays, Maps and Sets, and closures, and reports one that reaches a write, a dynamic value (also inside the fields of an object that becomes one), a call through a function value (also inside an array, Map or Set passed there), or a library call other than printing and `JSON.stringify` without a replacer. The report names the field read the array may come from;
+- the arrays fields hold, and their reads, not to be compared by identity: with `===` or `!==`, as an `indexOf` or `includes` argument, or as a Map key or Set element. Two reads of the field need not give the same array. Compare the elements instead, or keep a copy made with `slice()` where identity matters;
+- instances of the class not to be constructed through a class value, and not to become dynamic values (for example through a conversion to `unknown`), directly or inside another value that can hold one: an instance of a class with such a field, a record, array, Map or Set, or a value typed as a base class. A dynamic view reads the field as a new array each time, so a class whose instances can become dynamic values cannot hold an InlineArray field. Printing, `JSON.stringify` without a replacer, `publish` and a worker's `workerData` only read or copy the value and are allowed. The report names the conversion.
+
+scriptc reports a field that does not keep these rules (SC7009).
+
+### Reads
+
+Indexing the field, reading its `length` and iterating it with `for...of` read the elements from the instance, directly or through a `const` local used only in those ways. Any other use, such as passing the field to a function or calling one of its methods, gets an array of the elements, which is frozen in scriptc builds. Under Node.js the field holds the array the constructor stored. The two agree because the field rules keep that array from being written.
+
+### Inline array cost
+
+Each element takes 4 bytes in the instance, and the field's own slot holds the count. The constructor copies the elements from the array it is passed, so that array can be released. The elements start at the same offset in every class of the hierarchy, after the largest set of fixed fields. A read that gets an array allocates a small header that refers to the instance's elements, and frees it when the read's value is released.
+
+The elements stay in an ordinary array object, and the field behaves like any other array field, in builds with sanitizers, in programs that use [`LifetimeRegion`](#lifetimeregion), and on platforms other than 64-bit Linux.

docs/content/docs/how-it-works.mdx

@@ -34,7 +34,7 @@ Runtime pack and linker -> executable
 
 ## Build parallelism
 
-Most of a build runs on one CPU core. The LLVM helper uses more cores when it compiles an executable in partitions, and when it generates code for an optimized executable of one partition.
+Most of a build runs on one CPU core. The LLVM helper uses more cores when it compiles an executable in partitions, and when it generates code for an optimized executable that it optimizes whole.
 
 ### Frontend
 
@@ -42,16 +42,15 @@ Parsing, type checking, lowering, and LLVM IR generation run one after another o
 
 ### Partitions
 
-The LLVM helper divides an executable's LLVM IR into partitions by its size. Up to 1 MiB of LLVM text is one partition, and up to 2 MiB is four. Larger programs get one partition per 4 MiB of LLVM text, rounded up to a power of two, and at least 8 and at most 64. `--emit=llvm` writes this text, so its file size shows how many partitions a program gets. `dev` executables, whose partitions are compiled on their own with nothing imported, are divided from 64 KiB: up to 128 KiB is four partitions, and larger `dev` programs get at least 8.
+The LLVM helper divides an executable's LLVM IR into partitions by its size. Up to 128 KiB of LLVM text is one partition. Larger programs get one partition per 4 MiB of LLVM text, rounded up to a power of two, and at least 8 and at most 64. `--emit=llvm` writes this text, so its file size shows how many partitions a program gets. `dev` executables, whose partitions are compiled on their own with nothing imported, are divided from 64 KiB: up to 128 KiB is four partitions. An edit compiles again only the partitions whose code it changes, or that inline a function it changes.
 
 <table>
   <thead>
     <tr><th>LLVM text</th><th>Partitions</th></tr>
   </thead>
   <tbody>
-    <tr><td>Up to 1 MiB</td><td>1</td></tr>
-    <tr><td>Over 1 MiB, up to 2 MiB</td><td>4</td></tr>
-    <tr><td>Over 2 MiB, up to 32 MiB</td><td>8</td></tr>
+    <tr><td>Up to 128 KiB</td><td>1</td></tr>
+    <tr><td>Over 128 KiB, up to 32 MiB</td><td>8</td></tr>
     <tr><td>Over 32 MiB, up to 64 MiB</td><td>16</td></tr>
     <tr><td>Over 64 MiB, up to 128 MiB</td><td>32</td></tr>
     <tr><td>Over 128 MiB</td><td>64</td></tr>
@@ -60,6 +59,8 @@ The LLVM helper divides an executable's LLVM IR into partitions by its size. Up
 
 The count depends only on the size of the LLVM text, so a program has the same partitions, and builds the same executable, on every host. In programs with 32 partitions or more, functions from the same source file are placed in the same partition, so that their calls to each other stay inside it; a file larger than 1 MiB of LLVM text or than an average partition is divided into parts by function name. Other functions, and the functions of smaller programs, are placed by their name, and a small internal function goes with the one function that calls it. An edit leaves almost every other function in its partition.
 
+An optimized partition is optimized twice: on its own, and again after the small functions of other partitions it calls are imported for inlining. So in optimized builds, a program whose largest partition would hold more than two thirds of its function text is optimized whole instead, as one partition, and its code is generated in parts on several threads.
+
 These builds are not divided into partitions and compile on one thread: `--emit=asm`, `--emit=obj`, and `--print=native-link-info` output, library builds (`--lib`; `dev` library builds are compiled in pieces, one after another), and `--sanitize` builds, which compile the LLVM IR with the C compiler.
 
 The helper compiles partitions on a pool with one thread per hardware thread of the host, so an executable with four partitions uses at most four cores in the backend.
@@ -71,26 +72,15 @@ How partitions are compiled depends on `--optimization`:
 
 Reading the LLVM text and dividing it into partitions, and the import step, run on one thread.
 
-### Code generation of one partition
+### Code generation of a program optimized whole
 
-A `release` or `speed` executable of one partition is optimized as one module on one thread. When its LLVM text is larger than 128 KiB, the helper then divides the optimized module for code generation and generates machine code for the parts in parallel, on the same pool. Each part holds a run of consecutive functions with about the same amount of code, so the executable keeps the functions in the same order. The module is optimized before it is divided, so each function's machine code is the same as from one thread, except that a jump to a function that another part also uses takes the longer form. When one function holds most of the code, the first part holds the whole module and the others are empty.
+A `release` or `speed` executable whose largest partition would hold more than two thirds of its function text is optimized as one module on one thread. The helper then divides the optimized module for code generation into as many parts as the program has partitions and generates machine code for the parts in parallel, on the same pool. Each part holds a run of consecutive functions with about the same amount of code, so the executable keeps the functions in the same order. The module is optimized before it is divided, so each function's machine code is the same as from one thread, except that a jump to a function that another part also uses takes the longer form. When one function holds most of the code, the first part holds the whole module and the others are empty.
 
-<table>
-  <thead>
-    <tr><th>LLVM text</th><th>Code generation parts</th></tr>
-  </thead>
-  <tbody>
-    <tr><td>Up to 128 KiB</td><td>1</td></tr>
-    <tr><td>Over 128 KiB, up to 256 KiB</td><td>4</td></tr>
-    <tr><td>Over 256 KiB, up to 1 MiB</td><td>8</td></tr>
-  </tbody>
-</table>
-
-Like partitions, the count depends only on the size of the LLVM text. A `dev` executable of one partition is compiled on one thread.
+An executable of one partition (up to 128 KiB of LLVM text, or 64 KiB in `dev` builds) is compiled on one thread.
 
 ### Rebuilds
 
-With two or more partitions, the helper keeps each partition's summary and object in the build cache (see `SCRIPTC_CACHE_DIR` in the [CLI reference](/docs/cli#environment-variables)). A rebuild after an edit compiles again only the partitions whose functions changed and, in `release` and `speed` builds, the partitions that import a changed function; the others come from the cache. Dividing the program, the import step, and linking run on every build. A cold build, with nothing cached, compiles every partition.
+With two or more partitions, the helper keeps each partition's summary and object in the build cache (see `SCRIPTC_CACHE_DIR` in the [CLI reference](/docs/cli#environment-variables)). A rebuild after an edit compiles again only the partitions whose functions changed and, in `release` and `speed` builds, the partitions that import a changed function; the others come from the cache. Dividing the program, the import step, and linking run on every build. A cold build, with nothing cached, compiles every partition. A program optimized whole is compiled again whole after any edit.
 
 A program of one partition has no partition cache: a rebuild after an edit compiles the whole program again, and a rebuild that leaves the LLVM text unchanged takes the compiled code from the build cache.
 

docs/content/limitations/builtin-apis/streams-and-readline.mdx

@@ -65,4 +65,7 @@
   Their underlying sources, sinks, and transformers, queuing strategies with a `size` function, readers and writers, `tee`, `pipeTo` and `pipeThrough` with their options and an abort signal, `ReadableStream.from`, and async iteration settle promises in Node's order, and the streams take own properties.
   `TextDecoderStream` decodes UTF-8; its other encodings are unsupported.
   A JavaScript `ReadableStream` given as a `Response` or `Request` body, or as `fetch`'s body, is read through a stream of its own: `response.body` is that stream rather than the one passed in, and the stream passed in is locked from the start.
+  In JavaScript, a `Response` or `Request` body has the members of a `ReadableStream`, including `tee`, `pipeTo`, `pipeThrough`, `values`, and `for await`, and its lock and `bodyUsed` follow them.
+  After one of those members is used, a body given to a new `Response` or `Request` is locked from the start, as a JavaScript `ReadableStream` is.
+  In a program that names none of the web stream classes, those members are available when the program's JavaScript names one of them or iterates a body with `for await`; reached only through a computed key, they are not.
   In JavaScript, a byte stream (`type: 'bytes'`), a BYOB reader, a class that extends one of these classes, `Object.getPrototypeOf` on these values, their use as `WeakMap` and `WeakSet` keys, and `ReadableStream.from` of a function that is not iterable are unsupported.

docs/test/fixtures/migration-baseline.json

@@ -316,6 +316,22 @@
       {
         "text": "Slot cost",
         "id": "slot-cost"
+      },
+      {
+        "text": "InlineArray",
+        "id": "inlinearray"
+      },
+      {
+        "text": "Field rules",
+        "id": "field-rules"
+      },
+      {
+        "text": "Reads",
+        "id": "reads"
+      },
+      {
+        "text": "Inline array cost",
+        "id": "inline-array-cost"
       }
     ]
   },

native/llvm-codegen/CMakeLists.txt

@@ -88,6 +88,7 @@ add_executable(scriptc-llvm-codegen
   src/emit.cpp
   src/main.cpp
   src/partition.cpp
+  src/profile-blocks.cpp
   src/profile.cpp
   src/runtime-import.cpp
   src/runtime-unit.cpp

native/llvm-codegen/src/emit.cpp

@@ -3,6 +3,7 @@
 #include "cache.h"
 #include "diagnostics.h"
 #include "partition.h"
+#include "profile-blocks.h"
 #include "runtime-import.h"
 #include "target.h"
 
@@ -13,6 +14,7 @@
 #include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/StringSet.h"
+#include "llvm/Analysis/ProfileSummaryInfo.h"
 #include "llvm/Analysis/RuntimeLibcallInfo.h"
 #include "llvm/Analysis/TargetLibraryInfo.h"
 #include "llvm/Bitcode/BitcodeReader.h"
@@ -34,6 +36,7 @@
 #include "llvm/PassInfo.h"
 #include "llvm/PassRegistry.h"
 #include "llvm/Passes/PassBuilder.h"
+#include "llvm/ProfileData/InstrProf.h"
 #include "llvm/ProfileData/InstrProfReader.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/PGOOptions.h"
@@ -41,6 +44,7 @@
 #include "llvm/Support/Caching.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/Path.h"
 #include "llvm/Support/SHA256.h"
 #include "llvm/Support/SourceMgr.h"
 #include "llvm/Support/ThreadPool.h"
@@ -104,6 +108,10 @@ std::optional<EmitOptions> parseEmitOptions(int Argc, char **Argv) {
       Options.ProfileGenerate = Value.str();
     else if (Arg == "--profile-use")
       Options.ProfileUse = Value.str();
+    else if (Arg == "--profile-blocks")
+      Options.ProfileBlocks = Value.str();
+    else if (Arg == "--record-profile-blocks")
+      Options.RecordProfileBlocks = Value.str();
     else if (Arg == "--split" && Value == "codegen")
       Options.SplitCodegen = true;
     else
@@ -245,37 +253,46 @@ struct UnnameStructTypes : PassInfoMixin<UnnameStructTypes> {
 // control flow changed since it was collected (hash mismatch), and functions
 // it has no record of (new since it was collected, or never run in training
 // when the profile was merged with --sparse). A partition reused from the cache without its counts is counted
-// in Unknown.
+// in Unknown. Recovered counts the mismatched functions that --profile-blocks
+// annotated, so it never exceeds Mismatched.
 struct ProfileMatch {
   std::atomic<uint64_t> Annotated{0};
   std::atomic<uint64_t> Mismatched{0};
   std::atomic<uint64_t> Missing{0};
+  std::atomic<uint64_t> Recovered{0};
   std::atomic<uint64_t> Unknown{0};
+  // Set once the annotation before inlining has run. The context-sensitive
+  // annotation after inlining, which a single-module pipeline also runs,
+  // reports its mismatched and missing functions again; they are not
+  // counted twice.
+  std::atomic<bool> Counted{false};
 
   void add(const ProfileMatch &Other) {
     Annotated += Other.Annotated;
     Mismatched += Other.Mismatched;
     Missing += Other.Missing;
+    Recovered += Other.Recovered;
   }
 
   std::string serialize() const {
-    return formatv("{0} {1} {2}", Annotated.load(), Mismatched.load(),
-                   Missing.load())
+    return formatv("{0} {1} {2} {3}", Annotated.load(), Mismatched.load(),
+                   Missing.load(), Recovered.load())
         .str();
   }
 
   bool deserialize(StringRef Text) {
-    SmallVector<StringRef, 3> Parts;
+    SmallVector<StringRef, 4> Parts;
     Text.split(Parts, ' ');
-    uint64_t Values[3];
-    if (Parts.size() != 3)
+    uint64_t Values[4];
+    if (Parts.size() != 4)
       return false;
-    for (size_t I = 0; I < 3; ++I)
+    for (size_t I = 0; I < 4; ++I)
       if (Parts[I].getAsInteger(10, Values[I]))
         return false;
     Annotated += Values[0];
     Mismatched += Values[1];
     Missing += Values[2];
+    Recovered += Values[3];
     return true;
   }
 };
@@ -285,7 +302,9 @@ struct ProfileMatch {
 // context's default handling.
 class ProfileMatchHandler : public DiagnosticHandler {
 public:
-  explicit ProfileMatchHandler(ProfileMatch &Match) : Match(Match) {}
+  explicit ProfileMatchHandler(
+      ProfileMatch &Match, std::vector<std::string> *MismatchedNames = nullptr)
+      : Match(Match), MismatchedNames(MismatchedNames) {}
 
   bool handleDiagnostics(const DiagnosticInfo &Info) override {
     if (Info.getKind() != DK_PGOProfile)
@@ -294,17 +313,28 @@ public:
     raw_string_ostream Stream(Message);
     DiagnosticPrinterRawOStream Printer(Stream);
     Info.print(Printer);
-    if (StringRef(Message).contains("hash mismatch"))
-      ++Match.Mismatched;
-    else if (StringRef(Message).contains_insensitive("no profile data"))
-      ++Match.Missing;
-    else
+    bool Mismatched = StringRef(Message).contains("hash mismatch");
+    if (!Mismatched &&
+        !StringRef(Message).contains_insensitive("no profile data"))
       return false;
+    if (Match.Counted)
+      return true;
+    if (!Mismatched) {
+      ++Match.Missing;
+      return true;
+    }
+    ++Match.Mismatched;
+    // "...(hash mismatch) <function> Hash = <hash>"
+    if (MismatchedNames != nullptr) {
+      StringRef Name = StringRef(Message).split("(hash mismatch) ").second;
+      MismatchedNames->push_back(Name.split(" Hash = ").first.str());
+    }
     return true;
   }
 
 private:
   ProfileMatch &Match;
+  std::vector<std::string> *MismatchedNames;
 };
 
 // The annotation warns about functions the profile does not describe only
@@ -319,17 +349,61 @@ void reportMissingProfiles() {
   (void)Reported;
 }
 
-// Functions of `M` that the profile annotated with counts.
-uint64_t annotatedFunctions(const Module &M) {
-  uint64_t Count = 0;
-  for (const Function &F : M) {
+// The --profile-blocks counts, read once before any partition needs them
+// (inspectProfile), or null without them.
+std::unique_ptr<BlockCounts> FallbackCounts;
+// The block counts a --record-profile-blocks build collects, or null.
+std::unique_ptr<BlockRecorder> Recorder;
+
+// Whether the profile annotation found F's control flow changed: it tags
+// such a function with this annotation.
+bool hasHashMismatchTag(const Function &F) {
+  if (const MDNode *Annotation = F.getMetadata(LLVMContext::MD_annotation))
+    for (const MDOperand &Operand : Annotation->operands())
+      if (const auto *Name = dyn_cast<MDString>(Operand))
+        if (Name->getString() == "instr_prof_hash_mismatch")
+          return true;
+  return false;
+}
+
+// Whether F has counts from a profile.
+bool hasProfileCounts(const Function &F) {
+  std::optional<Function::ProfileCount> Entry = F.getEntryCount();
+  return Entry && !Entry->isSynthetic();
+}
+
+// Runs right after the profile annotation, before inlining and dead code
+// elimination change the functions it annotated: counts them and records
+// their block counts, then annotates the functions whose control flow
+// changed from FallbackCounts. A function the profile has no record of keeps
+// no counts, as without block counts. Returns the analyses that remain valid.
+PreservedAnalyses afterProfileAnnotation(Module &M, ProfileMatch &Match) {
+  std::vector<Function *> Mismatched;
+  for (Function &F : M) {
     if (F.isDeclaration())
       continue;
-    std::optional<Function::ProfileCount> Entry = F.getEntryCount();
-    if (Entry && !Entry->isSynthetic())
-      ++Count;
+    if (!hasProfileCounts(F)) {
+      if (hasHashMismatchTag(F))
+        Mismatched.push_back(&F);
+      continue;
+    }
+    ++Match.Annotated;
+    if (Recorder)
+      Recorder->add(F);
   }
-  return Count;
+  if (!FallbackCounts)
+    return PreservedAnalyses::all();
+  ProfileSummaryInfo Summary(M);
+  PreservedAnalyses Preserved = PreservedAnalyses::all();
+  for (Function *F : Mismatched) {
+    auto Record = FallbackCounts->find(F->getName());
+    if (Record == FallbackCounts->end())
+      continue;
+    annotateFromBlocks(*F, Record->second, Summary);
+    ++Match.Recovered;
+    Preserved = PreservedAnalyses::none();
+  }
+  return Preserved;
 }
 
 // Prints the profile match as the successful emit's one line of JSON.
@@ -337,6 +411,7 @@ void reportProfileMatch(const ProfileMatch &Match) {
   json::Object Profile{{"annotated", Match.Annotated.load()},
                        {"mismatched", Match.Mismatched.load()},
                        {"missing", Match.Missing.load()},
+                       {"recovered", Match.Recovered.load()},
                        {"unknown_partitions", Match.Unknown.load()}};
   json::Object Response{{"ok", true}, {"profile", std::move(Profile)}};
   outs() << formatv("{0}\n", json::Value(std::move(Response)));
@@ -369,9 +444,36 @@ std::optional<EmitFailure> inspectProfile(const EmitOptions &Options) {
     enableProfileLayout();
     enableExtTspPlacement();
   }
+  if (!Options.ProfileBlocks.empty()) {
+    std::string Error;
+    FallbackCounts = readProfileBlocks(Options.ProfileBlocks, Error);
+    if (!FallbackCounts)
+      return EmitFailure{"profile_blocks_read_failed", Error};
+  }
+  if (!Options.RecordProfileBlocks.empty())
+    Recorder = std::make_unique<BlockRecorder>();
   return std::nullopt;
 }
 
+// Writes the block counts a --record-profile-blocks build collected.
+std::optional<EmitFailure> writeRecordedBlocks(const EmitOptions &Options) {
+  if (!Recorder)
+    return std::nullopt;
+  if (std::optional<std::string> Error =
+          Recorder->write(Options.RecordProfileBlocks))
+    return EmitFailure{"profile_blocks_write_failed",
+                       Options.RecordProfileBlocks + ": " + *Error};
+  return std::nullopt;
+}
+
+// The source file name a program's modules carry: the entry's file name,
+// without its directory. Profiles name a module's local functions after it
+// ("<name>;<function>"), so a profile collected in one checkout matches the
+// builds of every other one.
+std::string moduleSourceName(StringRef SourcePath) {
+  return sys::path::filename(SourcePath).str();
+}
+
 // Owns the analysis managers a pipeline needs for one module.
 struct Pipeline {
   PassInstrumentationCallbacks Callbacks;
@@ -391,16 +493,41 @@ struct Pipeline {
     PB.crossRegisterProxies(LAM, FAM, CGAM, MAM);
   }
 
-  // Adds to Count the functions the profile annotation gave counts to, as
-  // soon as that pass has run: inlining and dead code elimination later in
-  // the pipeline delete annotated functions.
-  void countAnnotated(std::atomic<uint64_t> &Count) {
+  // Runs afterProfileAnnotation as soon as the profile annotation has run.
+  // The context-sensitive annotation after inlining runs the same pass
+  // again, on functions the first one already counted.
+  void afterProfileAnnotation(ProfileMatch &Match) {
     Callbacks.registerAfterPassCallback(
-        [&Count](StringRef PassID, Any IR, const PreservedAnalyses &) {
+        [this, &Match, Done = false](StringRef PassID, Any IR,
+                                     const PreservedAnalyses &) mutable {
+          if (Done || PassID != "PGOInstrumentationUse")
+            return;
+          const auto *M = llvm::any_cast<const Module *>(&IR);
+          if (M == nullptr)
+            return;
+          Done = true;
+          // The callback receives the module the pipeline runs on; only the
+          // signature is const.
+          Module &Annotated = const_cast<Module &>(**M);
+          MAM.invalidate(Annotated,
+                         scriptc::afterProfileAnnotation(Annotated, Match));
+          Match.Counted = true;
+        });
+  }
+
+  // Adds to Names the profile names of the functions the annotation gave
+  // counts to, as soon as that pass has run.
+  void collectAnnotated(std::vector<std::string> &Names) {
+    Callbacks.registerAfterPassCallback(
+        [&Names](StringRef PassID, Any IR, const PreservedAnalyses &) {
           if (PassID != "PGOInstrumentationUse")
             return;
           if (const auto *M = llvm::any_cast<const Module *>(&IR))
-            Count += annotatedFunctions(**M);
+            for (const Function &F : **M) {
+              std::optional<Function::ProfileCount> Entry = F.getEntryCount();
+              if (!F.isDeclaration() && Entry && !Entry->isSynthetic())
+                Names.push_back(getIRPGOFuncName(F));
+            }
         });
   }
 };
@@ -592,7 +719,7 @@ emitModule(Module &M, TargetMachine &Machine, OptimizationLevel Level,
         std::make_unique<ProfileMatchHandler>(Match));
   Pipeline P(Machine, Level, std::move(Profile));
   if (Annotating)
-    P.countAnnotated(Match.Annotated);
+    P.afterProfileAnnotation(Match);
   P.PB.buildPerModuleDefaultPipeline(Level).run(M, P.MAM);
   if (std::optional<EmitFailure> Failure =
           verify(M, "post_optimization_verification_failed"))
@@ -678,7 +805,7 @@ Expected<std::unique_ptr<Module>> loadPartition(const EmitOptions &Options,
     return createStringError(inconvertibleErrorCode(), Stream.str());
   }
   if (!Options.SourcePath.empty())
-    Mod->setSourceFileName(Options.SourcePath);
+    Mod->setSourceFileName(moduleSourceName(Options.SourcePath));
   Mod->setTargetTriple(Triple(Options.Target));
   Mod->setDataLayout(Machine.createDataLayout());
   if (std::optional<EmitFailure> Failure = verify(*Mod, "verification_failed"))
@@ -767,7 +894,9 @@ compileOptimizedPartitions(const EmitOptions &Options,
   SharedFailure Failures;
   forEachPartition(Count, [&](size_t I) {
     std::string Key = hashParts({Salt, "summary", Plan.Modules[I]});
-    if ((Summaries[I] = Cache.lookup(Key, "bc"))) {
+    // A reused summary has no block counts to record, so a recording build
+    // annotates every partition again.
+    if (!Recorder && (Summaries[I] = Cache.lookup(Key, "bc"))) {
       if (Annotating) {
         std::unique_ptr<MemoryBuffer> Counts = Cache.lookup(Key, "match");
         if (!Counts || !Match.deserialize(Counts->getBuffer()))
@@ -809,12 +938,13 @@ compileOptimizedPartitions(const EmitOptions &Options,
       // link; the post-link optimization reads the annotated weights.
       Pipeline P(*Machine, Level, profileOptions(Options));
       if (Annotating)
-        P.countAnnotated(PartitionMatch.Annotated);
+        P.afterProfileAnnotation(PartitionMatch);
       ModulePassManager MPM = P.PB.buildThinLTOPreLinkDefaultPipeline(Level);
       MPM.addPass(UnnameStructTypes());
+      // The module hash is set at the thin link instead (see Names).
       MPM.addPass(BitcodeWriterPass(Stream, /*ShouldPreserveUseListOrder=*/false,
                                     /*EmitSummaryIndex=*/true,
-                                    /*EmitModuleHash=*/true));
+                                    /*EmitModuleHash=*/false));
       MPM.run(Mod, P.MAM);
     }
     if (Annotating) {
@@ -921,6 +1051,24 @@ compileOptimizedPartitions(const EmitOptions &Options,
   std::vector<std::string> Names(Count);
   for (size_t I = 0; I < Count; ++I)
     Names[I] = "partition-" + std::to_string(I);
+  // An internal function or variable that another partition imports a
+  // reference to is renamed <name>.llvm.<first 64 bits of its module's hash>.
+  // A hash of the module's contents would change that name in every
+  // importing partition after any edit of the exporting one, and compile
+  // them all again. Each module's hash is its partition number instead,
+  // which keeps the names unique within the program and the same across
+  // builds.
+  Config.CombinedIndexHook = [&Names](const ModuleSummaryIndex &Index,
+                                      const DenseSet<GlobalValue::GUID> &) {
+    StringMap<ModuleHash> &Paths =
+        const_cast<ModuleSummaryIndex &>(Index).modulePaths();
+    for (size_t I = 0; I < Names.size(); ++I) {
+      auto It = Paths.find(Names[I]);
+      if (It != Paths.end())
+        It->second = {0, static_cast<uint32_t>(I + 1), 0, 0, 0};
+    }
+    return true;
+  };
   limitImports();
   if (!Options.ProfileUse.empty())
     limitProfileImports();
@@ -997,6 +1145,17 @@ compilePartitions(const EmitOptions &Options, const PartitionPlan &Plan,
                              Profile.getError().message()};
     Salt = hashParts({Salt, "profile-use", (*Profile)->getBuffer()});
   }
+  // The block counts change the partitions they annotate.
+  if (!Options.ProfileBlocks.empty()) {
+    ErrorOr<std::unique_ptr<MemoryBuffer>> Blocks =
+        MemoryBuffer::getFile(Options.ProfileBlocks, /*IsText=*/false,
+                              /*RequiresNullTerminator=*/false);
+    if (!Blocks)
+      return EmitFailure{"profile_blocks_read_failed",
+                         Options.ProfileBlocks + ": " +
+                             Blocks.getError().message()};
+    Salt = hashParts({Salt, "profile-blocks", (*Blocks)->getBuffer()});
+  }
   for (const std::string &Path : Options.ImportBitcode) {
     ErrorOr<std::unique_ptr<MemoryBuffer>> Bitcode =
         MemoryBuffer::getFile(Path, /*IsText=*/false,
@@ -1060,8 +1219,22 @@ int emitPartitions(const EmitOptions &Options) {
         MemoryBuffer::getFile(Options.Input, /*IsText=*/false,
                               /*RequiresNullTerminator=*/false);
     if (Input) {
+      bool Optimized = Options.OptLevel != "0";
       std::optional<ProgramPartitions> Text = partitionProgramText(
-          (*Input)->getBuffer(), static_cast<unsigned>(Options.Outputs.size()));
+          (*Input)->getBuffer(), static_cast<unsigned>(Options.Outputs.size()),
+          /*SkipDominated=*/Optimized);
+      // An optimized partition is simplified before the thin link and
+      // optimized again after it, so one that holds more than two thirds of
+      // the program's function text takes longer than optimizing the module
+      // whole: such a module is optimized whole, and its code is generated in
+      // parts.
+      if (Text && Optimized && dominatedPartitions(Text->FunctionBytes)) {
+        EmitOptions Whole = Options;
+        Whole.SplitCodegen = true;
+        Whole.CacheDirectory.clear();
+        Input->reset();
+        return emit(Whole);
+      }
       if (Text) {
         Input->reset();
         PartitionPlan Plan{std::move(Text->Sources),
@@ -1082,6 +1255,8 @@ int emitPartitions(const EmitOptions &Options) {
     Failure = writePartitionObjects(Options, Objects, Temporaries);
   for (size_t I = 0; !Failure && I < Temporaries.size(); ++I)
     Failure = publish(Temporaries[I], Options.Outputs[I]);
+  if (!Failure)
+    Failure = writeRecordedBlocks(Options);
   if (Failure) {
     for (const SmallString<256> &Temporary : Temporaries)
       sys::fs::remove(Temporary);
@@ -1276,7 +1451,7 @@ emitSplitCodegen(const EmitOptions &Options, Module &M, TargetMachine &Machine,
   {
     Pipeline P(Machine, Level, profileOptions(Options));
     if (Annotating)
-      P.countAnnotated(Match.Annotated);
+      P.afterProfileAnnotation(Match);
     P.PB.buildPerModuleDefaultPipeline(Level).run(M, P.MAM);
   }
   if (std::optional<EmitFailure> Failure =
@@ -1338,8 +1513,100 @@ emitSplitCodegen(const EmitOptions &Options, Module &M, TargetMachine &Machine,
   return Failure;
 }
 
+// Lists the annotated and the mismatched functions of one module in
+// `Result`, each with the sum of its counters in the profile. `ProfileNames`
+// maps the module's symbol names, which the mismatch warnings use, to
+// profile names.
+std::optional<std::string>
+weighProfileMatch(const std::string &ProfileUse,
+                  const StringMap<std::string> &ProfileNames,
+                  const std::vector<std::string> &AnnotatedNames,
+                  const std::vector<std::string> &MismatchedNames,
+                  ModuleProfileMatch &Result) {
+  Expected<std::unique_ptr<IndexedInstrProfReader>> Reader =
+      IndexedInstrProfReader::create(ProfileUse, *vfs::getRealFileSystem());
+  if (!Reader)
+    return ProfileUse + ": " + toString(Reader.takeError());
+  StringMap<uint64_t> Counts;
+  for (const NamedInstrProfRecord &Record : **Reader) {
+    uint64_t Sum = 0;
+    for (uint64_t Count : Record.Counts)
+      Sum += Count;
+    Counts[Record.Name] += Sum;
+  }
+  if (Error E = (*Reader)->getError())
+    return ProfileUse + ": " + toString(std::move(E));
+  auto CountsOf = [&Counts](StringRef Name) -> uint64_t {
+    auto It = Counts.find(Name);
+    return It == Counts.end() ? 0 : It->second;
+  };
+  for (const std::string &Name : AnnotatedNames)
+    Result.AnnotatedCounts.emplace_back(Name, CountsOf(Name));
+  for (const std::string &Name : MismatchedNames) {
+    auto It = ProfileNames.find(Name);
+    std::string ProfileName = It == ProfileNames.end() ? Name : It->second;
+    uint64_t Sum = CountsOf(ProfileName);
+    Result.MismatchedCounts.emplace_back(std::move(ProfileName), Sum);
+  }
+  return std::nullopt;
+}
+
 } // namespace
 
+std::optional<std::string> optimizeModule(Module &M, TargetMachine &Machine,
+                                          const std::string &OptLevel,
+                                          bool Instrument,
+                                          const std::string &ProfileGenerate,
+                                          const std::string &ProfileUse,
+                                          ModuleProfileMatch &Result) {
+  OptimizationLevel Level = optimizationLevel(OptLevel);
+  std::optional<PGOOptions> Profile;
+  if (Instrument) {
+    Profile = PGOOptions(ProfileGenerate, "", "", "", PGOOptions::IRInstr);
+  } else if (!ProfileUse.empty()) {
+    EmitOptions Inspected;
+    Inspected.ProfileUse = ProfileUse;
+    if (std::optional<EmitFailure> Failure = inspectProfile(Inspected))
+      return Failure->Message;
+    reportMissingProfiles();
+    Profile = PGOOptions(ProfileUse, "", "", "", PGOOptions::IRUse);
+  }
+  bool Annotating = Profile && Profile->Action == PGOOptions::IRUse;
+  ProfileMatch Match;
+  // The annotation reports a mismatched function by its symbol name; the
+  // profile names local functions after their source file.
+  StringMap<std::string> ProfileNames;
+  std::vector<std::string> MismatchedNames;
+  std::vector<std::string> AnnotatedNames;
+  if (Annotating) {
+    for (const Function &F : M)
+      if (!F.isDeclaration())
+        ProfileNames[F.getName()] = getIRPGOFuncName(F);
+    M.getContext().setDiagnosticHandler(
+        std::make_unique<ProfileMatchHandler>(Match, &MismatchedNames));
+  }
+  {
+    Pipeline P(Machine, Level, std::move(Profile));
+    if (Annotating) {
+      P.afterProfileAnnotation(Match);
+      P.collectAnnotated(AnnotatedNames);
+    }
+    P.PB.buildPerModuleDefaultPipeline(Level).run(M, P.MAM);
+  }
+  Result.Annotated = Match.Annotated.load();
+  Result.Mismatched = Match.Mismatched.load();
+  Result.Missing = Match.Missing.load();
+  if (Annotating) {
+    if (std::optional<std::string> Failure = weighProfileMatch(
+            ProfileUse, ProfileNames, AnnotatedNames, MismatchedNames, Result))
+      return Failure;
+  }
+  if (std::optional<EmitFailure> Failure =
+          verify(M, "post_optimization_verification_failed"))
+    return Failure->Message;
+  return std::nullopt;
+}
+
 int emit(const EmitOptions &Options) {
   if (!supportsTarget(Options.Target))
     return reportError("unsupported_target",
@@ -1379,6 +1646,12 @@ int emit(const EmitOptions &Options) {
     return reportError("invalid_import",
                        "runtime bitcode import requires an optimized build",
                        Options.DiagnosticFormat);
+  if (Options.ProfileUse.empty() &&
+      (!Options.ProfileBlocks.empty() || !Options.RecordProfileBlocks.empty()))
+    return reportError("invalid_profile_blocks",
+                       "--profile-blocks and --record-profile-blocks require "
+                       "--profile-use",
+                       Options.DiagnosticFormat);
   if (!Options.ProfileUse.empty())
     reportMissingProfiles();
   if (std::optional<EmitFailure> Failure = inspectProfile(Options))
@@ -1405,7 +1678,7 @@ int emit(const EmitOptions &Options) {
     return reportError("invalid_ir", Stream.str(), Options.DiagnosticFormat);
   }
   if (!Options.SourcePath.empty())
-    Mod->setSourceFileName(Options.SourcePath);
+    Mod->setSourceFileName(moduleSourceName(Options.SourcePath));
 
   std::string LookupError;
   std::unique_ptr<TargetMachine> Machine =
@@ -1451,8 +1724,11 @@ int emit(const EmitOptions &Options) {
                              : CodeGenFileType::AssemblyFile;
   if (Options.SplitCodegen) {
     ProfileMatch Match;
-    if (std::optional<EmitFailure> Failure =
-            emitSplitCodegen(Options, *Mod, *Machine, Level, Match))
+    std::optional<EmitFailure> Failure =
+        emitSplitCodegen(Options, *Mod, *Machine, Level, Match);
+    if (!Failure)
+      Failure = writeRecordedBlocks(Options);
+    if (Failure)
       return reportError(Failure->Code, Failure->Message,
                          Options.DiagnosticFormat);
     if (!Options.ProfileUse.empty())
@@ -1466,6 +1742,8 @@ int emit(const EmitOptions &Options) {
                  Options.Outputs[0], Temporaries, Match);
   if (!Failure)
     Failure = publish(Temporaries[0], Options.Outputs[0]);
+  if (!Failure)
+    Failure = writeRecordedBlocks(Options);
   if (Failure) {
     for (const SmallString<256> &Temporary : Temporaries)
       sys::fs::remove(Temporary);

native/llvm-codegen/src/emit.h

@@ -1,9 +1,16 @@
 #pragma once
 
+#include <cstdint>
 #include <optional>
 #include <string>
+#include <utility>
 #include <vector>
 
+namespace llvm {
+class Module;
+class TargetMachine;
+} // namespace llvm
+
 namespace scriptc {
 
 struct EmitOptions {
@@ -28,6 +35,13 @@ struct EmitOptions {
   std::string ProfileGenerate;
   // Indexed profile (.profdata) that guides optimization. Empty: none.
   std::string ProfileUse;
+  // With ProfileUse: block counts (profile-blocks.h) that annotate the
+  // functions whose control flow changed since the profile was collected.
+  // Empty: none.
+  std::string ProfileBlocks;
+  // With ProfileUse: writes the block counts of the functions the profile
+  // annotates to this path. Empty: none.
+  std::string RecordProfileBlocks;
   // With several outputs: optimize the module whole and divide only its code
   // generation among the outputs.
   bool SplitCodegen = false;
@@ -36,4 +50,32 @@ struct EmitOptions {
 std::optional<EmitOptions> parseEmitOptions(int Argc, char **Argv);
 int emit(const EmitOptions &Options);
 
+// How a profile matched the functions of one optimized module: functions
+// that received its counts, functions whose control flow changed since it
+// was collected, and functions it has no record of. The annotated and the
+// mismatched functions are also listed by profile name, with the sum of the
+// profile's counters for each, so a changed hot function can weigh more than
+// a changed cold one.
+struct ModuleProfileMatch {
+  uint64_t Annotated = 0;
+  uint64_t Mismatched = 0;
+  uint64_t Missing = 0;
+  std::vector<std::pair<std::string, uint64_t>> AnnotatedCounts;
+  std::vector<std::pair<std::string, uint64_t>> MismatchedCounts;
+};
+
+// Optimizes `M` with the per-module default pipeline at `OptLevel`, as `emit`
+// does for a whole program, with IR instrumentation when `Instrument` is set
+// or annotation from the indexed `ProfileUse` profile. Instrumentation writes
+// raw profiles to the `ProfileGenerate` pattern; an empty pattern names no
+// file, so the program that links the module names it. Returns an error
+// message, or nothing on success.
+std::optional<std::string> optimizeModule(llvm::Module &M,
+                                          llvm::TargetMachine &Machine,
+                                          const std::string &OptLevel,
+                                          bool Instrument,
+                                          const std::string &ProfileGenerate,
+                                          const std::string &ProfileUse,
+                                          ModuleProfileMatch &Match);
+
 } // namespace scriptc

native/llvm-codegen/src/main.cpp

@@ -58,6 +58,9 @@ int version(int Argc, char **Argv) {
       {"supported_targets", std::move(SupportedTargets)},
       {"default_target", scriptc::DefaultTarget},
       {"data_layout", Machine->createDataLayout().getStringRepresentation()},
+      // Commands and options added within one protocol version, for callers
+      // that also run against older helpers.
+      {"features", json::Array{"runtime-unit-profile", "profile-functions"}},
   };
   outs() << formatv("{0}\n", json::Value(std::move(Response)));
   return 0;
@@ -80,6 +83,8 @@ int main(int Argc, char **Argv) {
     return scriptc::pruneImport(Argc, Argv);
   if (Argc >= 2 && StringRef(Argv[1]) == "merge-profiles")
     return scriptc::mergeProfiles(Argc, Argv);
+  if (Argc >= 2 && StringRef(Argv[1]) == "profile-functions")
+    return scriptc::profileFunctions(Argc, Argv);
   if (Argc >= 2 && StringRef(Argv[1]) == "emit") {
     std::optional<scriptc::EmitOptions> Options =
         scriptc::parseEmitOptions(Argc, Argv);
@@ -95,5 +100,6 @@ int main(int Argc, char **Argv) {
   }
   return scriptc::reportError("usage",
                               "expected 'version --format=json', 'emit', 'runtime-unit', "
-                              "'prune-import', or 'merge-profiles'");
+                              "'prune-import', 'merge-profiles', or "
+                              "'profile-functions'");
 }

native/llvm-codegen/src/partition.cpp

@@ -36,6 +36,10 @@ struct Symbol {
   // Local definitions are rewritten with hidden visibility.
   std::string PromotedHeader;
   std::vector<unsigned> Refs;
+  // The functions its body calls directly (sorted).
+  std::vector<unsigned> Calls;
+  // A variable defined with "constant", whose loads the optimizer can fold.
+  bool Constant = false;
   std::vector<unsigned> Metadata;
   // The source-module group the emitter wrote before this definition, or
   // empty.
@@ -74,6 +78,8 @@ uint32_t stableHash(StringRef Name) {
   return Hash;
 }
 
+constexpr unsigned MinGroupedPartitions = 32;
+
 // Places the functions whose root carries a source-module group by group: a
 // module's functions call each other most, so keeping them together leaves
 // fewer calls for the thin link to import across partitions. A group larger
@@ -83,11 +89,11 @@ uint32_t stableHash(StringRef Name) {
 // lighter of two partitions chosen by hashes of its key, after the functions
 // placed by name. The placement depends only on the module text and the
 // partition count. Modules with fewer than MinGroupedPartitions partitions
-// keep the placement by name: with few partitions per core, the balance of
-// the largest partitions decides the build time more than the imports.
+// keep the placement by name, which balanceRoots evens out: with few
+// partitions per core, the balance of the largest partitions decides the
+// build time more than the imports.
 void placeGroups(std::vector<Symbol> &Symbols, ArrayRef<unsigned> Roots,
                  unsigned Count) {
-  constexpr unsigned MinGroupedPartitions = 32;
   if (Count < MinGroupedPartitions)
     return;
   auto Bytes = [](const Symbol &S) -> uint64_t {
@@ -153,6 +159,67 @@ void placeGroups(std::vector<Symbol> &Symbols, ArrayRef<unsigned> Roots,
       Symbols[I].Partition = Where[Keys[I]];
 }
 
+// In modules of fewer than MinGroupedPartitions partitions, which
+// placeGroups leaves placed by name, the functions of each root (a
+// function and the chain that follows it) move together to another partition
+// when their name's partition would hold more than 1.25 times the mean
+// function bytes: each root probes the next partitions in turn, keeping the
+// first with room, else its own. An edit then recompiles about a mean
+// partition, and the largest partition, which bounds a cold build, stays
+// near the mean unless one root is larger. Roots and their bytes, and so
+// placement, depend only on the module text.
+void balanceRoots(std::vector<Symbol> &Symbols, ArrayRef<unsigned> Roots,
+                  ArrayRef<char> Live, unsigned Count) {
+  if (Count >= MinGroupedPartitions)
+    return;
+  DenseMap<unsigned, uint64_t> RootBytes;
+  uint64_t Total = 0;
+  for (unsigned I = 0; I < Symbols.size(); ++I)
+    if (Symbols[I].Kind == SymbolKind::Function && Live[I])
+      RootBytes[Roots[I]] += Symbols[I].Header.size() + Symbols[I].Body.size();
+  // Each root counts in whole 4 KiB units, so that an edit that grows a root
+  // by less than that leaves every comparison, and every other root's
+  // partition, unchanged.
+  constexpr uint64_t Unit = 4096;
+  for (auto &Entry : RootBytes) {
+    Entry.second = (Entry.second + Unit - 1) / Unit * Unit;
+    Total += Entry.second;
+  }
+  std::vector<unsigned> Order;
+  for (auto &Entry : RootBytes)
+    Order.push_back(Entry.first);
+  llvm::sort(Order, [&](unsigned A, unsigned B) {
+    return Symbols[A].Name < Symbols[B].Name;
+  });
+  uint64_t Cap = Total * 5 / 4 / Count;
+  std::vector<uint64_t> Load(Count, 0);
+  DenseMap<unsigned, unsigned> Placed;
+  // Roots of more than half the cap are placed first, in name order, so
+  // that small roots fill around them; the others follow in name order.
+  auto Place = [&](unsigned Root) {
+    uint64_t Bytes = RootBytes[Root];
+    unsigned Home = stableHash(Symbols[Root].Name) % Count, Chosen = Home;
+    for (unsigned K = 0; K < Count; ++K) {
+      unsigned P = (Home + K) % Count;
+      if (Load[P] + Bytes <= Cap) {
+        Chosen = P;
+        break;
+      }
+    }
+    Load[Chosen] += Bytes;
+    Placed[Root] = Chosen;
+  };
+  for (unsigned Root : Order)
+    if (RootBytes[Root] * 2 > Cap)
+      Place(Root);
+  for (unsigned Root : Order)
+    if (RootBytes[Root] * 2 <= Cap)
+      Place(Root);
+  for (unsigned I = 0; I < Symbols.size(); ++I)
+    if (Symbols[I].Kind == SymbolKind::Function && Live[I])
+      Symbols[I].Partition = Placed[Roots[I]];
+}
+
 bool startsWithLinkage(StringRef Rest) {
   for (StringRef Linkage :
        {"linkonce", "weak", "common", "appending", "extern_weak",
@@ -278,6 +345,7 @@ bool parseGlobal(StringRef Line, Symbol &S) {
     }
     if (Cursor.consume_front("constant ")) {
       Keyword = "constant";
+      S.Constant = true;
       break;
     }
     size_t Space = Cursor.find(' ');
@@ -325,10 +393,13 @@ bool parseGlobal(StringRef Line, Symbol &S) {
 }
 
 // Collects the symbols (sorted) and metadata nodes (in order of first
-// appearance) that Text references outside quoted strings.
+// appearance) that Text references outside quoted strings, and in Calls the
+// symbols it names as direct callees, which the emitter writes followed by
+// "(".
 void scanReferences(StringRef Text, const StringMap<unsigned> &Index,
                     std::vector<unsigned> &Refs,
-                    std::vector<unsigned> &Metadata) {
+                    std::vector<unsigned> &Metadata,
+                    std::vector<unsigned> *Calls = nullptr) {
   DenseSet<unsigned> Seen(Metadata.begin(), Metadata.end());
   bool Quoted = false;
   for (size_t I = 0; I < Text.size(); ++I) {
@@ -343,8 +414,12 @@ void scanReferences(StringRef Text, const StringMap<unsigned> &Index,
       size_t Length = nameLength(Text, I + 1);
       if (Length != 0) {
         auto It = Index.find(Text.substr(I + 1, Length));
-        if (It != Index.end())
+        if (It != Index.end()) {
           Refs.push_back(It->second);
+          if (Calls && I + 1 + Length < Text.size() &&
+              Text[I + 1 + Length] == '(')
+            Calls->push_back(It->second);
+        }
         I += Length;
       }
     } else if (C == '!' && I + 1 < Text.size() && isDigit(Text[I + 1])) {
@@ -359,6 +434,10 @@ void scanReferences(StringRef Text, const StringMap<unsigned> &Index,
   }
   llvm::sort(Refs);
   Refs.erase(std::unique(Refs.begin(), Refs.end()), Refs.end());
+  if (Calls) {
+    llvm::sort(*Calls);
+    Calls->erase(std::unique(Calls->begin(), Calls->end()), Calls->end());
+  }
 }
 
 // Appends Text with every unquoted metadata reference renumbered by Map.
@@ -392,8 +471,18 @@ void appendRenumbered(StringRef Text, const DenseMap<unsigned, unsigned> &Map,
 
 } // namespace
 
+bool dominatedPartitions(ArrayRef<uint64_t> FunctionBytes) {
+  uint64_t Total = 0, Largest = 0;
+  for (uint64_t Bytes : FunctionBytes) {
+    Total += Bytes;
+    Largest = std::max(Largest, Bytes);
+  }
+  return Largest * 3 > Total * 2;
+}
+
 std::optional<ProgramPartitions> partitionProgramText(StringRef Text,
-                                                      unsigned Count) {
+                                                      unsigned Count,
+                                                      bool SkipDominated) {
   if (Count < 2)
     return std::nullopt;
   std::vector<StringRef> Header, Types, Attributes, NamedMetadata;
@@ -529,7 +618,7 @@ std::optional<ProgramPartitions> partitionProgramText(StringRef Text,
     scanReferences(S.Header, Index, S.Refs, S.Metadata);
     if (!S.Body.empty()) {
       std::vector<unsigned> Refs;
-      scanReferences(S.Body, Index, Refs, S.Metadata);
+      scanReferences(S.Body, Index, Refs, S.Metadata, &S.Calls);
       S.Refs.insert(S.Refs.end(), Refs.begin(), Refs.end());
       llvm::sort(S.Refs);
       S.Refs.erase(std::unique(S.Refs.begin(), S.Refs.end()), S.Refs.end());
@@ -584,15 +673,36 @@ std::optional<ProgramPartitions> partitionProgramText(StringRef Text,
   // inlining candidate of it, or data only it uses; keep it with that
   // definition so its partition can inline it without importing, and so a
   // helper an edit adds lands in the partition of the code that uses it. A
-  // chain through a global continues to the function that uses the global.
+  // function that its referrer only takes the address of is not an inlining
+  // candidate: a closure body whose address its creator passes to the
+  // runtime, or a compiled regular expression's matcher that a mutable
+  // global points to, gives the referrer nothing to inline, and joining it
+  // would make every edit of the referrer optimize it again. Such a function
+  // is placed by its own name, unless its body is under AddressOnlyBytes (a
+  // trampoline or thread entry, which costs little to optimize again). A
+  // chain through a constant global, whose loads the optimizer can turn into
+  // direct calls, continues to the function that uses the global.
   // A function body larger than this is past LLVM's inline cost limit even
   // for the only call to a local function, so it is placed by its own name.
-  // A chain stops where it would leave a function's source-module group:
-  // module initializers, which only the entry point calls, keep their own
-  // partitions instead of all sharing the entry point's.
+  // A chain stops where it would leave a function's source-module group for
+  // a function outside every group. In a module of more than
+  // GroupedChainBytes of LLVM text, it also stops where it would join a
+  // function of another group: module initializers, which only the entry
+  // point calls, keep their own partitions instead of all sharing the entry
+  // point's. In a smaller module, a function that one function of another
+  // source file calls stays with it, so it can still be inlined there as in
+  // the whole module; when that gathers most of the program into one
+  // partition, the program is optimized whole (see emitPartitions).
   constexpr size_t MaxInlinedBodyBytes = 256 * 1024;
+  constexpr size_t GroupedChainBytes = 1024 * 1024;
+  constexpr size_t AddressOnlyBytes = 512;
+  bool GroupedChains = Text.size() > GroupedChainBytes;
   constexpr unsigned Shared = ~1u;
   std::vector<unsigned> Referrer(Symbols.size(), NoPartition);
+  // Whether a definition may follow its first referrer: it is not a
+  // function, or the referrer calls it or is a constant global, or its body
+  // is under AddressOnlyBytes.
+  std::vector<char> MayFollow(Symbols.size(), 0);
   for (unsigned I = 0; I < Symbols.size(); ++I) {
     const Symbol &S = Symbols[I];
     if (S.Kind == SymbolKind::Declaration || !Live[I])
@@ -600,18 +710,29 @@ std::optional<ProgramPartitions> partitionProgramText(StringRef Text,
     for (unsigned Ref : S.Refs) {
       if (Ref == I)
         continue;
-      Referrer[Ref] = Referrer[Ref] == NoPartition ? I : Shared;
+      if (Referrer[Ref] == NoPartition) {
+        Referrer[Ref] = I;
+        MayFollow[Ref] =
+            Symbols[Ref].Kind != SymbolKind::Function ||
+            Symbols[Ref].Body.size() < AddressOnlyBytes ||
+            (S.Kind == SymbolKind::Function
+                 ? std::binary_search(S.Calls.begin(), S.Calls.end(), Ref)
+                 : S.Constant);
+      } else {
+        Referrer[Ref] = Shared;
+      }
     }
   }
   auto Follows = [&](unsigned Child, unsigned Parent) {
     const Symbol &S = Symbols[Child];
-    if (!S.Local || Parent == NoPartition || Parent == Shared ||
-        Symbols[Parent].Kind == SymbolKind::Declaration)
+    if (!S.Local || !MayFollow[Child] || Parent == NoPartition ||
+        Parent == Shared || Symbols[Parent].Kind == SymbolKind::Declaration)
       return false;
     if (S.Kind == SymbolKind::Function &&
         (S.Body.size() > MaxInlinedBodyBytes ||
          (Symbols[Parent].Kind == SymbolKind::Function && !S.Group.empty() &&
-          S.Group != Symbols[Parent].Group)))
+          S.Group != Symbols[Parent].Group &&
+          (GroupedChains || Symbols[Parent].Group.empty()))))
       return false;
     return true;
   };
@@ -637,6 +758,7 @@ std::optional<ProgramPartitions> partitionProgramText(StringRef Text,
       Symbols[I].Partition = stableHash(Symbols[Root].Name) % Count;
   }
   placeGroups(Symbols, Roots, Count);
+  balanceRoots(Symbols, Roots, Live, Count);
   // A global follows the function its chain ends at, then the first
   // function that references it, then its own name.
   for (unsigned I = 0; I < Symbols.size(); ++I)
@@ -663,10 +785,41 @@ std::optional<ProgramPartitions> partitionProgramText(StringRef Text,
       Result.PublicSymbols.insert(S.Name);
 
   Result.Sources.resize(Count);
+  Result.FunctionBytes.assign(Count, 0);
   std::vector<std::vector<unsigned>> Owned(Count);
   for (unsigned I = 0; I < Symbols.size(); ++I)
-    if (Symbols[I].Kind != SymbolKind::Declaration && Live[I])
+    if (Symbols[I].Kind != SymbolKind::Declaration && Live[I]) {
       Owned[Symbols[I].Partition].push_back(I);
+      if (Symbols[I].Kind == SymbolKind::Function)
+        Result.FunctionBytes[Symbols[I].Partition] +=
+            Symbols[I].Header.size() + Symbols[I].Body.size();
+    }
+  // The caller optimizes such a module whole, so its partitions' text is
+  // not needed.
+  if (SkipDominated && dominatedPartitions(Result.FunctionBytes)) {
+    Result.Sources.clear();
+    return Result;
+  }
+
+  // An internal definition stays internal, so its partition's optimizer can
+  // inline a function into its last caller and delete it as the whole module
+  // would. It is promoted to hidden visibility only when a definition of
+  // another partition or a metadata node names it.
+  std::vector<char> Exported(Symbols.size(), 0);
+  for (unsigned I : NamedSymbols)
+    Exported[I] = 1;
+  for (auto &Entry : Metadata)
+    for (unsigned I : Entry.second.Symbols)
+      Exported[I] = 1;
+  for (unsigned I = 0; I < Symbols.size(); ++I) {
+    const Symbol &S = Symbols[I];
+    if (S.Kind == SymbolKind::Declaration || !Live[I])
+      continue;
+    for (unsigned Ref : S.Refs)
+      if (Symbols[Ref].Kind != SymbolKind::Declaration &&
+          Symbols[Ref].Partition != S.Partition)
+        Exported[Ref] = 1;
+  }
 
   auto Assemble = [&](unsigned P) {
     std::vector<char> Needed(Symbols.size(), 0);
@@ -731,7 +884,7 @@ std::optional<ProgramPartitions> partitionProgramText(StringRef Text,
         const Symbol &S = Symbols[I];
         if (S.Kind != Kind)
           continue;
-        Line(S.Local ? StringRef(S.PromotedHeader) : S.Header);
+        Line(S.Local && Exported[I] ? StringRef(S.PromotedHeader) : S.Header);
         if (!S.Body.empty())
           Line(S.Body);
       }

native/llvm-codegen/src/partition.h

@@ -1,8 +1,10 @@
 #pragma once
 
+#include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/StringSet.h"
 
+#include <cstdint>
 #include <optional>
 #include <string>
 #include <vector>
@@ -14,9 +16,11 @@ struct ProgramPartitions {
   // One complete LLVM assembly module per partition.
   std::vector<std::string> Sources;
   // Definitions that were not internal in the original module. Every other
-  // definition was internal and is promoted to hidden visibility only so the
-  // partitions can reference each other.
+  // definition was internal, and is promoted to hidden visibility only when
+  // another partition references it.
   llvm::StringSet<> PublicSymbols;
+  // The bytes of the function definitions placed in each partition.
+  std::vector<uint64_t> FunctionBytes;
 };
 
 // Divides the LLVM assembly scriptc emits into Count modules without parsing
@@ -27,13 +31,21 @@ struct ProgramPartitions {
 // functions are placed by a stable hash of their name. An internal
 // definition referenced by exactly one other definition stays with it, also
 // through a global, unless it is a function whose body is too large to
-// inline or that would join a function of another source-module group; a
-// global otherwise stays with the first function that references it.
+// inline, that would leave its source-module group for a function outside
+// every group, or, in a module of more than 1 MiB, that would join a function
+// of another source-module group; a global otherwise stays with the first
+// function that references it.
 // Internal definitions that no non-internal definition, named metadata or
 // metadata node reaches are left out of every partition. Returns nullopt
 // for any construct outside the emitter's narrow top-level grammar; callers
-// then split a parsed module.
+// then split a parsed module. With SkipDominated, a division that
+// dominatedPartitions rejects has no Sources.
 std::optional<ProgramPartitions> partitionProgramText(llvm::StringRef Text,
-                                                      unsigned Count);
+                                                      unsigned Count,
+                                                      bool SkipDominated);
+
+// Whether the largest partition would hold more than two thirds of the
+// program's function bytes.
+bool dominatedPartitions(llvm::ArrayRef<uint64_t> FunctionBytes);
 
 } // namespace scriptc

native/llvm-codegen/src/profile-blocks.cpp

@@ -0,0 +1,503 @@
+#include "profile-blocks.h"
+
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/DepthFirstIterator.h"
+#include "llvm/ADT/SmallPtrSet.h"
+#include "llvm/ADT/SmallString.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringExtras.h"
+#include "llvm/Analysis/BlockFrequencyInfo.h"
+#include "llvm/Analysis/BranchProbabilityInfo.h"
+#include "llvm/Analysis/LoopInfo.h"
+#include "llvm/Analysis/ProfileSummaryInfo.h"
+#include "llvm/IR/CFG.h"
+#include "llvm/IR/Dominators.h"
+#include "llvm/IR/Function.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/ProfDataUtils.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/raw_ostream.h"
+#include "llvm/Transforms/Instrumentation/PGOInstrumentation.h"
+#include "llvm/Transforms/Utils/SampleProfileInference.h"
+
+#include <algorithm>
+
+using namespace llvm;
+
+namespace scriptc {
+
+namespace {
+
+constexpr StringRef Header = "scriptc-profile-blocks 1";
+
+// An edge whose branch weight is below this fraction of its branch's total
+// counts as unlikely in flow inference.
+constexpr uint64_t UnlikelyRatio = 1000;
+
+// Alignments of more blocks than this product fall back to matching labels
+// that occur once on each side.
+constexpr uint64_t MaxAlignmentCells = uint64_t(1) << 22;
+
+void writeEscaped(raw_ostream &OS, StringRef Text) {
+  if (Text.empty()) {
+    OS << '\\';
+    return;
+  }
+  for (unsigned char C : Text) {
+    if (C <= ' ' || C >= 0x7f || C == '\\')
+      OS << '\\' << hexdigit(C >> 4, /*LowerCase=*/true)
+         << hexdigit(C & 15, /*LowerCase=*/true);
+    else
+      OS << C;
+  }
+}
+
+bool readEscaped(StringRef Token, std::string &Out) {
+  Out.clear();
+  if (Token == "\\")
+    return true;
+  for (size_t I = 0; I < Token.size(); ++I) {
+    if (Token[I] != '\\') {
+      Out.push_back(Token[I]);
+      continue;
+    }
+    if (I + 2 >= Token.size())
+      return false;
+    unsigned High = hexDigitValue(Token[I + 1]);
+    unsigned Low = hexDigitValue(Token[I + 2]);
+    if (High > 15 || Low > 15)
+      return false;
+    Out.push_back(static_cast<char>(High << 4 | Low));
+    I += 2;
+  }
+  return true;
+}
+
+// A block label's kind: the label without its digits. The emitter numbers
+// each function's blocks in order (`if.t3`, `loop.c4`), so a block added or
+// removed earlier in the function renumbers the blocks after it but keeps
+// their kinds.
+std::string kindOf(StringRef Label) {
+  std::string Kind;
+  Kind.reserve(Label.size());
+  for (char C : Label)
+    if (!isDigit(C))
+      Kind.push_back(C);
+  return Kind;
+}
+
+struct Token {
+  std::string Kind;
+  StringRef Label;
+  size_t Successors;
+};
+
+bool sameKind(const Token &A, const Token &B) {
+  return A.Successors == B.Successors && A.Kind == B.Kind;
+}
+
+// Pairs every block of Blocks with the recorded block that holds its place,
+// or -1: the longest common subsequence of the two block orders by kind and
+// successor count, which prefers pairs whose labels are equal. Common
+// leading and trailing runs pair up first; a remaining middle too large to
+// align pairs the labels that occur once on each side.
+std::vector<int64_t> alignBlocks(const FunctionBlocks &Record,
+                                 ArrayRef<const BasicBlock *> Blocks) {
+  std::vector<Token> Old, New;
+  Old.reserve(Record.Blocks.size());
+  for (const BlockRecord &Block : Record.Blocks)
+    Old.push_back({kindOf(Block.Label), Block.Label, Block.Successors.size()});
+  New.reserve(Blocks.size());
+  for (const BasicBlock *Block : Blocks) {
+    SmallPtrSet<const BasicBlock *, 4> Distinct(succ_begin(Block),
+                                                succ_end(Block));
+    New.push_back(
+        {kindOf(Block->getName()), Block->getName(), Distinct.size()});
+  }
+  std::vector<int64_t> Match(New.size(), -1);
+  size_t Begin = 0;
+  while (Begin < Old.size() && Begin < New.size() &&
+         sameKind(Old[Begin], New[Begin])) {
+    Match[Begin] = static_cast<int64_t>(Begin);
+    ++Begin;
+  }
+  size_t OldEnd = Old.size(), NewEnd = New.size();
+  while (OldEnd > Begin && NewEnd > Begin &&
+         sameKind(Old[OldEnd - 1], New[NewEnd - 1])) {
+    --OldEnd;
+    --NewEnd;
+    Match[NewEnd] = static_cast<int64_t>(OldEnd);
+  }
+  size_t Rows = OldEnd - Begin, Columns = NewEnd - Begin;
+  if (Rows == 0 || Columns == 0)
+    return Match;
+  auto Score = [&](size_t I, size_t J) -> uint32_t {
+    const Token &A = Old[Begin + I], &B = New[Begin + J];
+    if (!sameKind(A, B))
+      return 0;
+    return A.Label == B.Label ? 2 : 1;
+  };
+  if (uint64_t(Rows) * Columns <= MaxAlignmentCells) {
+    // Best[I][J]: the best score of the middle's suffixes from I and J.
+    size_t Width = Columns + 1;
+    std::vector<uint32_t> Best((Rows + 1) * Width, 0);
+    for (size_t I = Rows; I-- > 0;)
+      for (size_t J = Columns; J-- > 0;) {
+        uint32_t Value =
+            std::max(Best[(I + 1) * Width + J], Best[I * Width + J + 1]);
+        if (uint32_t S = Score(I, J))
+          Value = std::max(Value, S + Best[(I + 1) * Width + J + 1]);
+        Best[I * Width + J] = Value;
+      }
+    size_t I = 0, J = 0;
+    while (I < Rows && J < Columns) {
+      uint32_t S = Score(I, J);
+      if (S && Best[I * Width + J] == S + Best[(I + 1) * Width + J + 1]) {
+        Match[Begin + J] = static_cast<int64_t>(Begin + I);
+        ++I;
+        ++J;
+      } else if (Best[I * Width + J] == Best[(I + 1) * Width + J]) {
+        ++I;
+      } else {
+        ++J;
+      }
+    }
+    return Match;
+  }
+  StringMap<int64_t> OldLabels, NewLabels;
+  for (size_t I = Begin; I < OldEnd; ++I) {
+    auto [It, Inserted] = OldLabels.try_emplace(Old[I].Label, I);
+    if (!Inserted)
+      It->second = -1;
+  }
+  for (size_t J = Begin; J < NewEnd; ++J) {
+    auto [It, Inserted] = NewLabels.try_emplace(New[J].Label, J);
+    if (!Inserted)
+      It->second = -1;
+  }
+  for (const auto &Entry : NewLabels) {
+    auto It = OldLabels.find(Entry.getKey());
+    if (Entry.second < 0 || It == OldLabels.end() || It->second < 0 ||
+        !sameKind(Old[It->second], New[Entry.second]))
+      continue;
+    Match[Entry.second] = It->second;
+  }
+  return Match;
+}
+
+// LLVM's profile inference costs, with recorded edge counts that cost as much
+// to change as recorded block counts. Its default leaves edges free to
+// change, and its rebalancing of the flow below a counted block spreads it
+// evenly over the block's edges whatever their counts.
+ProfiParams inferenceParams() {
+  ProfiParams Params;
+  Params.EvenFlowDistribution = true;
+  Params.RebalanceUnknown = false;
+  Params.JoinIslands = true;
+  Params.CostBlockInc = 10;
+  Params.CostBlockDec = 20;
+  Params.CostBlockEntryInc = 40;
+  Params.CostBlockEntryDec = 10;
+  Params.CostBlockZeroInc = 11;
+  Params.CostBlockUnknownInc = 0;
+  Params.CostJumpInc = 10;
+  Params.CostJumpFTInc = 10;
+  Params.CostJumpDec = 20;
+  Params.CostJumpFTDec = 20;
+  Params.CostJumpUnknownInc = 0;
+  Params.CostJumpUnknownFTInc = 0;
+  return Params;
+}
+
+void markFrequency(Function &F, uint64_t EntryCount, uint64_t MaxCount,
+                   const ProfileSummaryInfo &Summary) {
+  if (Summary.isHotCount(EntryCount))
+    F.addFnAttr(Attribute::InlineHint);
+  else if (Summary.isColdCount(MaxCount) && !F.hasFnAttribute(Attribute::Hot))
+    F.addFnAttr(Attribute::Cold);
+}
+
+} // namespace
+
+std::unique_ptr<BlockCounts> readProfileBlocks(StringRef Path,
+                                                 std::string &Error) {
+  ErrorOr<std::unique_ptr<MemoryBuffer>> Buffer = MemoryBuffer::getFile(Path);
+  if (!Buffer) {
+    Error = Path.str() + ": " + Buffer.getError().message();
+    return nullptr;
+  }
+  auto Blocks = std::make_unique<BlockCounts>();
+  StringRef Rest = (*Buffer)->getBuffer();
+  size_t Line = 0;
+  auto Fail = [&](StringRef Why) {
+    Error = (Path + ":" + Twine(Line) + ": " + Why).str();
+    return nullptr;
+  };
+  auto NextLine = [&]() {
+    auto [Current, Remaining] = Rest.split('\n');
+    Rest = Remaining;
+    ++Line;
+    return Current;
+  };
+  if (NextLine() != Header)
+    return Fail("not a scriptc profile blocks file");
+  SmallVector<StringRef, 16> Fields;
+  std::string Name, Label;
+  while (!Rest.empty()) {
+    Fields.clear();
+    NextLine().split(Fields, ' ', -1, /*KeepEmpty=*/false);
+    uint64_t Entry = 0, Count = 0;
+    if (Fields.size() != 4 || Fields[0] != "f" ||
+        !readEscaped(Fields[1], Name) || Fields[2].getAsInteger(10, Entry) ||
+        Fields[3].getAsInteger(10, Count))
+      return Fail("expected a function line");
+    FunctionBlocks &Function = (*Blocks)[Name];
+    Function.EntryCount = Entry;
+    Function.Blocks.resize(Count);
+    for (BlockRecord &Block : Function.Blocks) {
+      Fields.clear();
+      if (Rest.empty())
+        return Fail("expected a block line");
+      NextLine().split(Fields, ' ', -1, /*KeepEmpty=*/false);
+      if (Fields.size() < 2 || !readEscaped(Fields[0], Label) ||
+          Fields[1].getAsInteger(10, Block.Count))
+        return Fail("expected a block line");
+      Block.Label = Label;
+      for (StringRef Field : ArrayRef(Fields).drop_front(2)) {
+        auto [Position, EdgeCount] = Field.split(':');
+        uint32_t Successor = 0;
+        uint64_t Value = 0;
+        if (Position.getAsInteger(10, Successor) ||
+            EdgeCount.getAsInteger(10, Value) || Successor >= Count)
+          return Fail("expected a successor and its edge count");
+        Block.Successors.push_back({Successor, Value});
+      }
+    }
+  }
+  return Blocks;
+}
+
+void annotateFromBlocks(Function &F, const FunctionBlocks &Record,
+                        const ProfileSummaryInfo &Summary) {
+  if (Record.EntryCount == 0 || Record.Blocks.empty()) {
+    F.setEntryCount(Function::ProfileCount(0, Function::PCT_Real));
+    if (Summary.hasProfileSummary() && !F.hasFnAttribute(Attribute::Hot))
+      F.addFnAttr(Attribute::Cold);
+    return;
+  }
+  std::vector<const BasicBlock *> Order;
+  for (const BasicBlock &Block : F)
+    Order.push_back(&Block);
+  std::vector<int64_t> Match = alignBlocks(Record, Order);
+  std::vector<bool> Paired(Record.Blocks.size(), false);
+  for (int64_t Recorded : Match)
+    if (Recorded >= 0)
+      Paired[Recorded] = true;
+
+  // Flow runs through the blocks that the entry reaches and that reach an
+  // exit, in the function's order, as LLVM's profile inference requires.
+  df_iterator_default_set<const BasicBlock *> Forward, Backward;
+  for (const BasicBlock *Block : depth_first_ext(&F, Forward))
+    (void)Block;
+  for (const BasicBlock &Block : F)
+    if (succ_empty(&Block))
+      for (const BasicBlock *Reaching :
+           inverse_depth_first_ext(&Block, Backward))
+        (void)Reaching;
+  DenseMap<const BasicBlock *, size_t> Position, Node;
+  FlowFunction Flow;
+  for (size_t I = 0; I < Order.size(); ++I) {
+    Position[Order[I]] = I;
+    if (!Forward.contains(Order[I]) || !Backward.contains(Order[I]))
+      continue;
+    Node[Order[I]] = Flow.Blocks.size();
+    FlowBlock Block;
+    Block.Index = Flow.Blocks.size();
+    if (Match[I] >= 0) {
+      Block.Weight = Record.Blocks[Match[I]].Count;
+      Block.HasUnknownWeight = false;
+    } else if (I == 0) {
+      Block.Weight = Record.EntryCount;
+      Block.HasUnknownWeight = false;
+    }
+    Flow.Blocks.push_back(Block);
+  }
+  uint64_t EntryCount = Record.EntryCount, MaxCount = Record.EntryCount;
+  if (Flow.Blocks.size() > 1 && Node.count(&F.getEntryBlock())) {
+    for (const BasicBlock *Block : Order) {
+      auto Source = Node.find(Block);
+      if (Source == Node.end())
+        continue;
+      int64_t From = Match[Position[Block]];
+      SmallVector<const BasicBlock *, 4> Targets, Successors;
+      for (const BasicBlock *Successor : successors(Block)) {
+        Successors.push_back(Successor);
+        if (!is_contained(Targets, Successor))
+          Targets.push_back(Successor);
+      }
+      SmallVector<uint32_t, 4> Weights;
+      extractBranchWeights(*Block->getTerminator(), Weights);
+      const BlockRecord *Recorded = From >= 0 ? &Record.Blocks[From] : nullptr;
+      for (size_t K = 0; K < Targets.size(); ++K) {
+        auto Target = Node.find(Targets[K]);
+        if (Target == Node.end())
+          continue;
+        FlowJump Jump;
+        Jump.Source = Source->second;
+        Jump.Target = Target->second;
+        if (Recorded != nullptr) {
+          // An edge to the recorded target's pair takes that edge's count.
+          // An edge to an unpaired target takes the count of the recorded
+          // edge in its place among the block's targets, which a paired
+          // block has as many of, when that edge's target is unpaired too:
+          // a target whose label changed keeps its edge's count.
+          int64_t To = Match[Position[Targets[K]]];
+          auto Edge = llvm::find_if(Recorded->Successors, [&](const auto &E) {
+            return To >= 0 && E.first == static_cast<uint64_t>(To);
+          });
+          if (Edge == Recorded->Successors.end() && To < 0 &&
+              Recorded->Successors.size() == Targets.size() &&
+              !Paired[Recorded->Successors[K].first])
+            Edge = Recorded->Successors.begin() + K;
+          if (Edge != Recorded->Successors.end()) {
+            Jump.Weight = Edge->second;
+            Jump.HasUnknownWeight = false;
+          }
+        }
+        // An edge without a recorded count that the function's own branch
+        // weights (from llvm.expect) make unlikely takes flow only when no
+        // other path can.
+        if (Jump.HasUnknownWeight && Weights.size() == Successors.size()) {
+          uint64_t Total = 0, Weight = 0;
+          for (size_t I = 0; I < Successors.size(); ++I) {
+            Total += Weights[I];
+            if (Successors[I] == Targets[K])
+              Weight += Weights[I];
+          }
+          Jump.IsUnlikely = Weight * UnlikelyRatio < Total;
+        }
+        Flow.Jumps.push_back(Jump);
+      }
+    }
+    for (FlowJump &Jump : Flow.Jumps) {
+      Flow.Blocks[Jump.Source].SuccJumps.push_back(&Jump);
+      Flow.Blocks[Jump.Target].PredJumps.push_back(&Jump);
+    }
+    applyFlowInference(inferenceParams(), Flow);
+    EntryCount = Flow.Blocks[0].Flow;
+    MaxCount = 0;
+    for (const FlowBlock &Block : Flow.Blocks)
+      MaxCount = std::max(MaxCount, Block.Flow);
+    if (MaxCount > 0 && EntryCount == 0)
+      EntryCount = 1;
+    // Branch weights as the profile annotation sets them: the first
+    // successor position of each target takes the edge's count.
+    DenseMap<std::pair<size_t, size_t>, uint64_t> JumpFlow;
+    for (const FlowJump &Jump : Flow.Jumps)
+      JumpFlow[{Jump.Source, Jump.Target}] = Jump.Flow;
+    for (BasicBlock &Block : F) {
+      auto Source = Node.find(&Block);
+      Instruction *Terminator = Block.getTerminator();
+      if (Source == Node.end() || Flow.Blocks[Source->second].Flow == 0 ||
+          Terminator->getNumSuccessors() < 2 ||
+          !(isa<BranchInst>(Terminator) || isa<SwitchInst>(Terminator) ||
+            isa<IndirectBrInst>(Terminator) || isa<InvokeInst>(Terminator) ||
+            isa<CallBrInst>(Terminator)))
+        continue;
+      SmallVector<uint64_t, 4> Counts(Terminator->getNumSuccessors(), 0);
+      SmallPtrSet<const BasicBlock *, 4> Seen;
+      uint64_t Max = 0;
+      for (unsigned I = 0; I < Terminator->getNumSuccessors(); ++I) {
+        const BasicBlock *Successor = Terminator->getSuccessor(I);
+        auto Target = Node.find(Successor);
+        if (Target == Node.end() || !Seen.insert(Successor).second)
+          continue;
+        Counts[I] = JumpFlow.lookup({Source->second, Target->second});
+        Max = std::max(Max, Counts[I]);
+      }
+      if (Max > 0)
+        setProfMetadata(Terminator, Counts, Max);
+    }
+  }
+  F.setEntryCount(Function::ProfileCount(EntryCount, Function::PCT_Real));
+  markFrequency(F, EntryCount, MaxCount, Summary);
+}
+
+void BlockRecorder::add(const Function &F) {
+  std::optional<Function::ProfileCount> Entry = F.getEntryCount();
+  if (!Entry || Entry->isSynthetic())
+    return;
+  std::string Text;
+  raw_string_ostream OS(Text);
+  OS << "f ";
+  writeEscaped(OS, F.getName());
+  if (Entry->getCount() == 0) {
+    OS << " 0 0\n";
+  } else {
+    OS << ' ' << Entry->getCount() << ' ' << F.size() << '\n';
+    Function &Mutable = const_cast<Function &>(F);
+    DominatorTree Dominators(Mutable);
+    LoopInfo Loops(Dominators);
+    BranchProbabilityInfo Probabilities(F, Loops);
+    BlockFrequencyInfo Frequencies(F, Probabilities, Loops);
+    DenseMap<const BasicBlock *, uint32_t> Position;
+    uint32_t Next = 0;
+    for (const BasicBlock &Block : F)
+      Position[&Block] = Next++;
+    for (const BasicBlock &Block : F) {
+      uint64_t Count = Frequencies.getBlockProfileCount(&Block).value_or(0);
+      writeEscaped(OS, Block.getName());
+      OS << ' ' << Count;
+      // Edges to one target through several successor positions add up.
+      SmallVector<std::pair<uint32_t, uint64_t>, 4> Edges;
+      const Instruction *Terminator = Block.getTerminator();
+      for (unsigned I = 0, E = Terminator->getNumSuccessors(); I < E; ++I) {
+        uint32_t Target = Position.lookup(Terminator->getSuccessor(I));
+        uint64_t EdgeCount =
+            Probabilities.getEdgeProbability(&Block, I).scale(Count);
+        auto It = llvm::find_if(Edges, [&](const auto &Edge) {
+          return Edge.first == Target;
+        });
+        if (It == Edges.end())
+          Edges.push_back({Target, EdgeCount});
+        else
+          It->second += EdgeCount;
+      }
+      for (auto [Target, EdgeCount] : Edges)
+        OS << ' ' << Target << ':' << EdgeCount;
+      OS << '\n';
+    }
+  }
+  std::lock_guard<std::mutex> Guard(Lock);
+  Records[F.getName().str()] = std::move(Text);
+}
+
+std::optional<std::string> BlockRecorder::write(StringRef Path) {
+  SmallString<256> Temporary(Path);
+  Temporary += ".tmp-%%%%%%";
+  int Fd = -1;
+  if (std::error_code EC = sys::fs::createUniqueFile(Temporary, Fd, Temporary))
+    return EC.message();
+  {
+    raw_fd_ostream Stream(Fd, /*shouldClose=*/true);
+    Stream << Header << '\n';
+    std::lock_guard<std::mutex> Guard(Lock);
+    for (const auto &[Name, Text] : Records)
+      Stream << Text;
+    Stream.close();
+    if (Stream.has_error()) {
+      std::string Message = Stream.error().message();
+      Stream.clear_error();
+      sys::fs::remove(Temporary);
+      return Message;
+    }
+  }
+  if (std::error_code EC = sys::fs::rename(Temporary, Path)) {
+    sys::fs::remove(Temporary);
+    return EC.message();
+  }
+  return std::nullopt;
+}
+
+} // namespace scriptc

native/llvm-codegen/src/profile-blocks.h

@@ -0,0 +1,83 @@
+#pragma once
+
+#include "llvm/ADT/StringMap.h"
+#include "llvm/ADT/StringRef.h"
+
+#include <cstdint>
+#include <map>
+#include <memory>
+#include <mutex>
+#include <optional>
+#include <string>
+#include <utility>
+#include <vector>
+
+namespace llvm {
+class Function;
+class ProfileSummaryInfo;
+} // namespace llvm
+
+namespace scriptc {
+
+// A profile's block counts (a .blocks file), which annotate the functions
+// whose control flow changed since the profile was collected. The profile
+// annotation matches a function only when its control flow hashes as it did
+// then; the block counts describe each function by its blocks' labels, which
+// an unrelated change to the function leaves mostly as they were.
+//
+// The file is text. Its first line is `scriptc-profile-blocks 1`; each
+// function follows as a line `f <name> <entry count> <blocks>` and then one
+// line per block in the function's order: `<label> <count>` and, per
+// successor, `<position>:<edge count>`, where the position is the
+// successor's place in the function's block list. Names and labels escape
+// spaces, control characters, backslashes and non-ASCII bytes as `\hh`; an
+// empty label is a single backslash.
+
+// One block of a function's record.
+struct BlockRecord {
+  std::string Label;
+  uint64_t Count = 0;
+  // Successor positions with the count of the edge to each.
+  std::vector<std::pair<uint32_t, uint64_t>> Successors;
+};
+
+// One function's record: its entry count and blocks. A function that never
+// ran has an entry count of zero and no blocks.
+struct FunctionBlocks {
+  uint64_t EntryCount = 0;
+  std::vector<BlockRecord> Blocks;
+};
+
+using BlockCounts = llvm::StringMap<FunctionBlocks>;
+
+// Reads a .blocks file; the error describes a file that does not parse.
+std::unique_ptr<BlockCounts> readProfileBlocks(llvm::StringRef Path,
+                                                 std::string &Error);
+
+// Annotates F, which the profile left without counts, from its record:
+// blocks that keep their place in the function's labeled block order take
+// their recorded counts, and their edges the recorded edge counts, by target
+// or, between targets that lost their place, by position. The counts of the
+// other blocks and edges follow from flow conservation, which avoids the
+// edges the function's own branch weights make unlikely. Sets the entry
+// count, branch weights and the hot or cold attribute as the profile
+// annotation does.
+void annotateFromBlocks(llvm::Function &F, const FunctionBlocks &Record,
+                        const llvm::ProfileSummaryInfo &Summary);
+
+// The records of the functions a profile annotated, collected from every
+// partition and written as one .blocks file in name order.
+class BlockRecorder {
+public:
+  // Records F's block counts, which the profile annotation just gave it.
+  void add(const llvm::Function &F);
+  // Writes the file through a private sibling; returns an error message on
+  // failure.
+  std::optional<std::string> write(llvm::StringRef Path);
+
+private:
+  std::mutex Lock;
+  std::map<std::string, std::string> Records;
+};
+
+} // namespace scriptc

native/llvm-codegen/src/profile.cpp

@@ -2,10 +2,14 @@
 #include "diagnostics.h"
 
 #include "llvm/ADT/SmallString.h"
+#include "llvm/ADT/StringSet.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ProfileData/InstrProfReader.h"
 #include "llvm/ProfileData/InstrProfWriter.h"
 #include "llvm/Support/FileSystem.h"
+#include "llvm/Support/FormatVariadic.h"
+#include "llvm/Support/JSON.h"
+#include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/VirtualFileSystem.h"
 #include "llvm/Support/raw_ostream.h"
 
@@ -19,6 +23,7 @@ namespace scriptc {
 int mergeProfiles(int Argc, char **Argv) {
   std::string Output;
   bool Sparse = false;
+  std::string KeepFunctions;
   std::vector<std::string> Inputs;
   for (int I = 2; I < Argc; ++I) {
     StringRef Arg(Argv[I]);
@@ -26,17 +31,35 @@ int mergeProfiles(int Argc, char **Argv) {
       Output = Argv[++I];
     else if (Arg == "--sparse")
       Sparse = true;
+    else if (Arg == "--keep-functions" && I + 1 < Argc)
+      KeepFunctions = Argv[++I];
     else if (!Arg.starts_with("--"))
       Inputs.push_back(Arg.str());
     else
       return reportError("usage", "merge-profiles requires --output <file> "
                                   "and one or more input profiles, and "
-                                  "accepts --sparse");
+                                  "accepts --sparse and --keep-functions "
+                                  "<file>");
   }
   if (Output.empty() || Inputs.empty())
     return reportError("usage", "merge-profiles requires --output <file> and "
                                 "one or more input profiles");
 
+  // --keep-functions names the functions to keep, one profile name per
+  // line; the records of every other function are left out.
+  StringSet<> Kept;
+  if (!KeepFunctions.empty()) {
+    ErrorOr<std::unique_ptr<MemoryBuffer>> List =
+        MemoryBuffer::getFile(KeepFunctions);
+    if (!List)
+      return reportError("profile_read_failed",
+                         KeepFunctions + ": " + List.getError().message());
+    SmallVector<StringRef, 0> Names;
+    (*List)->getBuffer().split(Names, '\n', -1, /*KeepEmpty=*/false);
+    for (StringRef Name : Names)
+      Kept.insert(Name);
+  }
+
   // Sparse profiles leave out the functions that never ran, which then
   // optimize as without a profile instead of as cold.
   InstrProfWriter Writer(Sparse);
@@ -52,6 +75,8 @@ int mergeProfiles(int Argc, char **Argv) {
                          Path + ": " + toString(std::move(E)));
     std::string RecordError;
     for (NamedInstrProfRecord &Record : **Reader) {
+      if (!KeepFunctions.empty() && !Kept.contains(Record.Name))
+        continue;
       Writer.addRecord(std::move(Record), /*Weight=*/1, [&](Error E) {
         if (RecordError.empty())
           RecordError = toString(std::move(E));
@@ -100,4 +125,26 @@ int mergeProfiles(int Argc, char **Argv) {
   return 0;
 }
 
+int profileFunctions(int Argc, char **Argv) {
+  if (Argc != 4 || StringRef(Argv[2]) != "--input")
+    return reportError("usage",
+                       "profile-functions requires --input <file.profdata>");
+  std::string Path = Argv[3];
+  IntrusiveRefCntPtr<vfs::FileSystem> FS = vfs::getRealFileSystem();
+  Expected<std::unique_ptr<InstrProfReader>> Reader =
+      InstrProfReader::create(Path, *FS);
+  if (!Reader)
+    return reportError("profile_read_failed",
+                       Path + ": " + toString(Reader.takeError()));
+  json::Array Functions;
+  for (const NamedInstrProfRecord &Record : **Reader)
+    Functions.push_back(Record.Name.str());
+  if ((*Reader)->hasError())
+    return reportError("profile_read_failed",
+                       Path + ": " + toString((*Reader)->getError()));
+  json::Object Response{{"ok", true}, {"functions", std::move(Functions)}};
+  outs() << formatv("{0}\n", json::Value(std::move(Response)));
+  return 0;
+}
+
 } // namespace scriptc

native/llvm-codegen/src/profile.h

@@ -2,9 +2,14 @@
 
 namespace scriptc {
 
-// `merge-profiles --output <file.profdata> [--sparse] <input>...`: merges raw
-// (.profraw) or indexed (.profdata) IR profiles into one indexed profile.
-// --sparse leaves out functions that never ran.
+// `merge-profiles --output <file.profdata> [--sparse] [--keep-functions
+// <file>] <input>...`: merges raw (.profraw) or indexed (.profdata) IR
+// profiles into one indexed profile. --sparse leaves out functions that never
+// ran, and --keep-functions every function the file does not name.
 int mergeProfiles(int Argc, char **Argv);
 
+// `profile-functions --input <file.profdata>`: prints the names of the
+// functions an indexed profile has records for, as {"ok":true,"functions":[]}.
+int profileFunctions(int Argc, char **Argv);
+
 } // namespace scriptc

native/llvm-codegen/src/runtime-unit.cpp

@@ -13,6 +13,7 @@
 #include "runtime-unit.h"
 
 #include "diagnostics.h"
+#include "emit.h"
 #include "runtime-import.h"
 #include "target.h"
 
@@ -27,12 +28,16 @@
 #include "llvm/IR/Module.h"
 #include "llvm/IR/Verifier.h"
 #include "llvm/IRReader/IRReader.h"
+#include "llvm/ProfileData/InstrProf.h"
 #include "llvm/Support/FileSystem.h"
+#include "llvm/Support/FormatVariadic.h"
+#include "llvm/Support/JSON.h"
 #include "llvm/Support/SourceMgr.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Target/TargetMachine.h"
 #include "llvm/TargetParser/Triple.h"
 #include "llvm/Transforms/Utils/Cloning.h"
+#include "llvm/Transforms/Utils/SizeOpts.h"
 
 #include <memory>
 #include <optional>
@@ -53,6 +58,16 @@ struct UnitOptions {
   std::string Tag;
   bool FunctionSections = false;
   bool DataSections = false;
+  // When set, the input is unoptimized and the helper optimizes it at this
+  // level, with the profile-guided optimization below; otherwise the input
+  // is already optimized and is only promoted and emitted.
+  std::string OptLevel;
+  // Instruments the unit for IR profiling; raw profiles go to this pattern,
+  // or, when it is empty, to the file the program that links the unit names.
+  bool ProfileGenerate = false;
+  std::string ProfileGeneratePattern;
+  // Indexed profile (.profdata) that guides the unit's optimization.
+  std::string ProfileUse;
 };
 
 std::optional<UnitOptions> parse(int Argc, char **Argv) {
@@ -80,12 +95,26 @@ std::optional<UnitOptions> parse(int Argc, char **Argv) {
       Options.Target = Value.str();
     else if (Arg == "--tag")
       Options.Tag = Value.str();
+    else if (Arg == "--opt-level")
+      Options.OptLevel = Value.str();
+    else if (Arg == "--profile-generate") {
+      Options.ProfileGenerate = true;
+      Options.ProfileGeneratePattern = Value.str();
+    } else if (Arg == "--profile-use")
+      Options.ProfileUse = Value.str();
     else
       return std::nullopt;
   }
   if (Options.Input.empty() || Options.Object.empty() ||
       Options.Bitcode.empty() || Options.Target.empty() || Options.Tag.empty())
     return std::nullopt;
+  // A profile only guides an optimization the helper runs, and a unit is
+  // either instrumented or annotated.
+  if ((Options.ProfileGenerate || !Options.ProfileUse.empty()) &&
+      Options.OptLevel.empty())
+    return std::nullopt;
+  if (Options.ProfileGenerate && !Options.ProfileUse.empty())
+    return std::nullopt;
   return Options;
 }
 
@@ -98,10 +127,22 @@ bool keepLocal(const GlobalValue &GV) {
   return false;
 }
 
+// The profile's counters and per-function records (instrumented units only)
+// stay private to the unit: the profile runtime finds them by section.
+bool profileData(const GlobalValue &GV) {
+  if (const auto *Object = dyn_cast<GlobalObject>(&GV))
+    if (Object->getSection().starts_with("__llvm_prf"))
+      return true;
+  return GV.getName().starts_with("__llvm_prf") ||
+         GV.getName().starts_with("__profc_") ||
+         GV.getName().starts_with("__profd_") ||
+         GV.getName().starts_with("__profvp_");
+}
+
 void promoteLocals(Module &M, StringRef Tag) {
   unsigned Anonymous = 0;
   auto Promote = [&](GlobalValue &GV) {
-    if (!GV.hasLocalLinkage() || keepLocal(GV))
+    if (!GV.hasLocalLinkage() || keepLocal(GV) || profileData(GV))
       return;
     std::string Base =
         GV.hasName() ? GV.getName().str() : ("anon." + Twine(Anonymous++)).str();
@@ -157,7 +198,9 @@ int runtimeUnit(int Argc, char **Argv) {
     return fail("usage",
                 "runtime-unit requires --input <bc> --object <o> --bitcode <bc> "
                 "--target <triple> --tag <unit> and accepts "
-                "--function-sections --data-sections");
+                "--function-sections --data-sections, and --opt-level <level> "
+                "with either --profile-generate <pattern> or --profile-use "
+                "<profdata>");
   LLVMContext Context;
   SMDiagnostic Diagnostic;
   std::unique_ptr<Module> M = parseIRFile(Options->Input, Diagnostic, Context);
@@ -208,6 +251,53 @@ int runtimeUnit(int Argc, char **Argv) {
     if (F.getFnAttribute("probe-stack").getValueAsString() == "__chkstk_darwin")
       F.addFnAttr("probe-stack", "inline-asm");
 
+  // Profiles name a unit's internal functions after its source file. The
+  // unit's own name keeps those names the same in every checkout, so a
+  // committed profile matches wherever the pack is built.
+  if (!Options->OptLevel.empty()) {
+    M->setSourceFileName(Options->Tag + ".c");
+    // The runtime profile comes from a fixed set of training programs, and a
+    // runtime function they rarely call can be hot in another program (a
+    // long UTF-8 decode, a busy worker channel). Code the profile finds cold
+    // is therefore optimized for speed as without a profile, not for size:
+    // the profile still guides inlining, block layout and branch weights.
+    EnablePGSO.setValue(false);
+    // The profile names of the functions an instrumented unit defines, so
+    // a merged profile can keep the runtime's records and drop the
+    // training programs'.
+    json::Array Functions;
+    if (Options->ProfileGenerate)
+      for (const Function &F : *M)
+        if (!F.isDeclaration() && !F.hasAvailableExternallyLinkage())
+          Functions.push_back(getIRPGOFuncName(F));
+    ModuleProfileMatch Match;
+    if (std::optional<std::string> Failure = optimizeModule(
+            *M, *Machine, Options->OptLevel, Options->ProfileGenerate,
+            Options->ProfileGeneratePattern, Options->ProfileUse, Match))
+      return fail("optimization_failed", *Failure);
+    if (!Options->ProfileUse.empty()) {
+      // Each matched function's counters in the profile, by profile name.
+      auto Counts =
+          [](const std::vector<std::pair<std::string, uint64_t>> &List) {
+            json::Object Object;
+            for (const auto &[Name, Sum] : List)
+              Object[Name] = Sum;
+            return Object;
+          };
+      json::Object Profile{
+          {"annotated", Match.Annotated},
+          {"mismatched", Match.Mismatched},
+          {"missing", Match.Missing},
+          {"annotated_counts", Counts(Match.AnnotatedCounts)},
+          {"mismatched_counts", Counts(Match.MismatchedCounts)}};
+      json::Object Response{{"ok", true}, {"profile", std::move(Profile)}};
+      outs() << formatv("{0}\n", json::Value(std::move(Response)));
+    } else if (Options->ProfileGenerate) {
+      json::Object Response{{"ok", true}, {"functions", std::move(Functions)}};
+      outs() << formatv("{0}\n", json::Value(std::move(Response)));
+    }
+  }
+
   promoteLocals(*M, Options->Tag);
   std::string VerifyError;
   raw_string_ostream VerifyStream(VerifyError);

packages/cli/test/profile.test.ts

@@ -1,6 +1,6 @@
 import { execFile } from "node:child_process";
 import { createRequire } from "node:module";
-import { mkdtemp, readdir, rm, stat, writeFile } from "node:fs/promises";
+import { copyFile, mkdtemp, readFile, readdir, rm, stat, writeFile } from "node:fs/promises";
 import { tmpdir } from "node:os";
 import { dirname, join } from "node:path";
 import { promisify } from "node:util";
@@ -85,6 +85,15 @@ test.skipIf(process.platform !== "linux")(
       await scriptc(["build", entry, "--profile-use", sparse, "-o", optimized], env);
       expect((await execFileAsync(optimized, ["1000"])).stdout).toBe("total 167499\n");
 
+      // SCRIPTC_RECORD_PROFILE_BLOCKS writes the block counts of the
+      // functions the profile annotates.
+      const recorded = join(dir, "recorded.blocks");
+      await scriptc(["build", entry, "--profile-use", merged, "-o", optimized], {
+        ...env,
+        SCRIPTC_RECORD_PROFILE_BLOCKS: recorded,
+      });
+      expect((await readFile(recorded, "utf8")).split("\n")[0]).toBe("scriptc-profile-blocks 1");
+
       // A profile from an older version of the program still builds it. The
       // edited function no longer matches its counts, and the build warns.
       await writeFile(entry, PROGRAM + 'console.log("edited");\n');
@@ -94,6 +103,18 @@ test.skipIf(process.platform !== "linux")(
       expect(stale.stderr).toMatch(
         /\nscriptc: warning: [1-9]\d* of the \d+ functions .* describes \([\d.]+%\) changed since it was collected and are optimized without their counts; collect the profile again\n$/,
       );
+      // The block counts beside the profile annotate the edited function.
+      await copyFile(recorded, join(dir, "app.blocks"));
+      const recovered = await scriptc(
+        ["build", entry, "--profile-use", merged, "-o", optimized],
+        env,
+      );
+      expect((await execFileAsync(optimized, ["1000"])).stdout).toBe("total 167499\nedited\n");
+      expect(recovered.stderr).toMatch(/ [1-9]\d* mismatched, /);
+      expect(recovered.stderr).toMatch(
+        / [1-9]\d* functions? without profile counts annotated from \S+app\.blocks\n/,
+      );
+      expect(recovered.stderr).not.toContain("warning");
     } finally {
       await rm(dir, { recursive: true, force: true });
     }

packages/compiler/ambient/node-fallback/fs.d.ts

@@ -42,6 +42,11 @@ declare module "node:fs" {
   /* The bare-encoding spelling — the options record's encoding key alone. */
   export function writeFileSync(path: string | URL, data: string, encoding: "utf8" | "utf-8"): void;
   export function writeFileSync(path: string | URL, data: string | Uint8Array): void;
+  /* The file-descriptor form: Node writes the whole data to an already
+   * open descriptor and leaves it open (closeSync owns its lifetime, and
+   * the write lands at the descriptor's current offset). */
+  export function writeFileSync(fd: number, data: string): void;
+  export function writeFileSync(fd: number, data: string, encoding: "utf8" | "utf-8"): void;
   export function appendFileSync(path: string | URL, data: string | Uint8Array): void;
   export function existsSync(path: string | URL): boolean;
   export function mkdirSync(path: string | URL): void;

packages/compiler/native/host.c

@@ -73,12 +73,14 @@ void scriptc_native_collect_cycles(void) {
 int32_t scriptc_native_fork(void) { return -1; }
 void scriptc_native_exit(int32_t code) { ExitProcess((UINT)code); }
 int32_t scriptc_native_wait(int32_t pid) { (void)pid; return -1; }
+int32_t scriptc_native_exited(void) { return -1; }
 uint8_t scriptc_native_lock_open(const uint8_t *bytes, size_t size) { (void)bytes; (void)size; return 0; }
 void scriptc_native_lock(void) {}
 void scriptc_native_unlock(void) {}
 #else
 #include <errno.h>
 #include <fcntl.h>
+#include <signal.h>
 #include <sys/file.h>
 #include <sys/wait.h>
 
@@ -119,6 +121,18 @@ int32_t scriptc_native_wait(int32_t pid) {
   return -1;
 }
 
+/* The pid of a child that has ended, waiting for one if none has, or -1
+ * when waitid fails. The child is left to scriptc_native_wait: a child
+ * another part of the process waits for is not reaped here. */
+int32_t scriptc_native_exited(void) {
+  siginfo_t info;
+  memset(&info, 0, sizeof info);
+  while (waitid(P_ALL, 0, &info, WEXITED | WNOWAIT) < 0) {
+    if (errno != EINTR) return -1;
+  }
+  return info.si_pid > 0 ? (int32_t)info.si_pid : -1;
+}
+
 static int scriptc_native_lock_fd = -1;
 
 uint8_t scriptc_native_lock_open(const uint8_t *bytes, size_t size) {

packages/compiler/native/host.ffi.json

@@ -6,6 +6,7 @@
     { "name": "compilerNativeFork", "symbol": "scriptc_native_fork", "params": [], "returns": "i32" },
     { "name": "compilerNativeExit", "symbol": "scriptc_native_exit", "params": ["i32"], "returns": "void" },
     { "name": "compilerNativeWait", "symbol": "scriptc_native_wait", "params": ["i32"], "returns": "i32" },
+    { "name": "compilerNativeExited", "symbol": "scriptc_native_exited", "params": [], "returns": "i32" },
     { "name": "compilerNativeLockOpen", "symbol": "scriptc_native_lock_open", "params": ["string"], "returns": "bool" },
     { "name": "compilerNativeLock", "symbol": "scriptc_native_lock", "params": [], "returns": "void" },
     { "name": "compilerNativeUnlock", "symbol": "scriptc_native_unlock", "params": [], "returns": "void" },

packages/compiler/src/backend/llvm/arenas.test.ts

@@ -67,6 +67,38 @@ console.log(total);
     expect(body).not.toContain("ScrArena.closed");
     expect(body).not.toContain("@scr_arena_disposal");
   }
+  // A record literal stores every field, and no dispose entry reads them:
+  // the record's constructor does not clear the object. A class constructor
+  // may leave fields unset, so the class's does.
+  const record = constructors.find((body) => body.includes("@sc_rnew_"))!;
+  const cls = constructors.find((body) => !body.includes("@sc_rnew_"))!;
+  expect(record).not.toContain("@llvm.memset");
+  expect(cls).toContain("@llvm.memset");
+});
+
+test("record literals in an arena that is not closed keep the clear their dispose entries need", async () => {
+  // The items hold outside strings, so their groups are not closed.
+  const llvm = await llvmFor(`
+interface Item {
+  label: string;
+  next: Item | null;
+}
+function chain(labels: string[], n: number): Item {
+  return n === 0 ? { label: labels[0]!, next: null } : { label: labels[n % labels.length]!, next: chain(labels, n - 1) };
+}
+function length(item: Item): number {
+  return item.next === null ? item.label.length : item.label.length + length(item.next);
+}
+const labels = ["a", "bb", "ccc"];
+let total = 0;
+for (let round = 0; round < 4; round++) total += length(chain(labels, 10 + round));
+console.log(total);
+`);
+  expect(llvm).toMatch(/@scr_arena_open\(ptr %[\w.]+, i32 0\)/);
+  const items = arenaConstructors(llvm).filter((body) => body.includes("@sc_rnew_"));
+  expect(items.length).toBe(1);
+  expect(items[0]).toContain(".dispose");
+  expect(items[0]).toContain("@llvm.memset");
 });
 
 test("an arena that is not closed keeps the dispose list in every arena-aware constructor", async () => {

packages/compiler/src/backend/llvm/arenas.ts

@@ -275,6 +275,12 @@ export function arenaCurrentAddress(host: ArenaHost, dest: string): string[] | n
  * the arena is full. The bump, the clear, the header and the dispose entry
  * are inline (scr_arena.c's ScrArena ABI); scr_arena_obj takes a new chunk
  * and scr_arena_disposal grows the list.
+ *
+ * `clear` is what the bump path zeroes: the whole object, or only the
+ * collector header, for callers that store every word of the object before
+ * anything reads it (a record literal of a shape with no dispose entries:
+ * `init` stores the count and the shape number, and the literal every
+ * field). scr_arena_obj and the heap constructor return zeroed objects.
  */
 export function arenaNewLines(
   host: ArenaHost,
@@ -284,6 +290,7 @@ export function arenaNewLines(
   dispose: string | null,
   init: string[],
   result: string,
+  clear: "object" | "header" = "object",
 ): string[] {
   const S = host.sizeType;
   // ScrArena's fields and dispose entries are pointer-sized (scr_arena.c).
@@ -328,8 +335,13 @@ export function arenaNewLines(
     `carve:`,
     `  %end = getelementptr inbounds i8, ptr %bp, ${S} %n`,
     `  store ptr %end, ptr %a`,
-    `  call void @llvm.memset.p0.${S}(ptr align 8 %bp, i8 0, ${S} %n, i1 false)`,
   );
+  if (clear === "object")
+    lines.push(`  call void @llvm.memset.p0.${S}(ptr align 8 %bp, i8 0, ${S} %n, i1 false)`);
+  else if (cyc !== null)
+    lines.push(
+      `  call void @llvm.memset.p0.${S}(ptr align 8 %bp, i8 0, ${S} ${CYC_HDR}, i1 false)`,
+    );
   if (cyc !== null)
     lines.push(
       `  %id = trunc i32 %tid to i16`,

packages/compiler/src/backend/llvm/blocks.ts

@@ -97,12 +97,22 @@ export class BlockBuilder {
       if (s === UNWIND_RESUME) {
         const exception = this.tmp();
         this.line(`${exception} = load { ptr, i32 }, ptr ${UNWIND_SLOT}`);
-        s = `resume { ptr, i32 } ${exception}`;
+        this.resumeUnwind(exception);
+        return;
       }
       this.cur.term = `  ${this.withDebugLocation(s)}`;
     }
   }
 
+  /** Ends the block by continuing an unwind with `exception`, the value
+   * of a landing pad (UNWIND_RESUME_DECLARATION). */
+  resumeUnwind(exception: string): void {
+    const token = this.tmp();
+    this.line(`${token} = extractvalue { ptr, i32 } ${exception}, 0`);
+    this.line(`call void @scr_unwind_resume(ptr ${token})`);
+    this.terminate("unreachable");
+  }
+
   br(label: string, attachment = ""): void {
     this.terminate(`br label %${label}${attachment}`);
   }

packages/compiler/src/backend/llvm/call-lifetimes.test.ts

@@ -594,6 +594,35 @@ test("closure bodies get local facts but never parameter facts", () => {
   expect(analyze(fn).locals.get("closure")).toEqual(new Set(["item"]));
 });
 
+test("union locals copied to other locals are reported with their copies", () => {
+  const fn = helper("copies");
+  fn.locals.push(local("source"), { ...local("first"), mutable: true }, local("second"));
+  fn.body = [
+    { kind: "varDecl", localId: "source", init: ref("value"), loc },
+    { kind: "varDecl", localId: "first", init: ref("source"), loc },
+    { kind: "assign", localId: "first", value: ref("source"), loc },
+    { kind: "varDecl", localId: "second", init: ref("first"), loc },
+    { kind: "exprStmt", expr: tag("second"), loc },
+  ];
+  const result = analyze(fn);
+  expect(result.projectedLocals.get("copies")).toEqual(new Set(["second"]));
+  expect(result.copiedLocals.get("copies")).toEqual(
+    new Map([
+      ["source", ["first", "first"]],
+      ["first", ["second"]],
+    ]),
+  );
+  // A copied parameter stays rejected: callers may lend it a stack box.
+  expect(result.parameters.has("copies")).toBe(false);
+  // A copy inside an expression is an ordinary whole-value use.
+  fn.body.push({
+    kind: "exprStmt",
+    expr: { kind: "assignExpr", localId: "first", value: ref("source"), type: optional, loc },
+    loc,
+  });
+  expect(analyze(fn).copiedLocals.get("copies")?.has("source") ?? false).toBe(false);
+});
+
 test("stores into nullable-pointer class fields project the stored union", () => {
   const self: IrType = { kind: "object", className: "Node" };
   const store = (className: string, field: string, value: IrExpr): IrStmt => ({

packages/compiler/src/backend/llvm/call-lifetimes.ts

@@ -34,6 +34,10 @@ interface Uses {
   /** Narrowed payloads of union bindings passed directly as call arguments:
    * the payload stays in place if the callee's parameter is proved. */
   payloadForwards: Map<string, ForwardedUse[]>;
+  /** Union bindings whose whole value initializes or is assigned to
+   * another union local (`const x = y`, `x = y` as a statement), with the
+   * locals it is copied to. */
+  copies: Map<string, string[]>;
 }
 
 interface Parameter {
@@ -59,6 +63,11 @@ export interface CallLifetimes {
   /** Projection-only local uses, allowing statement assignments. Each
    * initializer and assignment still needs a separate storage proof. */
   projectedLocals: Map<string, Set<string>>;
+  /** Locals that projectedLocals leaves out only because their whole value
+   * is copied to other union locals, with those locals. Such a local is
+   * only projected when every local it is copied to snapshots the copy
+   * (local-union-storage.ts) instead of keeping the box. */
+  copiedLocals: Map<string, Map<string, readonly string[]>>;
   /** Generator bodies own their parameters for the whole body, including
    * every suspension: the frame releases them only on its final exit. A
    * reference parameter nothing rebinds, declares or captures is therefore
@@ -132,6 +141,7 @@ function collectUses(
     forwards: new Map(),
     payloadKept: new Set(),
     payloadForwards: new Map(),
+    copies: new Map(),
   };
   const forward = (
     table: Map<string, ForwardedUse[]>,
@@ -292,6 +302,18 @@ function collectUses(
     }
     return everyExprChild(node, expr, (node) => stmt(node, true));
   }
+  /** `const x = y` or a statement `x = y` of union locals: the copy is
+   * recorded instead of rejecting `y`. A parameter's copy still rejects it,
+   * since callers may lend projection-only parameters a stack box. */
+  const params = new Set(fn.params.map((param) => param.localId));
+  const unionCopy = (target: string, value: IrExpr | null | undefined): boolean => {
+    if (value?.kind !== "varRef" || value.type.kind !== "union" || params.has(value.localId))
+      return false;
+    let targets = uses.copies.get(value.localId);
+    if (!targets) uses.copies.set(value.localId, (targets = []));
+    targets.push(target);
+    return true;
+  };
   function stmt(node: IrStmt, inExpression: boolean): boolean {
     switch (node.kind) {
       case "return":
@@ -300,6 +322,7 @@ function collectUses(
         break;
       case "varDecl":
         uses.declarations.set(node.localId, (uses.declarations.get(node.localId) ?? 0) + 1);
+        if (unionCopy(node.localId, node.init)) return true;
         // A receiver temporary (`%compoundReceiver = this`) projects its
         // source as long as the temporary itself is only projected.
         if (
@@ -342,6 +365,7 @@ function collectUses(
         // Sequence expressions can rebind a previous call argument before
         // its consumer runs. Those locals need an owned heap snapshot.
         if (inExpression) uses.invalid.add(node.localId);
+        else if (unionCopy(node.localId, node.value)) return true;
         break;
       case "forOf":
         // The binding is initialized afresh for each iteration. Its uses
@@ -429,6 +453,7 @@ export function analyzeCallLifetimes(
     borrowed: new Map(),
     bindings: new Map(),
     projectedLocals: new Map(),
+    copiedLocals: new Map(),
     generatorParameters: new Map(),
   };
   for (const fn of functions.values()) {
@@ -602,6 +627,7 @@ export function analyzeCallLifetimes(
     const params = new Set(fn.params.map((param) => param.localId));
     const safe = new Set<string>();
     const projected = new Set<string>();
+    const copied = new Map<string, readonly string[]>();
     for (const local of fn.locals) {
       if (
         params.has(local.id) ||
@@ -611,13 +637,18 @@ export function analyzeCallLifetimes(
       )
         continue;
       const forwards = uses.forwards.get(local.id) ?? [];
-      if (forwards.every((use) => target(use)?.safe === true)) {
-        projected.add(local.id);
-        if (eligible(local) && !uses.written.has(local.id)) safe.add(local.id);
+      if (!forwards.every((use) => target(use)?.safe === true)) continue;
+      const copies = uses.copies.get(local.id);
+      if (copies) {
+        copied.set(local.id, copies);
+        continue;
       }
+      projected.add(local.id);
+      if (eligible(local) && !uses.written.has(local.id)) safe.add(local.id);
     }
     result.locals.set(fn.name, safe);
     result.projectedLocals.set(fn.name, projected);
+    if (copied.size > 0) result.copiedLocals.set(fn.name, copied);
   }
   // Consumed parameters take the owned convention, unless only projected.
   for (const { fn, index } of consumed) {

packages/compiler/src/backend/llvm/class-unions.test.ts

@@ -0,0 +1,142 @@
+import { execFileSync } from "node:child_process";
+import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { expect, test } from "vitest";
+import { F64, VOID, type IrModule, type IrType } from "../../ir/ir.js";
+import { mangleClassNew, mangleClassRelease, mangleClassRetain } from "../mangle.js";
+import { emitLlvmModule } from "./emitter.js";
+import { classReleaseFn, classRetainFn, classTagFn } from "./nullable-unions.js";
+
+const loc = { file: "class-unions.ts", start: 0, end: 1 };
+const tree: IrType = { kind: "union", unionId: "tree" };
+
+/** `Leaf | Pair`, two standalone classes, and a hierarchy beside them. */
+function treeModule(): IrModule {
+  return {
+    irVersion: 17,
+    sourceFile: loc.file,
+    entry: "main",
+    functions: [{ name: "main", params: [], returnType: VOID, locals: [], body: [], loc }],
+    classes: [
+      { name: "Leaf", fields: [{ name: "value", type: F64 }], methods: [], loc },
+      {
+        name: "Pair",
+        fields: [
+          { name: "left", type: tree },
+          { name: "right", type: tree },
+        ],
+        methods: [],
+        loc,
+      },
+      { name: "Other", fields: [{ name: "value", type: F64 }], methods: [], loc },
+    ],
+    unions: [
+      {
+        id: "tree",
+        arms: [
+          { kind: "object", className: "Leaf" },
+          { kind: "object", className: "Pair" },
+        ],
+      },
+    ],
+  };
+}
+
+function definition(llvm: string, symbol: string): string {
+  const escaped = symbol.replace(/[.$]/g, "\\$&");
+  const body = new RegExp(`^define internal [^@]*@${escaped}\\([^]*?^}`, "m").exec(llvm)?.[0];
+  expect(body, symbol).toBeDefined();
+  return body!;
+}
+
+test("standalone arms keep their class id in the rc word's high half", () => {
+  const llvm = emitLlvmModule(treeModule(), { pointerBits: 64 });
+  // Arms store the id beside a count of 1; other standalone classes keep a
+  // plain count.
+  expect(definition(llvm, mangleClassNew("Leaf"))).toMatch(/store i64 \d{10,}, ptr %o\.vt/);
+  expect(definition(llvm, mangleClassNew("Other"))).toMatch(/store i64 1, ptr %o/);
+  expect(definition(llvm, mangleClassRetain("Leaf"))).toContain("load i32, ptr %o");
+  expect(definition(llvm, mangleClassRetain("Other"))).toContain("load i64, ptr %o");
+  // The tag reads the id at byte 4 and switches on it.
+  const tag = definition(llvm, classTagFn("tree").slice(1));
+  expect(tag).toContain("getelementptr inbounds i8, ptr %o, i64 4");
+  expect(tag).toMatch(/switch i32 %id/);
+  // 32-bit targets keep the box and the plain count.
+  const llvm32 = emitLlvmModule(treeModule(), { pointerBits: 32 });
+  expect(llvm32).not.toContain(classTagFn("tree"));
+  expect(definition(llvm32, mangleClassRetain("Leaf"))).toContain("load i32, ptr %o");
+  expect(definition(llvm32, mangleClassNew("Leaf"))).toMatch(/store i32 1, ptr %o/);
+});
+
+test("a tagged class's 32-bit count saturates at the immortal count instead of wrapping", () => {
+  const llvm = emitLlvmModule(treeModule(), { pointerBits: 64 });
+  // The class's and the union's retain and release, run on an rc word whose
+  // count is two below the immortal all-ones count.
+  const symbols = [
+    [mangleClassRetain("Leaf"), "retain_leaf"],
+    [mangleClassRelease("Leaf"), "release_leaf"],
+    [classRetainFn("tree").slice(1), "retain_tree"],
+    [classReleaseFn("tree").slice(1), "release_tree"],
+  ] as const;
+  let ir = "";
+  for (const [symbol] of symbols) ir += `${definition(llvm, symbol)}\n\n`;
+  // A last reference never reaches these: the harness aborts if one does.
+  const last = [
+    [`${mangleClassRelease("Leaf")}_slow`, "last_leaf"],
+    [`${classReleaseFn("tree").slice(1)}.last`, "last_tree"],
+  ] as const;
+  for (const [symbol, name] of [...symbols, ...last])
+    ir = ir.replaceAll(`@${symbol}(`, `@${name}(`);
+  ir = ir.replaceAll("define internal", "define").replaceAll(" #0 ", " ");
+  ir += `declare void @last_leaf(ptr)\ndeclare void @last_tree(ptr)\n`;
+  const dir = mkdtempSync(join(tmpdir(), "scriptc-class-unions-"));
+  try {
+    writeFileSync(join(dir, "rc.ll"), ir);
+    writeFileSync(
+      join(dir, "main.c"),
+      `#include <stdint.h>
+#include <stdlib.h>
+void *retain_leaf(void *);
+void release_leaf(void *);
+void *retain_tree(void *);
+void release_tree(void *);
+void last_leaf(void *o) { (void)o; abort(); }
+void last_tree(void *o) { (void)o; abort(); }
+static int check(void *(*retain)(void *), void (*release)(void *)) {
+  const uint64_t id = (uint64_t)42 << 32;
+  uint64_t word = id | 0xfffffffdu;
+  retain(&word);
+  if (word != (id | 0xfffffffeu)) return 1;
+  retain(&word);
+  if (word != (id | 0xffffffffu)) return 2;
+  retain(&word);
+  if (word != (id | 0xffffffffu)) return 3;
+  release(&word);
+  if (word != (id | 0xffffffffu)) return 4;
+  word = id | 3;
+  release(&word);
+  if (word != (id | 2)) return 5;
+  return 0;
+}
+int main(void) {
+  int leaf = check(retain_leaf, release_leaf);
+  if (leaf != 0) return leaf;
+  int tree = check(retain_tree, release_tree);
+  return tree == 0 ? 0 : 10 + tree;
+}
+`,
+    );
+    execFileSync("clang", [
+      "-Wno-override-module",
+      "-O1",
+      join(dir, "rc.ll"),
+      join(dir, "main.c"),
+      "-o",
+      join(dir, "rc"),
+    ]);
+    execFileSync(join(dir, "rc"));
+  } finally {
+    rmSync(dir, { recursive: true, force: true });
+  }
+});

packages/compiler/src/backend/llvm/class-unions.ts

@@ -0,0 +1,255 @@
+/* Class-union helpers (nullable-unions.ts ClassUnion). A class union's value
+ * is the instance pointer; its arm is the arm whose preorder interval holds
+ * the instance's class id, which every arm's hierarchy keeps in its header
+ * (classes.ts LlClassMeta.idInRc). Each union gets a tag helper, and the
+ * unions over the same classes share one retain, release and trace. */
+import { InternalCompilerError } from "../../errors.js";
+import type { IrModule } from "../../ir/ir.js";
+import { mangleClassRelease, mangleClassTrace } from "../mangle.js";
+import {
+  classIdLoadLines,
+  programLifetimeClass,
+  type ClassHost,
+  type LlClassMeta,
+} from "./classes.js";
+import {
+  NULLABLE_NULL,
+  classReleaseFn,
+  classRetainFn,
+  classTagFn,
+  classTraceFn,
+  type ClassUnion,
+} from "./nullable-unions.js";
+import { FN_ATTRS, cycleRootLines, retainBody } from "./shapes.js";
+
+/** The widest span of class ids a union's arms may cover for its helpers
+ * to switch on the id; wider unions test each arm's interval. */
+const CLASS_TAG_SWITCH_SPAN = 32;
+
+/** One arm's class ids: its class's preorder interval plus 1. */
+interface ArmIds {
+  tag: number;
+  className: string;
+  lo: number;
+  hi: number;
+}
+
+function armIds(
+  union: ClassUnion,
+  classMeta: ReadonlyMap<string, LlClassMeta>,
+): { arms: ArmIds[]; first: LlClassMeta } {
+  const arms = union.arms.map((arm) => {
+    const meta = classMeta.get(arm.className);
+    if (meta === undefined || !meta.idInRc)
+      throw new InternalCompilerError(
+        `llvm emitter bug: class-union arm ${arm.className} keeps no class id`,
+      );
+    return { tag: arm.tag, className: arm.className, lo: meta.pre + 1, hi: meta.post + 1 };
+  });
+  return { arms, first: classMeta.get(union.arms[0]!.className)! };
+}
+
+/** Lines that branch from the open block on the class id of the instance
+ * `%o`: to `%a<tag>` for the arm holding it and to `missing` for none. */
+function armBranchLines(arms: readonly ArmIds[], first: LlClassMeta, missing: string): string[] {
+  const lines = classIdLoadLines(first, "%o", "%id", "");
+  const span = Math.max(...arms.map((arm) => arm.hi)) - Math.min(...arms.map((arm) => arm.lo)) + 1;
+  if (span <= CLASS_TAG_SWITCH_SPAN) {
+    const cases: string[] = [];
+    for (const arm of arms)
+      for (let id = arm.lo; id <= arm.hi; id++) cases.push(`i32 ${id}, label %a${arm.tag}`);
+    lines.push(`  switch i32 %id, label %${missing} [ ${cases.join(" ")} ]`);
+    return lines;
+  }
+  arms.forEach((arm, i) => {
+    const next = i + 1 < arms.length ? `n${arm.tag}` : missing;
+    lines.push(
+      `  %r${arm.tag} = sub i32 %id, ${arm.lo}`,
+      `  %in${arm.tag} = icmp ult i32 %r${arm.tag}, ${arm.hi - arm.lo + 1}`,
+      `  br i1 %in${arm.tag}, label %a${arm.tag}, label %${next}`,
+    );
+    if (next !== missing) lines.push(`${next}:`);
+  });
+  return lines;
+}
+
+/** Lines that leave for `unit` on NULL and for `nullArm` on the null
+ * sentinel, and continue in a new block for an instance. */
+function unitLines(union: ClassUnion, unit: string, nullArm: string): string[] {
+  const lines = [
+    `  %isnull = icmp eq ptr %o, null`,
+    `  br i1 %isnull, label %${unit}, label %ref`,
+    `ref:`,
+  ];
+  if (union.nullTag >= 0)
+    lines.push(
+      `  %isnullarm = icmp eq ptr %o, ${NULLABLE_NULL}`,
+      `  br i1 %isnullarm, label %${nullArm}, label %obj`,
+      `obj:`,
+    );
+  return lines;
+}
+
+/** A dispatch on the arm of `%o` that runs `call(className)` and returns. */
+function armDispatchLines(
+  arms: readonly ArmIds[],
+  first: LlClassMeta,
+  call: (className: string) => string,
+): string[] {
+  return [
+    ...armBranchLines(arms, first, "done"),
+    ...arms.flatMap((arm) => [`a${arm.tag}:`, call(arm.className), `  br label %done`]),
+    `done:`,
+  ];
+}
+
+/** A class union's value is whichever instance it holds, and the collector
+ * visits it as that instance: when one arm carries a collector header,
+ * every arm must. Every class of a hierarchy shares its header decision, so
+ * a header reaches each arm's whole hierarchy, and unions that share a
+ * hierarchy pass headers on to each other: this repeats until no union
+ * adds one. ProgramLifetime classes carry none. */
+export function classUnionHeaders(
+  mod: IrModule,
+  unions: readonly ClassUnion[],
+  tracedShapes: Set<string>,
+  tracedUnions: ReadonlySet<string>,
+): void {
+  if (unions.length === 0) return;
+  const bases = new Map<string, string | undefined>();
+  for (const cls of mod.classes ?? []) bases.set(cls.name, cls.base);
+  const rootOf = (name: string): string => {
+    let root = name;
+    for (let base = bases.get(root); base !== undefined; base = bases.get(root)) root = base;
+    return root;
+  };
+  const members = new Map<string, string[]>();
+  for (const cls of mod.classes ?? []) {
+    const root = rootOf(cls.name);
+    const list = members.get(root);
+    if (list) list.push(cls.name);
+    else members.set(root, [cls.name]);
+  }
+  const lifetime = new Set(mod.contracts?.programLifetime ?? []);
+  for (let changed = true; changed;) {
+    changed = false;
+    for (const union of unions) {
+      if (lifetime.has(rootOf(union.arms[0]!.className))) continue;
+      const names = union.arms.flatMap((arm) => members.get(rootOf(arm.className)) ?? []);
+      if (
+        !tracedUnions.has(union.unionId) &&
+        !names.some((name) => tracedShapes.has(`object:${name}`))
+      )
+        continue;
+      for (const name of names) {
+        const key = `object:${name}`;
+        if (tracedShapes.has(key)) continue;
+        tracedShapes.add(key);
+        changed = true;
+      }
+    }
+  }
+}
+
+/** The tag helper of every class union, and the retain, release and trace
+ * of each one that owns them (NullableUnions.helperOwner).
+ * - Retain and release of unions over ProgramLifetime classes do nothing:
+ *   those instances count nothing.
+ * - Otherwise every arm counts in the low half of its rc word (idInRc), so
+ *   retain is the shared increment and release the shared decrement; only
+ *   a release of the last reference dispatches on the arm, to the arm's
+ *   own release (which tears the dynamic class down).
+ * - Trace dispatches on the arm. Every arm carries a collector header when
+ *   one does (the emitter's traced fixpoint). */
+export function emitClassUnionHelpers(
+  host: ClassHost,
+  classMeta: ReadonlyMap<string, LlClassMeta>,
+): string[] {
+  const out: string[] = [];
+  for (const union of host.nullableUnions.allClassUnions()) {
+    const { arms, first } = armIds(union, classMeta);
+    out.push(
+      `define internal i32 ${classTagFn(union.unionId)}(ptr %o) alwaysinline ${FN_ATTRS} { ; class union tag`,
+      `entry:`,
+      ...unitLines(union, "unit", "nullarm"),
+      ...armBranchLines(arms, first, "bad"),
+      ...arms.flatMap((arm) => [`a${arm.tag}:`, `  ret i32 ${arm.tag}`]),
+      `unit:`,
+      `  ret i32 ${union.unitTag}`,
+      ...(union.nullTag >= 0 ? [`nullarm:`, `  ret i32 ${union.nullTag}`] : []),
+      `bad:`,
+      `  ret i32 -1`,
+      `}`,
+      ``,
+    );
+    if (host.nullableUnions.helperOwner(union.unionId) !== union.unionId) continue;
+    const retain = classRetainFn(union.unionId).slice(1);
+    const release = classReleaseFn(union.unionId);
+    if (programLifetimeClass(host, first)) {
+      out.push(
+        `define internal ptr @${retain}(ptr %o) ${FN_ATTRS} { ; class union retain (immortal)`,
+        `entry:`,
+        `  ret ptr %o`,
+        `}`,
+        ``,
+        `define internal void ${release}(ptr %o) ${FN_ATTRS} { ; class union release (immortal)`,
+        `entry:`,
+        `  ret void`,
+        `}`,
+        ``,
+      );
+      continue;
+    }
+    const traced = host.tracedShapes.has(`object:${first.def.name}`);
+    out.push(
+      ...retainBody(host, retain, "class union retain", false, "i32"),
+      ``,
+      `define internal void ${release}(ptr %o) ${FN_ATTRS} { ; class union release (fast path)`,
+      `entry:`,
+      ...unitLines(union, "done", "done"),
+      `  %rc = load i32, ptr %o`,
+      `  %rcm2 = sub i32 %rc, 2`,
+      `  %shared = icmp ult i32 %rcm2, -3`,
+      `  br i1 %shared, label %dec, label %slow`,
+      `dec:`,
+      `  %n = sub i32 %rc, 1`,
+      `  store i32 %n, ptr %o`,
+      ...(traced ? cycleRootLines(host, "done") : [`  br label %done`]),
+      `slow:`,
+      `  %imm = icmp eq i32 %rc, -1`,
+      `  br i1 %imm, label %done, label %last`,
+      `last:`,
+      `  call void ${release}.last(ptr %o)`,
+      `  br label %done`,
+      `done:`,
+      `  ret void`,
+      `}`,
+      ``,
+      `define internal void ${release}.last(ptr %o) noinline ${FN_ATTRS} { ; class union release (last reference)`,
+      `entry:`,
+      ...armDispatchLines(
+        arms,
+        first,
+        (name) => `  call void @${mangleClassRelease(name)}(ptr %o)`,
+      ),
+      `  ret void`,
+      `}`,
+      ``,
+    );
+    if (traced)
+      out.push(
+        `define internal void ${classTraceFn(union.unionId)}(ptr %o, ptr %visit, ptr %ctx) ${FN_ATTRS} { ; class union trace`,
+        `entry:`,
+        ...unitLines(union, "done", "done"),
+        ...armDispatchLines(
+          arms,
+          first,
+          (name) => `  call void @${mangleClassTrace(name)}(ptr %o, ptr %visit, ptr %ctx)`,
+        ),
+        `  ret void`,
+        `}`,
+        ``,
+      );
+  }
+  return out;
+}

packages/compiler/src/backend/llvm/classes.ts

@@ -43,6 +43,7 @@ import { llvmCommentText } from "./common.js";
 import { INLINE_UNION_FIELD } from "./inline-unions.js";
 import { NullableRefFields } from "./nullable-fields.js";
 import type { Int32Slots } from "../../ir/int32-slots.js";
+import type { InlineArrayClass } from "../../ir/inline-arrays.js";
 import {
   cycleRootLines,
   traceFieldVisitLines,
@@ -100,7 +101,9 @@ export class LlClassMeta {
    * a vtable word, the rc word holds the class id (classIdRcWord) in its
    * high 32 bits (CLASS_ID_OFFSET) and the reference count in its low 32
    * bits, and the hierarchy's vtables are one array indexed by the id
-   * (mangleVtArray). */
+   * (mangleVtArray). A standalone class that is an arm of a class union
+   * (nullable-unions.ts taggedClasses) keeps its id the same way, without
+   * vtables. */
   idInRc = false;
   /** A ProgramLifetime class with the 32-bit header (assignCompactLayouts):
    * the i32 at offset 0 is compactHeaderWord, with the class id in place of
@@ -115,12 +118,30 @@ export class LlClassMeta {
    * a compact class instance as a 32-bit index (COMPACT_REF) rather than a
    * pointer: compactRefLoadLines decodes it, compactRefStoreLines encodes it. */
   compressed: boolean[] = [];
+  /** Compact classes with an inline array field (ir/inline-arrays.ts): the
+   * field's slot holds the element count as an i32, and the elements follow
+   * the fixed fields as 32-bit references from `tailOffset`, the same offset
+   * in every class of the declaring class's subtree. Absent otherwise. */
+  inlineTail?: LlInlineTail;
 
   constructor(readonly def: IrClassDef) {
     this.root = this;
   }
 }
 
+/** A class's inline array field (LlClassMeta.inlineTail). */
+export interface LlInlineTail {
+  /** The field's index in def.fields. */
+  readonly field: number;
+  /** Byte offset of the first element. */
+  readonly tailOffset: number;
+  /** The elements' class. */
+  readonly elemClass: string;
+  /** The index in a `new` expression's args of the array the field is
+   * initialized from, whose length the allocation reads. */
+  readonly lengthArg: number;
+}
+
 /** The class graph: link base/children, number the forest in preorder
  * (roots and children in module class order, matching the
  * runtime's stamped error vtables), and compute each hierarchy's virtual
@@ -131,6 +152,7 @@ export function buildClassGraph(
   mod: IrModule,
   fnByName: Map<string, IrFunction>,
   sizeType: "i32" | "i64" = "i64",
+  tagged: ReadonlySet<string> = new Set(),
 ): Map<string, LlClassMeta> {
   const metaMap = new Map<string, LlClassMeta>();
   for (const cls of mod.classes ?? []) {
@@ -161,7 +183,10 @@ export function buildClassGraph(
   // EventEmitter, EventTarget, Event and the streams) and 32-bit targets
   // have no spare half in the rc word: those keep the vtable word.
   for (const meta of metaMap.values())
-    meta.idInRc = sizeType === "i64" && meta.hierarchy && meta.root.def.runtime !== true;
+    meta.idInRc =
+      sizeType === "i64" &&
+      (meta.hierarchy || tagged.has(meta.def.name)) &&
+      meta.root.def.runtime !== true;
   const slotRoots = new Set<LlClassMeta>();
   for (const key of mod.threadSlotKeys ?? []) {
     const meta = metaMap.get(key);
@@ -445,6 +470,12 @@ export function classFieldIndex(
     throw new InternalCompilerError(
       `llvm emitter bug: unknown field ${field} on class ${meta.def.name}`,
     );
+  // An inline array field's slot holds its element count: its reads and
+  // writes go through inline-arrays.ts.
+  if (meta.inlineTail?.field === idx && meta.compact)
+    throw new InternalCompilerError(
+      `llvm emitter bug: field slot of the inline array field ${meta.def.name}.${field}`,
+    );
   const compressed = meta.compressed[idx] === true;
   // Callers that read or write a field's storage as a pointer must not see
   // a compressed one; callers that decode and encode pass "compressed".
@@ -628,10 +659,20 @@ export function classIdRcWord(meta: LlClassMeta, lifetime: boolean): string {
   return (BigInt(meta.pre + 1) * 2n ** 32n + (lifetime ? 2n ** 31n : 1n)).toString();
 }
 
+/** The rc word of an immortal LlClassMeta.idInRc instance (a frame
+ * object): the class id in the high half and the immortal count, all ones,
+ * in the low half, which scr_rc_immortal reads as immortal. */
+export function classIdImmortalRcWord(meta: LlClassMeta): string {
+  if (meta.pre + 1 >= LAYOUT_ID_LIMIT)
+    throw new InternalCompilerError(`class id ${meta.pre + 1} is not a layout id`);
+  return (BigInt(meta.pre + 1) * 2n ** 32n + 0xffffffffn).toString();
+}
+
 /** The 32-bit header of a compact class instance (LlClassMeta.compact):
  * COMPACT_HEADER_TAG, the class id (classIdRcWord's preorder number + 1, 0
- * outside a hierarchy) in the low 28 bits, and COMPACT_PUBLISHED once
- * publication marks the object. No other object's count reaches
+ * for a class without LlClassMeta.idInRc) in the low 28 bits, and
+ * COMPACT_PUBLISHED once publication marks the object. No other object's
+ * count reaches
  * 0x70000000, so the runtime reads a word of 0xf0000000 and up as published
  * for every object (scr_rc.h SCR_RC_PUBLISHED). */
 export const COMPACT_HEADER_TAG = 0x70000000;
@@ -650,7 +691,7 @@ export function storedHeaderBytes(meta: LlClassMeta): number {
 
 /** The header word a compact instance is created with. */
 export function compactHeaderWord(meta: LlClassMeta): number {
-  return COMPACT_HEADER_TAG | (meta.hierarchy ? meta.pre + 1 : 0);
+  return COMPACT_HEADER_TAG | (meta.idInRc ? meta.pre + 1 : 0);
 }
 
 /** Lines that decode the 32-bit reference `idx` (an i32) of a compressed
@@ -767,6 +808,7 @@ export function assignCompactLayouts(
     field: number,
     isCompact: (className: string) => boolean,
   ) => boolean,
+  inlineArrays?: ReadonlyMap<string, InlineArrayClass>,
 ): void {
   const isCompact = (className: string): boolean => {
     const target = metaMap.get(className);
@@ -798,7 +840,18 @@ export function assignCompactLayouts(
         ? base!.compressed[i]!
         : compressedRef !== undefined && compressedRef(meta.def, i, isCompact),
     );
+    // An inline array field (ir/inline-arrays.ts) whose elements are compact
+    // keeps its element count in its slot.
+    const inline = inlineArrays?.get(meta.def.name);
+    const inlineField =
+      inline !== undefined && isCompact(inline.elemClass) && isCompact(inline.declaringClass)
+        ? inline.fieldIndex
+        : -1;
     const types = fieldTypes(meta.def).map((t, i) => {
+      if (i === inlineField) {
+        compressed[i] = false;
+        return "i32";
+      }
       if (!compressed[i]) return t;
       if (t !== "ptr")
         throw new InternalCompilerError(
@@ -846,6 +899,44 @@ export function assignCompactLayouts(
     meta.compactSlots = slots;
     meta.compressed = compressed;
   }
+  if (inlineArrays !== undefined) assignInlineTails(metaMap, inlineArrays);
+}
+
+/** Gives each compact class of an inline array field's subtree its tail
+ * (LlClassMeta.inlineTail): the elements start after the largest fixed part
+ * of any class in the subtree, so that a read through any static type of
+ * the subtree finds them at the same offset. */
+function assignInlineTails(
+  metaMap: ReadonlyMap<string, LlClassMeta>,
+  inlineArrays: ReadonlyMap<string, InlineArrayClass>,
+): void {
+  const subtrees = new Map<string, LlClassMeta[]>();
+  for (const [name, plan] of inlineArrays) {
+    const meta = metaMap.get(name);
+    if (meta === undefined || !meta.compact) continue;
+    // The layout gave the field its i32 count only when the elements are
+    // compact (assignCompactLayouts).
+    const slot = meta.compactSlots[plan.fieldIndex];
+    if (slot === undefined || meta.compactElems[slot]?.type !== "i32") continue;
+    const members = subtrees.get(plan.declaringClass);
+    if (members === undefined) subtrees.set(plan.declaringClass, [meta]);
+    else members.push(meta);
+  }
+  for (const members of subtrees.values()) {
+    let end = 0;
+    for (const meta of members)
+      for (const e of meta.compactElems) end = Math.max(end, e.offset + e.size);
+    const tailOffset = alignTo(end, 4);
+    for (const meta of members) {
+      const plan = inlineArrays.get(meta.def.name)!;
+      meta.inlineTail = {
+        field: plan.fieldIndex,
+        tailOffset,
+        elemClass: plan.elemClass,
+        lengthArg: plan.lengthArg,
+      };
+    }
+  }
 }
 
 function alignTo(n: number, align: number): number {
@@ -885,6 +976,8 @@ export function classUndefinedFieldInits(
     if (f.name === "%cause") return;
     // A nullable-pointer field's zeroed slot already is its unit arm.
     if (nullable?.get(meta.def.name, f.name)) return;
+    // An inline array field's slot holds its count, which `new` stores.
+    if (meta.inlineTail?.field === i && meta.compact) return;
     // Only union and dyn fields get a store here; a compressed field is a
     // class reference, which starts NULL.
     const { index, compressed } = classFieldIndex(meta, f.name, "compressed");
@@ -1024,7 +1117,7 @@ export function emitClassShapes(
   // instance is the element at its number minus the root's.
   for (const name of roots) {
     const root = metaMap.get(name)!;
-    if (!root.idInRc) continue;
+    if (!root.hierarchy || !root.idInRc) continue;
     const members = [...metaMap.values()]
       .filter((m) => m.root === root)
       .sort((a, b) => a.pre - b.pre);
@@ -1395,6 +1488,7 @@ export function emitClassShapes(
                 count,
               )
             : null,
+          count,
         ),
         ``,
       );
@@ -1404,8 +1498,12 @@ export function emitClassShapes(
     // members, undefined-admitting union fields at the interned unit
     // instance. Traced shapes allocate with the collector header; alloc.ts
     // inlines the allocator fast paths and the alloc note.
+    // An inline array field's instance (LlClassMeta.inlineTail) takes the
+    // field's element count and has room for the elements after its fixed
+    // fields.
+    const tail = meta.compact && regionRefs ? (meta.inlineTail ?? null) : null;
     const nw: string[] = [
-      `define internal ptr @${mangleClassNew(cls.name)}() ${FN_ATTRS} { ; new ${cls.name}`,
+      `define internal ptr @${mangleClassNew(cls.name)}(${tail !== null ? "i32 %inl.len" : ""}) ${FN_ATTRS} { ; new ${cls.name}`,
       `entry:`,
     ];
     const cycType = `@${mangleClassTrace(cls.name)}.cyctype`;
@@ -1433,7 +1531,14 @@ export function emitClassShapes(
       // Compressed fields refer to compact instances by their index in the
       // program-lifetime region, so every compact instance lives there:
       // zeroed, 8-byte aligned and never freed.
-      nw.push(...plrAllocLines(host, sizeOf));
+      if (tail !== null) {
+        nw.push(
+          `  %inl.n = zext i32 %inl.len to ${host.sizeType}`,
+          `  %inl.bytes = shl nuw nsw ${host.sizeType} %inl.n, 2`,
+          `  %inl.size = add nuw nsw ${host.sizeType} %inl.bytes, ${tail.tailOffset}`,
+        );
+        nw.push(...plrAllocLines(host, "%inl.size"));
+      } else nw.push(...plrAllocLines(host, sizeOf));
     } else
       nw.push(
         ...emitObjectAlloc(
@@ -1510,6 +1615,11 @@ export function emitClassShapes(
       );
     }
     nw.push(...undefFieldInits(host, meta, nullable));
+    if (tail !== null)
+      nw.push(
+        `  %inl.countp = getelementptr inbounds %${struct}, ptr %o, i64 0, i32 ${meta.compactSlots[tail.field]!}`,
+        `  store i32 %inl.len, ptr %inl.countp ; ${llvmCommentText(cls.name)} inline array count`,
+      );
     // Young instances: program-owned traced classes. Classes rooted at a
     // runtime base (errors, emitters, events, streams) carry runtime state
     // and stay with trial deletion, like every runtime family.
@@ -1661,6 +1771,10 @@ export function emitClassObjDefs(
       throw new InternalCompilerError(
         `llvm emitter bug: class object for ${className} without a constructor`,
       );
+    if (meta.inlineTail !== undefined)
+      throw new InternalCompilerError(
+        `llvm emitter bug: class object for ${className}, which has an inline array field`,
+      );
     const params = ctor.params.slice(valueCtor.firstArg);
     const paramDecls = params.map((p, i) => `${llType(p.type)} %a${i}`).join(", ");
     const ctorArgs = params.map((p, i) => `${llType(p.type)} %a${i}`);

packages/compiler/src/backend/llvm/emit-function.ts

@@ -62,7 +62,7 @@ import { emitIntegerClone, emitIntegerDispatcher } from "./integer-clones.js";
 import { beginFunctionLoops, LOOPS } from "./loop-state.js";
 import { runningExtremumUpdates } from "./running-extrema.js";
 import { findIntegerOrNaNLocals } from "./wrapping-sums.js";
-import { UNWIND_PERSONALITY, UNWIND_SLOT } from "./unwind-abi.js";
+import { UNWIND_PERSONALITY, UNWIND_RESUME_DECLARATION, UNWIND_SLOT } from "./unwind-abi.js";
 
 export function emitFunction(emitter: LlEmitter, fn: IrFunction): string {
   const B = new BlockBuilder();
@@ -681,7 +681,10 @@ export function emitFunction(emitter: LlEmitter, fn: IrFunction): string {
   emitter.privateGlobalSlots.clear();
   // Landing pads (emitDirectCall) need the personality.
   const personality = emitter.usesLandingPads ? ` ${UNWIND_PERSONALITY}` : "";
-  if (emitter.usesLandingPads) emitter.declare(`declare i32 @__gcc_personality_v0(...)`);
+  if (emitter.usesLandingPads) {
+    emitter.declare(`declare i32 @__gcc_personality_v0(...)`);
+    emitter.declare(UNWIND_RESUME_DECLARATION);
+  }
   let body = `define internal ${ret} @${symbol}${int32Abi ? INT32_ABI_SUFFIX : ""}(${params.join(", ")}) ${attrs}${personality}${debug} { ; ${fn.name}\n${B.render()}\n}`;
   if (int32Abi) body += "\n" + emitter.emitInt32AbiWrapper(fn, symbol, int32Abi);
   if (emitter.integerClones.members.has(fn.name)) {

packages/compiler/src/backend/llvm/emit-module.ts

@@ -115,24 +115,56 @@ export function workerMinFunctions(): number {
   return Number.isFinite(value) && value > 0 ? value : 2000;
 }
 
+/** A function's estimated emission cost beyond its IR nodes, in nodes:
+ * fitted to the worker emission times of a large program's ranges. */
+const FUNCTION_EMIT_COST = 22;
+
+/** Where each of about `count` contiguous ranges of `functions` starts,
+ * followed by the end: ranges of about equal estimated emission cost, a
+ * function's node count plus FUNCTION_EMIT_COST. No range is empty. */
+export function emitRangeStarts(
+  functions: readonly IrFunction[],
+  count: number,
+  nodeCount: (fn: IrFunction) => number | undefined,
+): number[] {
+  const costs = functions.map((fn) => (nodeCount(fn) ?? 0) + FUNCTION_EMIT_COST);
+  const total = costs.reduce((sum, cost) => sum + cost, 0);
+  const starts = [0];
+  let sum = 0;
+  for (let i = 0; i < functions.length - 1 && starts.length < count; i++) {
+    sum += costs[i]!;
+    if (sum >= (total * starts.length) / count) starts.push(i + 1);
+  }
+  starts.push(functions.length);
+  return starts;
+}
+
 /** The function bodies emitted on forked worker processes, `workers` at a
- * time: the function list splits into contiguous ranges, each emitted in
- * its own process, which inherits this emitter as it is at the fork and
- * records a journal (emit-journal.ts). Ranges merge in function order, so
- * the text is the serial text. A range whose worker fails is emitted here
- * when its turn comes. Null without fork primitives (Node, Windows, inside
- * a worker) or when the module is small. */
+ * time: the function list splits into contiguous ranges of about equal
+ * estimated cost (emitRangeStarts), each emitted in its own process, which
+ * inherits this emitter as it is at the fork and records a journal
+ * (emit-journal.ts). A new range starts whenever a worker ends, and ranges
+ * merge in function order, so the text is the serial text. A range whose
+ * worker fails is emitted here when its turn comes. Null without fork
+ * primitives (Node, Windows, inside a worker) or when the module is small. */
 function emitFunctionsOnWorkers(emitter: LlEmitter, workers: number): string[] | null {
   const fork = emitter.targetOptions.forkPrimitives;
   const functions = emitter.mod.functions;
   if (fork === undefined || workers < 2) return null;
   if (functions.length < workerMinFunctions()) return null;
   // More ranges than workers: a range's merge overlaps the later ranges'
-  // emission, and a slow range holds back less.
-  const count = Math.min(functions.length, workers * 3);
-  const size = Math.ceil(functions.length / count);
+  // emission, and a slow range holds back less. Ranges of equal cost finish
+  // at about the same time.
+  const starts = emitRangeStarts(functions, Math.min(functions.length, workers * 3), (fn) =>
+    emitter.nodeIndex.nodeCount(fn),
+  );
+  const count = starts.length - 1;
+  const rangeFunctions = (range: number): readonly IrFunction[] =>
+    functions.slice(starts[range], starts[range + 1]);
   const dir = mkdtempSync(join(tmpdir(), "scriptc-emit-"));
   const pids: number[] = [];
+  // The workers not yet reaped, by pid, with their ranges.
+  const running = new Map<number, number>();
   const start = (range: number): void => {
     const pid = fork.fork();
     if (pid === 0) {
@@ -141,7 +173,7 @@ function emitFunctionsOnWorkers(emitter: LlEmitter, workers: number): string[] |
         emitter.journal = new EmitJournal();
         emitter.debug?.record();
         const texts: string[] = [];
-        for (const fn of functions.slice(range * size, (range + 1) * size)) {
+        for (const fn of rangeFunctions(range)) {
           texts.push(emitter.emitFunction(fn));
           relieveHeapPressure();
         }
@@ -155,17 +187,37 @@ function emitFunctionsOnWorkers(emitter: LlEmitter, workers: number): string[] |
       fork.exit(code);
     }
     pids.push(pid);
+    if (pid > 0) running.set(pid, range);
   };
   const fnDefs: string[] = [];
+  // Ended ranges by index, with what their workers wrote (null when a worker
+  // failed), waiting to merge in function order.
+  const ended = new Map<number, { result: ChunkResult; texts: string[] } | null>();
+  // Reaps a worker and starts the next range in its place. A worker's journal
+  // records requests, not table contents, so its range's text doesn't depend
+  // on what merged before it forked.
+  const reap = (pid: number, range: number): void => {
+    const status = pid > 0 ? fork.wait(pid) : -1;
+    running.delete(pid);
+    if (pids.length < count) start(pids.length);
+    ended.set(range, status === 0 ? readRange(dir, range) : null);
+  };
   try {
     for (let range = 0; range < Math.min(workers, count); range++) start(range);
     for (let range = 0; range < count; range++) {
-      const pid = pids[range]!;
-      const status = pid > 0 ? fork.wait(pid) : -1;
-      // The next worker starts from this emitter as merged so far, which
-      // the journal allows: it records requests, not table contents.
-      const merged = status === 0 ? readRange(dir, range) : null;
-      const slice = functions.slice(range * size, (range + 1) * size);
+      // Until this range's worker ends, reap whichever worker ends first, so
+      // that `workers` keep running while ranges merge in order. A child
+      // that isn't a worker (or no `exited`) waits for this range's worker.
+      while (!ended.has(range)) {
+        const pid = pids[range]!;
+        const first = pid > 0 && fork.exited !== undefined ? fork.exited() : pid;
+        const other = running.get(first);
+        if (first !== pid && other !== undefined) reap(first, other);
+        else reap(pid, range);
+      }
+      const merged = ended.get(range)!;
+      ended.delete(range);
+      const slice = rangeFunctions(range);
       if (merged !== null && merged.texts.length === slice.length) {
         const rename = emitter.mergeChunk(merged.result);
         slice.forEach((fn, index) => {
@@ -180,7 +232,6 @@ function emitFunctionsOnWorkers(emitter: LlEmitter, workers: number): string[] |
           fnDefs.push(def);
           relieveHeapPressure();
         }
-      if (pids.length < count) start(pids.length);
     }
   } finally {
     rmSync(dir, { recursive: true, force: true });

packages/compiler/src/backend/llvm/emit-statements.ts

@@ -1,4 +1,5 @@
 /* LlEmitter's statement emission: the statement switch, try/catch, switch, loop copies, speculative writes and string self-concatenation. */
+import { emitInlineArrayInit, inlineFieldTail } from "./inline-arrays.js";
 import { emitLoopBytesHeaders } from "./bytes-headers.js";
 import { noteBytesReceiver } from "./expr-bytes.js";
 import {
@@ -73,7 +74,7 @@ import { bytesHeaderAlias } from "./bytes-alias.js";
 import { mangleLocal } from "../mangle.js";
 import { LlvmUnsupportedError } from "./unsupported.js";
 import type { LlValue } from "./expr-context.js";
-import { everyExprChild, everyStmtChild } from "../../ir/traverse.js";
+import { everyExprChild, everyStmtChild, everyStmtList } from "../../ir/traverse.js";
 import { widenInteger } from "./integer-values.js";
 import { SCALAR_UNION_RESULT } from "./int32-abi.js";
 import { emitLiteralSwitch } from "./switch-dispatch.js";
@@ -620,6 +621,25 @@ export function emitStmtBody(emitter: LlEmitter, s: IrStmt): void {
       // release — a release can trigger a cycle collection, which must
       // never see a heap edge whose count was already given up).
       // Classes and records share the struct layout, so one emission.
+      // The constructor's write of an inline array field copies the array's
+      // elements into the instance (backend/llvm/inline-arrays.ts).
+      const tail =
+        s.kind === "fieldSet" ? inlineFieldTail(emitter, s.className, s.field) : undefined;
+      if (tail !== undefined && s.kind === "fieldSet") {
+        const obj = emitter.emitStableReceiver(s.obj, [s.value]);
+        const v = emitter.emitExpr(s.value);
+        emitter.emitStoreGuard(obj.name, s);
+        emitter.heapCensus?.countStore(
+          (line) => B.line(line),
+          () => B.tmp(),
+          s.className,
+          s.field,
+          s.value,
+          v.name,
+        );
+        emitInlineArrayInit(emitter, obj.name, s.className, tail, v.name);
+        break;
+      }
       const nullable =
         s.kind === "fieldSet" ? emitter.nullableFields.get(s.className, s.field) : null;
       if (nullable && s.kind === "fieldSet") {
@@ -1290,6 +1310,37 @@ export function emitStmtBody(emitter: LlEmitter, s: IrStmt): void {
         emitter.workerLoopPolls(s) &&
         (loopAccessesTypedArrays(s) || (countedLoop !== null && innermost));
       const workerBudget = stripMined || bounded ? null : emitter.workerLoopBudget(s);
+      // An innermost counted loop, or a two-counter loop closing in by one
+      // step on each side, has a run whose length its counters fix on entry.
+      // With a small body, a run of at most WORKER_UNPOLLED_RUN steps takes a
+      // plain copy of the loop without a poll, which polls once as it leaves
+      // when it ran more than WORKER_POLL_BLOCK steps, like a strip-mined run;
+      // longer runs take the loop as it is emitted otherwise.
+      const shortRuns: { stride: number; left: () => string } | null =
+        !innermost ||
+        bounded ||
+        capturedLets.length > 0 ||
+        !emitter.workerLoopPolls(s) ||
+        !withinNodes(s.body, MAX_SHORT_RUN_NODES)
+          ? null
+          : countedLoop !== null && countedLimit !== null && integerSlot !== null
+            ? {
+                stride: Math.abs(countedLoop.step),
+                left: () => {
+                  const start = B.tmp(),
+                    left = B.tmp();
+                  B.line(`${start} = load i64, ptr ${integerSlot}`);
+                  B.line(
+                    countedLoop.step > 0
+                      ? `${left} = sub nsw i64 ${countedLimit}, ${start}`
+                      : `${left} = sub nsw i64 ${start}, ${countedLimit}`,
+                  );
+                  return left;
+                },
+              }
+            : pairedLoop !== null && pairedLoop.low.step === 1 && pairedLoop.high.step === -1
+              ? { stride: 2, left: () => emitPairedLoopGap(emitter, pairedSlots) }
+              : null;
       // One slot for every copy of the loop: a speculative copy resumes an
       // iteration in another copy's body, which then reads this bound.
       let blockBound: string | null = null;
@@ -1305,48 +1356,12 @@ export function emitStmtBody(emitter: LlEmitter, s: IrStmt): void {
         }
       }
       freshenBindings();
-      emitter.emitLoopCopies(s, capturedLets.length === 0, le, (exit) => {
-        if (stripMined) {
-          const counter: StripMinedCounter = countedLoop
-            ? {
-                localId: countedLoop.localId,
-                slot: integerSlot!,
-                type: "i64",
-                signed: true,
-                step: countedLoop.step,
-                followers: countedLoop.followers.map((f, i) => ({
-                  slot: followerSlots[i]!,
-                  step: f.step,
-                })),
-                limit: () => countedLimit!,
-              }
-            : {
-                localId: integerLoop!.localId,
-                slot: integerSlot!,
-                type: emitter.sizeType,
-                signed: false,
-                step: 1,
-                followers: [],
-                limit: () => {
-                  const receiver = emitter.emitStableReceiver(integerLoop!.limitReceiver, []);
-                  const lenPtr = B.tmp();
-                  const len = B.tmp();
-                  B.line(
-                    `${lenPtr} = getelementptr inbounds %ScrBytes, ptr ${receiver.name}, i64 0, i32 1`,
-                  );
-                  B.line(
-                    `${len} = load ${emitter.sizeType}, ptr ${lenPtr}${bytesHeaderAlias(emitter)}`,
-                  );
-                  return len;
-                },
-              };
-          emitStripMinedCountedLoop(emitter, s, counter, blockBound!, exit, runSlot);
-          return;
-        }
+      // The loop as a plain header, body and update, polling by `budget`.
+      const emitPlainLoop = (exit: string, budget: string | null): void => {
         const lc = B.newLabel("loop.c");
         const lb = B.newLabel("loop.b");
         const lu =
-          s.update || capturedLets.length > 0 || workerBudget !== null ? B.newLabel("loop.u") : lc;
+          s.update || capturedLets.length > 0 || budget !== null ? B.newLabel("loop.u") : lc;
         // A periodic guard on the counter (periodic-guards.ts): the back
         // edge asks LLVM to unroll the loop by the guard's period.
         const period =
@@ -1417,7 +1432,7 @@ export function emitStmtBody(emitter: LlEmitter, s: IrStmt): void {
         B.br(lu, lu === lc ? backEdge : "");
         if (lu !== lc) {
           B.startBlock(lu);
-          emitter.emitWorkerLoopCheck(workerBudget);
+          emitter.emitWorkerLoopCheck(budget);
           freshenBindings();
           if (integerLoop && integerSlot) {
             const old = B.tmp();
@@ -1449,6 +1464,78 @@ export function emitStmtBody(emitter: LlEmitter, s: IrStmt): void {
           } else if (s.update) emitter.emitStmt(s.update);
           B.br(lc, backEdge);
         }
+      };
+      emitter.emitLoopCopies(s, capturedLets.length === 0, le, (exit) => {
+        // Speculative copies resume iterations inside the copy they enter,
+        // which must keep its polls.
+        if (
+          shortRuns &&
+          emitter.speculation === null &&
+          emitter.speculationResume === null &&
+          emitter.speculationHeader === null &&
+          emitter.int32Kinds === null
+        ) {
+          // The run's length in counter steps (two counter steps an
+          // iteration for a two-counter loop), signed: a run that does not
+          // enter the body is short too.
+          const left = shortRuns.left();
+          const inRun = B.tmp(),
+            pollsAfter = B.tmp();
+          const shortLabel = B.newLabel("loop.short"),
+            longLabel = B.newLabel("loop.long"),
+            shortEnd = B.newLabel("loop.short.end"),
+            shortPoll = B.newLabel("loop.short.poll");
+          B.line(`${inRun} = icmp sle i64 ${left}, ${shortRuns.stride * WORKER_UNPOLLED_RUN}`);
+          B.condBr(inRun, shortLabel, longLabel);
+          B.startBlock(shortLabel);
+          emitPlainLoop(shortEnd, null);
+          // A run longer than one block polls once as it leaves.
+          B.startBlock(shortEnd);
+          B.line(`${pollsAfter} = icmp sgt i64 ${left}, ${shortRuns.stride * WORKER_POLL_BLOCK}`);
+          B.condBr(pollsAfter, shortPoll, exit);
+          B.startBlock(shortPoll);
+          emitter.emitWorkerPoll();
+          B.br(exit);
+          B.startBlock(longLabel);
+        }
+        if (stripMined) {
+          const counter: StripMinedCounter = countedLoop
+            ? {
+                localId: countedLoop.localId,
+                slot: integerSlot!,
+                type: "i64",
+                signed: true,
+                step: countedLoop.step,
+                followers: countedLoop.followers.map((f, i) => ({
+                  slot: followerSlots[i]!,
+                  step: f.step,
+                })),
+                limit: () => countedLimit!,
+              }
+            : {
+                localId: integerLoop!.localId,
+                slot: integerSlot!,
+                type: emitter.sizeType,
+                signed: false,
+                step: 1,
+                followers: [],
+                limit: () => {
+                  const receiver = emitter.emitStableReceiver(integerLoop!.limitReceiver, []);
+                  const lenPtr = B.tmp();
+                  const len = B.tmp();
+                  B.line(
+                    `${lenPtr} = getelementptr inbounds %ScrBytes, ptr ${receiver.name}, i64 0, i32 1`,
+                  );
+                  B.line(
+                    `${len} = load ${emitter.sizeType}, ptr ${lenPtr}${bytesHeaderAlias(emitter)}`,
+                  );
+                  return len;
+                },
+              };
+          emitStripMinedCountedLoop(emitter, s, counter, blockBound!, exit, runSlot);
+          return;
+        }
+        emitPlainLoop(exit, workerBudget);
       });
       B.startBlock(le);
       emitter.releaseScope(emitter.scopes.pop()!);
@@ -2781,6 +2868,34 @@ function boundedLoopRuns(emitter: LlEmitter, loop: IrStmt): number {
   return trips * (1 + nested);
 }
 
+/** The most IR nodes the body of a loop with an unpolled copy for short
+ * runs may have: the copy duplicates the loop. */
+const MAX_SHORT_RUN_NODES = 256;
+
+/** Whether `body` has at most `limit` statement and expression nodes. */
+function withinNodes(body: IrStmt[], limit: number): boolean {
+  let nodes = 0;
+  return everyStmtList(body, {
+    expr: () => ++nodes <= limit,
+    stmt: () => ++nodes <= limit,
+  });
+}
+
+/** The gap between the counters of a two-counter loop stepping them by 1
+ * and -1 (`for (let i = a, j = b; i < j; i++, j--)`), from the values in
+ * `slots`: each iteration closes it by two, so a run takes about half the
+ * gap in iterations, and none when the gap is closed. */
+function emitPairedLoopGap(emitter: LlEmitter, slots: readonly string[]): string {
+  const B = emitter.B;
+  const low = B.tmp(),
+    high = B.tmp(),
+    gap = B.tmp();
+  B.line(`${low} = load i64, ptr ${slots[0]}`);
+  B.line(`${high} = load i64, ptr ${slots[1]}`);
+  B.line(`${gap} = sub nsw i64 ${high}, ${low}`);
+  return gap;
+}
+
 /** The counter of a strip-mined loop: its binding, its slot, the LLVM
  * integer type and signedness of its values, its step, the counters stepped
  * beside it, and the limit, emitted in the outer header before each block. */

packages/compiler/src/backend/llvm/emitter.ts

@@ -88,6 +88,13 @@ import {
 import { specializeNumericCalls } from "../../ir/numeric-call-specialization.js";
 import { versionLoopBoundParameters } from "../../ir/parameter-versions.js";
 import { inlineExpressionFunctions } from "../../ir/expression-inlining.js";
+import {
+  inlineArrayReads,
+  NO_INLINE_ARRAYS,
+  planInlineArrays,
+  type InlineArrayPlan,
+} from "../../ir/inline-arrays.js";
+import { inlineArraysApply, programLifetimeClassOf } from "./inline-arrays.js";
 import { pairRowSums } from "../../ir/row-pairs.js";
 import {
   allocatedLiteral,
@@ -131,7 +138,12 @@ import {
   type Int32Abi,
 } from "./int32-abi.js";
 import { type Int32Local } from "./int32-locals.js";
-import { findIntegerClones, NO_INTEGER_CLONES, type IntegerClones } from "./integer-clones.js";
+import {
+  findIntegerClones,
+  NO_INTEGER_CLONES,
+  type CloneReferences,
+  type IntegerClones,
+} from "./integer-clones.js";
 import { writeRepresentationReport } from "./repr-report.js";
 import { heapCensusOf, type HeapCensus } from "./heap-census.js";
 import { exactInteger, narrowInteger, widenInteger, type LlInteger } from "./integer-values.js";
@@ -148,6 +160,7 @@ import {
   UNWIND_PERSONALITY,
   UNWIND_RAISE,
   UNWIND_RAISE_DECLARATION,
+  UNWIND_RESUME_DECLARATION,
   UNWIND_RESUME,
   UNWIND_SLOT,
   UnwindAbiViolation,
@@ -348,6 +361,7 @@ import {
   classStructSym,
   programLifetimeClass,
   type LlClassMeta,
+  type LlInlineTail,
 } from "./classes.js";
 import { LifetimeRegions, REGION_TYPES } from "./regions.js";
 import {
@@ -364,7 +378,16 @@ import { VirtualBorrows } from "./virtual-borrows.js";
 import { planRecordViews, recordViewFieldPtr } from "./record-views.js";
 import { LlvmUnsupportedError } from "./unsupported.js";
 import { LlWalkers } from "./walkers.js";
-import { isObjectArm, NULLABLE_ABSENT, NULLABLE_NULL, NullableUnions } from "./nullable-unions.js";
+import {
+  isObjectArm,
+  isPayloadTag,
+  NULLABLE_ABSENT,
+  NULLABLE_NULL,
+  NullableUnions,
+  pointerTagFn,
+  type NullableUnion,
+} from "./nullable-unions.js";
+import { classUnionHeaders, emitClassUnionHelpers } from "./class-unions.js";
 import { emitNullablePresent, emitUnionPeek, emitUnionTag } from "./union-repr.js";
 import {
   emitConcatInputs,
@@ -500,6 +523,10 @@ export interface ForkPrimitives {
   exit(code: number): void;
   /** The worker's exit status, 128 plus a signal number, or -1. */
   wait(pid: number): number;
+  /** The pid of a child process that has ended, waiting for one if none
+   * has, or -1, without reaping it. Without it, workers are waited for in
+   * order. */
+  exited?(): number;
 }
 
 export interface LlvmTargetOptions {
@@ -677,6 +704,13 @@ function resultEntryTargets(mod: IrModule): ReadonlySet<string> {
   return targets;
 }
 
+/** The reads and constructions of inline array fields prepared for their
+ * layout (ir/inline-arrays.ts), where the target keeps them inline. */
+function inlineArraysPass(options: LlvmTargetOptions, mod: IrModule): IrModule {
+  if (!inlineArraysApply(mod, options)) return mod;
+  return inlineArrayReads(mod, planInlineArrays(mod, programLifetimeClassOf(mod)));
+}
+
 /** The emitter for `input` after the module passes every emission runs. */
 function moduleEmitter(input: IrModule, options: LlvmTargetOptions): LlEmitter {
   // Facts are keyed by node identity: emit the same module whether it was
@@ -722,14 +756,17 @@ function moduleEmitter(input: IrModule, options: LlvmTargetOptions): LlEmitter {
           specializeCallbackCalls(
             specializeNumericCalls(
               pairRowSums(
-                inlineExpressionFunctions(
-                  foldConstantGlobals(
-                    copyStableCaptures(
-                      resolveSelfCaptures(
-                        hoistFinalFieldReads(
-                          freezeInitOnlyGlobals(
-                            holdCharLocalsAsUnits(
-                              dropUnwrittenPropertyBags(runPairedWhileLoopsAsFor(mod)),
+                inlineArraysPass(
+                  options,
+                  inlineExpressionFunctions(
+                    foldConstantGlobals(
+                      copyStableCaptures(
+                        resolveSelfCaptures(
+                          hoistFinalFieldReads(
+                            freezeInitOnlyGlobals(
+                              holdCharLocalsAsUnits(
+                                dropUnwrittenPropertyBags(runPairedWhileLoopsAsFor(mod)),
+                              ),
                             ),
                           ),
                         ),
@@ -1750,6 +1787,8 @@ export class LlEmitter {
   readonly int32Abis: ReadonlyMap<string, Int32Abi>;
   /** Pure number functions with speculative integer clones. */
   readonly integerClones: IntegerClones;
+  /** The records integer clones read (64-bit layouts). */
+  readonly cloneReferences: CloneReferences | undefined;
   /** The IR function being emitted. */
   currentFunctionName = "";
 
@@ -1762,6 +1801,9 @@ export class LlEmitter {
   /** ProgramLifetime hierarchy roots (contract types): their instances are
    * immortal from construction (classes.ts programLifetimeClass). */
   readonly programLifetime: ReadonlySet<string>;
+  /** Inline array fields this emission may keep inline (ir/inline-arrays.ts);
+   * the classes that do have LlClassMeta.inlineTail. */
+  readonly inlineArrays: InlineArrayPlan;
   numericLocals = new Map<string, IrLocal>();
   captureIds = new Set<string>();
   /** Active induction bindings retain their width independently of size_t. */
@@ -2039,6 +2081,7 @@ export class LlEmitter {
       mod,
       options.debugSources === undefined,
       this.nodeIndex,
+      options.pointerBits !== 32,
     );
     this.heapCensus = heapCensusOf(mod, this.nullableUnions);
     for (const decl of this.heapCensus?.declarations() ?? []) this.declare(decl);
@@ -2207,6 +2250,12 @@ export class LlEmitter {
         this.immutableFields,
         this.nodeIndex,
       );
+    // Inline array fields (ir/inline-arrays.ts), where the module passes
+    // prepared their reads and constructions.
+    this.inlineArrays =
+      options.debugSources === undefined && inlineArraysApply(mod, options)
+        ? planInlineArrays(mod, programLifetimeClassOf(mod))
+        : NO_INLINE_ARRAYS;
     const walkParameters = new Map<string, Set<number>>();
     if (this.debug === null) {
       const host = {
@@ -2275,26 +2324,6 @@ export class LlEmitter {
       };
     }
     this.constantCallbacks = findConstantCallbacks(mod, this.callLifetimes, this.nodeIndex);
-    // Integer clones signal a stack overflow through their result, which
-    // needs the single-threaded guard; worker programs also poll for
-    // termination, and debug builds and stack traces keep every frame.
-    // Clones read number arrays their function borrows (64-bit layouts).
-    this.integerClones =
-      this.debug === null &&
-      this.stackChecks !== null &&
-      mod.workers !== true &&
-      mod.lib === undefined &&
-      !this.stackTraces
-        ? findIntegerClones(
-            mod,
-            this.sizeType === "i64"
-              ? (fn, index) => this.callLifetimes.borrowed.get(fn)?.has(index) === true
-              : undefined,
-          )
-        : NO_INTEGER_CLONES;
-    // Integer clones and their dispatchers call each other through their
-    // own lowering, which keeps plain calls and flag tests.
-    for (const name of this.integerClones.members) this.unwindAbi.delete(name);
     this.stackCallbacks = new StackCallbacks(this.fnByName);
     this.receiverReaders = new AmbientReceiverReaders(this.fnByName, this.nodeIndex);
     this.stackCaptures = new StackCaptures(this.fnByName, this.callLifetimes, this.stackCallbacks);
@@ -2323,6 +2352,13 @@ export class LlEmitter {
         }
       }
     }
+    // The same holds for a class union's arms (class-unions.ts).
+    classUnionHeaders(
+      mod,
+      this.nullableUnions.allClassUnions(),
+      this.tracedShapes,
+      this.tracedUnions,
+    );
     this.shapeHost = {
       // Journal hooks only on emitters that can emit ranges (chunk
       // emitters, and emitters whose workers take a journal after the
@@ -2450,6 +2486,30 @@ export class LlEmitter {
       this.shapeHost.recordViews = views;
       this.recordViewTargets = views.targets as Set<string>;
     }
+    // Integer clones signal a stack overflow through their result. Worker
+    // programs have them only as ELF executables, where the stack guard
+    // and the pending word clones poll for termination are plain
+    // thread-locals; debug builds and stack traces keep every frame.
+    // Clones read number arrays and record graphs their function borrows
+    // (64-bit layouts), but no record that may be a view (planned above).
+    this.cloneReferences = this.sizeType === "i64" ? this.recordCloneReferences() : undefined;
+    this.integerClones =
+      this.debug === null &&
+      this.stackChecks !== null &&
+      (mod.workers !== true || this.executableTls) &&
+      mod.lib === undefined &&
+      !this.stackTraces
+        ? findIntegerClones(
+            mod,
+            this.sizeType === "i64"
+              ? (fn, index) => this.callLifetimes.borrowed.get(fn)?.has(index) === true
+              : undefined,
+            this.cloneReferences,
+          )
+        : NO_INTEGER_CLONES;
+    // Integer clones and their dispatchers call each other through their
+    // own lowering, which keeps plain calls and flag tests.
+    for (const name of this.integerClones.members) this.unwindAbi.delete(name);
     this.walkers = new LlWalkers(this.shapeHost);
     this.dyn = new LlDyn(this.shapeHost);
     for (const g of mod.globals ?? []) {
@@ -2488,7 +2548,12 @@ export class LlEmitter {
     // The class graph: base/children links, hierarchy membership, the
     // whole-program preorder numbering for instanceof over runtime and
     // compiled error objects, and the per-hierarchy virtual slot lists.
-    this.classMeta = buildClassGraph(mod, this.fnByName, this.sizeType);
+    this.classMeta = buildClassGraph(
+      mod,
+      this.fnByName,
+      this.sizeType,
+      this.nullableUnions.taggedClasses(),
+    );
     if (mod.lifetimeRegions === true) {
       this.regions = new LifetimeRegions(
         this.shapeHost,
@@ -2532,6 +2597,7 @@ export class LlEmitter {
               return nullable !== null && isCompact(nullable.arm.className);
             }
           : undefined,
+        this.inlineArrays.classes,
       );
     this.compactRegion =
       mod.lib === undefined &&
@@ -2756,6 +2822,7 @@ export class LlEmitter {
       ...late,
       ...emitNullableRetainWrappers(this.shapeHost),
       ...emitShapeUnionHelpers(this.shapeHost),
+      ...emitClassUnionHelpers(this.shapeHost, this.classMeta),
       ...emitInlineRcHelpers(this.shapeHost),
     ];
   }
@@ -4702,11 +4769,7 @@ export class LlEmitter {
     const nullable = this.nullableUnions.get(unionId);
     if (nullable) {
       // The owned payload IS the nullable union's value.
-      if (
-        nullable.repr === "single"
-          ? tag !== nullable.refTag
-          : !nullable.arms.some((arm) => arm.tag === tag)
-      )
+      if (!isPayloadTag(nullable, tag))
         throw new InternalCompilerError(
           `llvm emitter bug: payload wrap into unit arm ${tag} of ${unionId}`,
         );
@@ -4759,6 +4822,13 @@ export class LlEmitter {
     refs: "pointer" | "compressed" = "pointer",
   ): { ptr: string; type: IrType; inline: boolean; compressed: boolean } {
     const meta = this.classMetaOf(className);
+    // An inline array field's slot holds its element count: its reads and
+    // writes go through backend/llvm/inline-arrays.ts.
+    const tail = this.inlineTailOf(className);
+    if (tail !== undefined && meta.def.fields[tail.field]!.name === field)
+      throw new InternalCompilerError(
+        `llvm emitter bug: field pointer to the inline array field ${className}.${field}`,
+      );
     const { index, type, compressed } = classFieldIndex(meta, field, refs);
     const p = this.B.tmp();
     this.B.line(
@@ -4906,6 +4976,49 @@ export class LlEmitter {
 
   /** The field-slot pointer of a record member (rc header at index 0).
    * `inline`: the slot stores a scalar union's tag and bits. */
+  /** The record layouts integer clones read in place: record pointers,
+   * `R | null` unions of a record (NULL is `null`), and fields stored as a
+   * double, an i8 boolean or such a pointer. Records that may be views
+   * (record-views.ts) are not among them. */
+  private recordCloneReferences(): CloneReferences {
+    // A view's fields live in the record it views (recordViewFieldPtr), not
+    // at its own shape's offsets.
+    const plain = (shapeId: string): boolean =>
+      this.recordsById.has(shapeId) && this.shapeHost.recordViews?.targets.has(shapeId) !== true;
+    const nullable = (unionId: string): NullableUnion | null => {
+      const single = this.nullableUnions.single(unionId);
+      if (single === null || single.nullTag !== -1 || single.arm.kind !== "record") return null;
+      if (!plain(single.arm.shapeId)) return null;
+      const unit = this.unionsById.get(unionId)?.arms[single.unitTag];
+      return unit?.kind === "nullT" ? single : null;
+    };
+    const reference = (type: IrType): boolean =>
+      type.kind === "record"
+        ? plain(type.shapeId)
+        : type.kind === "union" && nullable(type.unionId) !== null;
+    return {
+      reference,
+      arm: (unionId, tag) => {
+        const single = nullable(unionId);
+        if (single === null) return null;
+        return tag === single.refTag ? "record" : tag === single.unitTag ? "null" : null;
+      },
+      field: (shapeId, field) => {
+        const shape = this.recordsById.get(shapeId);
+        const index = shape?.fields.findIndex((f) => f.name === field) ?? -1;
+        if (shape === undefined || index < 0 || !plain(shapeId)) return null;
+        const type = shape.fields[index]!.type;
+        if (this.inlineUnions.record(type)) return null;
+        if (type.kind !== "f64" && type.kind !== "bool" && !reference(type)) return null;
+        return {
+          struct: mangleRecordStruct(shapeId),
+          index: index + this.nullableUnions.fieldBase(shapeId),
+          type,
+        };
+      },
+    };
+  }
+
   recordFieldPtr(
     objName: string,
     shapeId: string,
@@ -5998,7 +6111,8 @@ export class LlEmitter {
       const exception = B.tmp();
       B.line(`${exception} = landingpad { ptr, i32 } cleanup`);
       for (const index of borrowed) this.releaseFrame(`%p${index}`, fn.params[index]!.type);
-      B.terminate(`resume { ptr, i32 } ${exception}`);
+      this.declare(UNWIND_RESUME_DECLARATION);
+      B.resumeUnwind(exception);
       B.startBlock(ok);
     } else B.line(ret === "void" ? call : `%result = ${call}`);
     // A borrowed result is reachable from the parameters: own it first. A
@@ -7148,11 +7262,26 @@ export class LlEmitter {
         const nullable = this.nullableUnions.of(t);
         return nullable?.repr === "single" ? nullable.arm : null;
       },
-      shapeUnion: (t) => this.nullableUnions.of(t)?.repr === "shapes",
+      pointerTagFn: (t) => {
+        const nullable = this.nullableUnions.of(t);
+        return nullable === null || nullable.repr === "single" ? null : pointerTagFn(nullable);
+      },
       fieldBase: (shapeId) => this.nullableUnions.fieldBase(shapeId),
       classInfo: (className) => {
         const meta = this.classMetaOf(className);
+        const tail = this.inlineTailOf(className);
+        const tailField = tail === undefined ? undefined : meta.def.fields[tail.field]!;
         return {
+          ...(tail !== undefined && tailField !== undefined
+            ? {
+                tail: {
+                  name: tailField.name,
+                  countIndex: meta.compactSlots[tail.field]!,
+                  offset: tail.tailOffset,
+                  elem: { kind: "object" as const, className: tail.elemClass },
+                },
+              }
+            : {}),
           def: meta.def,
           hierarchy: meta.hierarchy,
           pre: meta.pre,
@@ -7160,9 +7289,10 @@ export class LlEmitter {
           runtimeRooted: meta.root.def.runtime === true,
           idInRc: meta.idInRc,
           compact: meta.compact,
-          // Inline scalar-union fields are plain bits: nothing to push.
-          fields: meta.def.fields.flatMap((f) =>
-            this.inlineUnions.classField(className, f.name)
+          // Inline scalar-union fields are plain bits: nothing to push. An
+          // inline array field's elements follow the fixed fields (`tail`).
+          fields: meta.def.fields.flatMap((f, i) =>
+            this.inlineUnions.classField(className, f.name) || tail?.field === i
               ? []
               : [
                   {
@@ -7385,6 +7515,13 @@ export class LlEmitter {
     return emitReadReceiver(this, e);
   }
 
+  /** The inline array field of `className`'s instances, where they keep it
+   * inline (LlClassMeta.inlineTail; classes.ts gives their `new` the count). */
+  inlineTailOf(className: string): LlInlineTail | undefined {
+    const meta = this.classMeta.get(className);
+    return meta !== undefined && meta.compact && this.compactRegion ? meta.inlineTail : undefined;
+  }
+
   /** The frame slot a reassigned stack-object local keeps for its value. */
   stackObjectSlot(localId: string, type: IrType): string {
     let slot = this.stackObjectSlots.get(localId);
@@ -7574,6 +7711,15 @@ export class LlEmitter {
     return canBorrowReceiver(this, e);
   }
 
+  /** A read of an inline array field (ir/inline-arrays.ts). It makes a view
+   * of the elements that the read owns: no field holds an array that could
+   * keep a borrowed copy alive. Known before the class layouts are, from
+   * the module's plan; layouts that keep the field an ordinary array only
+   * lose the borrow. */
+  inlineArrayRead(e: IrExpr): boolean {
+    return e.kind === "fieldGet" && this.inlineArrays.classes.get(e.className)?.field === e.field;
+  }
+
   /** A reference held as one plain pointer that emitReadReceiver can read
    * without owning: class instances, records, arrays, strings, typed arrays
    * and nullable-pointer unions. Tagged union boxes, closures and dynamic
@@ -7607,7 +7753,7 @@ export class LlEmitter {
     if (matchMapRead(e, this.boxedUnionsById)) return false;
     switch (e.kind) {
       case "fieldGet":
-        return this.nullableFieldGet(e) === null;
+        return this.nullableFieldGet(e) === null && !this.inlineArrayRead(e);
       case "recordGet":
         return true;
       // A keyed read whose hits are all the result's own pointer.
@@ -7656,11 +7802,7 @@ export class LlEmitter {
     const nullable = this.nullableUnions.get(e.unionId);
     if (!nullable) return null;
     if (isUnitType(e.value.type)) return e.value.kind === "unitLit" ? "unit" : null;
-    const refArm =
-      nullable.repr === "single"
-        ? e.tag === nullable.refTag
-        : nullable.arms.some((arm) => arm.tag === e.tag);
-    return refArm && isRefCounted(e.value.type) ? "ref" : null;
+    return isPayloadTag(nullable, e.tag) && isRefCounted(e.value.type) ? "ref" : null;
   }
 
   materializeSplitLocal(localId: string): void {

packages/compiler/src/backend/llvm/expr-calls.ts

@@ -54,6 +54,7 @@ import {
   ffiNativeTypeLl,
   ffiNativeParamLl,
   ffiNativeReturnLl,
+  SCR_ARR_FIELD,
 } from "./common.js";
 import { canStackUnion, emitStackUnion } from "./stack-unions.js";
 import { emitStackRecordKeyGet, scalarRecordKeyGet } from "./expr-records.js";
@@ -1121,8 +1122,28 @@ export function emitCallExpr(
         throw new InternalCompilerError(
           `llvm emitter bug: new ${e.className} without a constructor`,
         );
+      // An instance with an inline array field is allocated with room for
+      // the elements of the array its constructor writes the field from
+      // (ir/inline-arrays.ts made that argument a local).
+      const tail = host.inlineTailOf(e.className);
+      let count = "";
+      if (tail !== undefined) {
+        const source = e.args[tail.lengthArg];
+        if (source?.kind !== "varRef")
+          throw new InternalCompilerError(
+            `llvm emitter bug: new ${e.className} reads its inline array from an expression`,
+          );
+        const arr = host.emitReadReceiver(source);
+        const lenp = B.tmp();
+        const len = B.tmp();
+        B.line(
+          `${lenp} = getelementptr inbounds %ScrArr, ptr ${arr.name}, i32 0, i32 ${SCR_ARR_FIELD.len}`,
+        );
+        B.line(`${len} = load i32, ptr ${lenp}`);
+        count = `i32 ${len}`;
+      }
       const o = B.tmp();
-      B.line(`${o} = call ptr @${host.allocSym(mangleClassNew(e.className), e)}()`);
+      B.line(`${o} = call ptr @${host.allocSym(mangleClassNew(e.className), e)}(${count})`);
       host.heapCensus?.countNew((line) => B.line(line), e.className);
       const out = host.own({ name: o, type: e.type });
       emitConstruction(host, e, o, ctor);

packages/compiler/src/backend/llvm/expr-containers.ts

@@ -1,5 +1,6 @@
 import { dynDesc } from "../../ir/analysis.js";
 /* Focused LLVM expression emission extracted from emitter.ts. */
+import { emitInlineArrayLength, inlineFieldTail } from "./inline-arrays.js";
 import { arraySearchHelperName, arraySearchKind } from "./array-search.js";
 import { InternalCompilerError } from "../../errors.js";
 import { isStableReceiverOperand, undefinedArmTag } from "../../ir/analysis.js";
@@ -650,6 +651,14 @@ export function emitArrIntrinsic(
   into?: { box: string; tag: string; payload: string },
 ): LlValue {
   const B = host.B;
+  // The length of an inline array field, read from the instance.
+  if (e.method === "length" && e.receiver.kind === "fieldGet") {
+    const tail = inlineFieldTail(host, e.receiver.className, e.receiver.field);
+    if (tail !== undefined) {
+      const obj = host.emitStableReceiver(e.receiver.obj, e.args);
+      return emitInlineArrayLength(host, obj.name, e.receiver.className, tail, e.type);
+    }
+  }
   // Every array intrinsic borrows its receiver, including mutation and
   // retained-identity results. Scalar reads also admit the narrower proof
   // that a mutable binding survives the remaining operand evaluations.

packages/compiler/src/backend/llvm/expr-primitives.ts

@@ -1,4 +1,10 @@
 /* Focused LLVM expression emission extracted from emitter.ts. */
+import {
+  emitInlineArrayGet,
+  emitInlineArrayState,
+  emitInlineArrayView,
+  inlineFieldTail,
+} from "./inline-arrays.js";
 import { InternalCompilerError } from "../../errors.js";
 import { matchStringSelfConcat, undefinedArmTag } from "../../ir/analysis.js";
 import { isRefCounted, type IrExpr, type IrStrCmpOp, type IrType } from "../../ir/ir.js";
@@ -1181,6 +1187,26 @@ export function emitContainerExpr(
       return out;
     }
     case "arrayGet": {
+      // An element of an inline array field, read from the instance.
+      if (e.arr.kind === "fieldGet" && e.arr.type.kind === "array") {
+        const tail = inlineFieldTail(host, e.arr.className, e.arr.field);
+        if (tail !== undefined) {
+          const obj = host.emitStableReceiver(e.arr.obj, [e.index]);
+          const idx = host.emitExpr(e.index);
+          const value = emitInlineArrayGet(
+            host,
+            obj.name,
+            e.arr.className,
+            tail,
+            idx,
+            e.arr.type.elem,
+            true,
+          );
+          return isRefCounted(e.arr.type.elem)
+            ? host.own({ name: value, type: e.type })
+            : { name: value, type: e.type };
+        }
+      }
       // Element reads run no user code: a projection receiver (an array in
       // a parameter's field) stays rooted while the index preserves edges.
       const arr = host.emitStableReceiver(e.arr, [e.index]);
@@ -1233,6 +1259,16 @@ export function emitContainerExpr(
       return host.own({ name: read.value, type: e.type });
     }
     case "arrayHas": {
+      // An inline array field has no holes: presence is the bounds check.
+      if (e.arr.kind === "fieldGet") {
+        const tail = inlineFieldTail(host, e.arr.className, e.arr.field);
+        if (tail !== undefined) {
+          const obj = host.emitStableReceiver(e.arr.obj, [e.index]);
+          const idx = host.emitExpr(e.index);
+          const name = emitInlineArrayState(host, obj.name, e.arr.className, tail, idx, "has");
+          return { name, type: e.type };
+        }
+      }
       const arr = host.emitStableReceiver(e.arr, [e.index]);
       const idx = host.emitExpr(e.index);
       if (e.arr.type.kind !== "array")
@@ -1252,6 +1288,16 @@ export function emitContainerExpr(
       return { name: t, type: e.type };
     }
     case "arrayState": {
+      // An inline array field has no holes: presence is the bounds check.
+      if (e.arr.kind === "fieldGet") {
+        const tail = inlineFieldTail(host, e.arr.className, e.arr.field);
+        if (tail !== undefined) {
+          const obj = host.emitStableReceiver(e.arr.obj, [e.index]);
+          const idx = host.emitExpr(e.index);
+          const name = emitInlineArrayState(host, obj.name, e.arr.className, tail, idx, "state");
+          return { name, type: e.type };
+        }
+      }
       const arr = host.emitStableReceiver(e.arr, [e.index]);
       const idx = host.emitExpr(e.index);
       if (e.arr.type.kind !== "array")
@@ -1356,6 +1402,14 @@ export function emitRecordExpr(
   switch (e.kind) {
     case "fieldGet": {
       const obj = host.emitReadReceiver(e.obj);
+      // An inline array field read as a value: a frozen array over the
+      // elements (backend/llvm/inline-arrays.ts).
+      const tail = inlineFieldTail(host, e.className, e.field);
+      if (tail !== undefined)
+        return host.own({
+          name: emitInlineArrayView(host, obj.name, e.className, tail),
+          type: e.type,
+        });
       const { ptr, type, inline } = host.classFieldPtr(
         obj.name,
         e.className,

packages/compiler/src/backend/llvm/expr-records.ts

@@ -74,13 +74,13 @@ export function emitUnionRetype(
   const to = host.nullableUnions.get(e.unionId);
   if (from && to) {
     // Equal encodings make the pointer itself the target value. A shape
-    // union derives its record arms from the record, so only the unit
-    // encodings must agree.
+    // union derives its record arms from the record and a class union its
+    // class arms from the class id, so only the unit encodings must agree.
     const same =
       from.repr === "single" && to.repr === "single"
         ? from.refTag === to.refTag && from.unitTag === to.unitTag && from.nullTag === to.nullTag
-        : from.repr === "shapes" &&
-          to.repr === "shapes" &&
+        : from.repr !== "single" &&
+          from.repr === to.repr &&
           from.unitTag === to.unitTag &&
           from.nullTag === to.nullTag;
     if (same) return host.own({ name: value.name, type: e.type });

packages/compiler/src/backend/llvm/expr-stream-bridges.ts

@@ -35,6 +35,7 @@ import {
 import { SCR_ARR_FIELD, llvmCommentText } from "./common.js";
 import {
   FN_ATTRS,
+  arrOps,
   elemAccess,
   llFieldType,
   loadStructField,
@@ -44,6 +45,7 @@ import {
   type ShapeHost,
 } from "./shapes.js";
 import type { LlvmEmitterContext, LlStreamTypedRefAdapter } from "./expr-context.js";
+import { inlineFieldTail } from "./inline-arrays.js";
 import type { NullableRefField } from "./nullable-fields.js";
 import { emitUnionPeek, emitUnionTag } from "./union-repr.js";
 import { inlineUnionCopyLines } from "./inline-unions.js";
@@ -1703,6 +1705,27 @@ function classFieldDynValue(
   t: Extract<IrType, { kind: "object" }>,
   field: LlClassMeta["def"]["fields"][number],
 ): { boxed: string; nullable: NullableRefField | null; fieldValue: string } {
+  // An inline array field reads as a frozen array over the instance's
+  // elements (inline-arrays.ts), which the box keeps.
+  const tail = inlineFieldTail(host, t.className, field.name);
+  if (tail !== undefined) {
+    const countp = B.tmp(),
+      count = B.tmp(),
+      at = B.tmp(),
+      view = B.tmp();
+    B.line(
+      `${countp} = getelementptr inbounds %${classStructSym(t.className)}, ptr %p, i64 0, i32 ${meta.compactSlots[tail.field]!}`,
+    );
+    B.line(`${count} = load i32, ptr ${countp}`);
+    B.line(`${at} = getelementptr inbounds i8, ptr %p, i64 ${tail.tailOffset}`);
+    const ops = arrOps(host.shapeHost, { kind: "object", className: tail.elemClass });
+    host.declare(`declare ptr @scr_arr_view_cref(ptr, ptr, i32)`);
+    host.declare(`declare void @scr_arr_release(ptr)`);
+    B.line(`${view} = call ptr @scr_arr_view_cref(ptr ${ops}, ptr ${at}, i32 ${count})`);
+    const boxed = host.streamTypedRefBoxValue(B, field.type, view);
+    B.line(`call void @scr_arr_release(ptr ${view})`);
+    return { boxed, nullable: null, fieldValue: view };
+  }
   const { index, compressed } = classFieldIndex(meta, field.name, "compressed");
   const fieldPtr = B.tmp();
   let fieldValue = B.tmp();

packages/compiler/src/backend/llvm/inline-arrays.ts

@@ -0,0 +1,257 @@
+/* Inline array fields in the LLVM backend (ir/inline-arrays.ts plans them).
+ *
+ * A compact class with such a field (LlClassMeta.inlineTail) keeps the
+ * field's element count as an i32 in the field's slot and the elements as
+ * 32-bit region references after its fixed fields, at the subtree's tail
+ * offset. `new` reads the count from the constructor argument the field is
+ * written from and allocates the instance with room for them; the
+ * constructor's write copies the argument's elements there
+ * (scr_arr_copy_cref_tail). Reads that index the field, read its length or
+ * iterate it load the elements directly; any other read gives a frozen array
+ * over them (scr_arr_view_cref). */
+
+import { isRefCounted, type IrModule, type IrType } from "../../ir/ir.js";
+import { mangleClassStruct } from "../mangle.js";
+import { compactRefLoadLines, type LlInlineTail } from "./classes.js";
+import { f64Lit } from "./common.js";
+import type { LlvmTargetOptions } from "./emitter.js";
+import type { LlValue, LlvmEmitterContext } from "./expr-context.js";
+import { arrOps } from "./shapes.js";
+
+/** Whether `mod` emitted with `options` can keep inline array fields inline:
+ * an executable for a 64-bit target whose compact instances live in the low
+ * region (32-bit references), without lifetime regions (which keep their
+ * own class layouts), and with fields to keep. */
+export function inlineArraysApply(mod: IrModule, options: LlvmTargetOptions): boolean {
+  return (
+    (mod.contracts?.inlineArrays?.length ?? 0) > 0 &&
+    options.pointerBits !== 32 &&
+    options.lowRegion === true &&
+    mod.lib === undefined &&
+    mod.lifetimeRegions !== true
+  );
+}
+
+/** Whether instances of `className` are ProgramLifetime instances: the
+ * root of its hierarchy is declared `implements ProgramLifetime`. */
+export function programLifetimeClassOf(mod: IrModule): (className: string) => boolean {
+  const roots = new Set(mod.contracts?.programLifetime ?? []);
+  const bases = new Map<string, string | undefined>();
+  for (const cls of mod.classes ?? []) bases.set(cls.name, cls.base);
+  return (className) => {
+    let name: string | undefined = className;
+    let root = className;
+    while (name !== undefined && bases.has(name)) {
+      root = name;
+      name = bases.get(name);
+    }
+    return bases.has(className) && roots.has(root);
+  };
+}
+
+/** The field of `className` whose elements its instances keep inline, when
+ * `field` is it. */
+export function inlineFieldTail(
+  host: LlvmEmitterContext,
+  className: string,
+  field: string,
+): LlInlineTail | undefined {
+  const tail = host.inlineTailOf(className);
+  if (tail === undefined) return undefined;
+  return host.classMeta.get(className)!.def.fields[tail.field]!.name === field ? tail : undefined;
+}
+
+/** Loads the element count of the inline array field of `obj` (an instance
+ * of `className`) as an i32; returns the value's name. */
+export function emitInlineCount(
+  host: LlvmEmitterContext,
+  obj: string,
+  className: string,
+  tail: LlInlineTail,
+): string {
+  const B = host.B;
+  const meta = host.classMeta.get(className)!;
+  const countp = B.tmp();
+  const count = B.tmp();
+  B.line(
+    `${countp} = getelementptr inbounds %${mangleClassStruct(className)}, ptr ${obj}, i64 0, i32 ${meta.compactSlots[tail.field]!}`,
+  );
+  B.line(`${count} = load i32, ptr ${countp}`);
+  return count;
+}
+
+/** The address of the first element of the inline array field of `obj`. */
+export function emitInlineElements(
+  host: LlvmEmitterContext,
+  obj: string,
+  tail: LlInlineTail,
+): string {
+  const B = host.B;
+  const at = B.tmp();
+  B.line(`${at} = getelementptr inbounds i8, ptr ${obj}, i64 ${tail.tailOffset}`);
+  return at;
+}
+
+/** The constructor's write of the inline array field of `obj`: copies the
+ * elements of `value` (an array) into the instance, which `new` allocated
+ * for that array's length. */
+export function emitInlineArrayInit(
+  host: LlvmEmitterContext,
+  obj: string,
+  className: string,
+  tail: LlInlineTail,
+  value: string,
+): void {
+  const count = emitInlineCount(host, obj, className, tail);
+  const at = emitInlineElements(host, obj, tail);
+  host.declare(`declare void @scr_arr_copy_cref_tail(ptr, ptr, i32)`);
+  host.B.line(`call void @scr_arr_copy_cref_tail(ptr ${value}, ptr ${at}, i32 ${count})`);
+}
+
+/** The inline array field of `obj` as an array value: a frozen array over
+ * the elements (scr_arr_view_cref). Returns the owned (+1) array's name. */
+export function emitInlineArrayView(
+  host: LlvmEmitterContext,
+  obj: string,
+  className: string,
+  tail: LlInlineTail,
+): string {
+  const count = emitInlineCount(host, obj, className, tail);
+  const at = emitInlineElements(host, obj, tail);
+  const ops = arrOps(host.shapeHost, { kind: "object", className: tail.elemClass });
+  host.declare(`declare ptr @scr_arr_view_cref(ptr, ptr, i32)`);
+  const view = host.B.tmp();
+  host.B.line(`${view} = call ptr @scr_arr_view_cref(ptr ${ops}, ptr ${at}, i32 ${count})`);
+  return view;
+}
+
+/** The index as an i64 slot, and whether the number is an integer below
+ * `count` (an i32). */
+function emitInlineIndex(
+  host: LlvmEmitterContext,
+  count: string,
+  index: LlValue,
+): { slot: string; within: string } {
+  const B = host.B;
+  const bound = B.tmp();
+  B.line(`${bound} = zext i32 ${count} to i64`);
+  const integer = index.integer;
+  if (integer !== undefined) {
+    let slot = integer.name;
+    if (integer.type === "i32") {
+      slot = B.tmp();
+      B.line(`${slot} = ${integer.signed ? "sext" : "zext"} i32 ${integer.name} to i64`);
+    }
+    const within = B.tmp();
+    B.line(`${within} = icmp ult i64 ${slot}, ${bound}`);
+    return { slot, within };
+  }
+  host.declare(`declare i64 @llvm.fptosi.sat.i64.f64(double)`);
+  const truncated = B.tmp();
+  const back = B.tmp();
+  const exact = B.tmp();
+  const below = B.tmp();
+  const within = B.tmp();
+  B.line(`${truncated} = call i64 @llvm.fptosi.sat.i64.f64(double ${index.name})`);
+  B.line(`${back} = sitofp i64 ${truncated} to double`);
+  B.line(`${exact} = fcmp oeq double ${back}, ${index.name}`);
+  B.line(`${below} = icmp ult i64 ${truncated}, ${bound}`);
+  B.line(`${within} = and i1 ${exact}, ${below}`);
+  return { slot: truncated, within };
+}
+
+/** `index in obj.field` (`has`, an i1) or the slot state (`state`, a double:
+ * HOLE=0, VALUE=1) for an inline array field. Its elements are instances,
+ * never holes or undefined, so an index is present exactly when it is an
+ * integer within the count. */
+export function emitInlineArrayState(
+  host: LlvmEmitterContext,
+  obj: string,
+  className: string,
+  tail: LlInlineTail,
+  index: LlValue,
+  kind: "has" | "state",
+): string {
+  const count = emitInlineCount(host, obj, className, tail);
+  const { within } = emitInlineIndex(host, count, index);
+  if (kind === "has") return within;
+  const state = host.B.tmp();
+  host.B.line(`${state} = select i1 ${within}, double ${f64Lit(1)}, double ${f64Lit(0)}`);
+  return state;
+}
+
+/** `obj.field[index]` for an inline array field: the element at an index
+ * within the count, loaded from the instance. Any other index takes the
+ * runtime getter over a view, which reports it like a read of an ordinary
+ * array. With `retain`, the result is an owned reference when the element
+ * type is reference counted (like emitDenseArrayRead); without, borrowed
+ * from the instance. */
+export function emitInlineArrayGet(
+  host: LlvmEmitterContext,
+  obj: string,
+  className: string,
+  tail: LlInlineTail,
+  index: LlValue,
+  element: IrType,
+  retain: boolean,
+): string {
+  const B = host.B;
+  const count = emitInlineCount(host, obj, className, tail);
+  const at = emitInlineElements(host, obj, tail);
+  const { slot, within } = emitInlineIndex(host, count, index);
+  const hit = B.newLabel("inl.get.hit");
+  const miss = B.newLabel("inl.get.miss");
+  const join = B.newLabel("inl.get.join");
+  B.condBr(within, hit, miss);
+  B.startBlock(hit);
+  const p = B.tmp();
+  const ref = B.tmp();
+  const raw = B.tmp();
+  B.line(`${p} = getelementptr inbounds i32, ptr ${at}, i64 ${slot}`);
+  B.line(`${ref} = load i32, ptr ${p}`);
+  for (const line of compactRefLoadLines(ref, raw, `${raw}.r`)) B.line(line.trimStart());
+  const value = retain && isRefCounted(element) ? host.retainFrame(raw, element) : raw;
+  const hitEnd = B.currentLabel();
+  B.br(join);
+  B.startBlock(miss);
+  // Outside the count: the getter of an ordinary array over the elements
+  // decides, as for any array (an out-of-bounds read traps).
+  const ops = arrOps(host.shapeHost, { kind: "object", className: tail.elemClass });
+  const getter = retain ? "scr_arr_get_ref" : "scr_arr_borrow_ref";
+  host.declare(`declare ptr @scr_arr_view_cref(ptr, ptr, i32)`);
+  host.declare(`declare ptr @${getter}(ptr, double)`);
+  host.declare(`declare void @scr_arr_release(ptr)`);
+  const view = B.tmp();
+  const slow = B.tmp();
+  B.line(`${view} = call ptr @scr_arr_view_cref(ptr ${ops}, ptr ${at}, i32 ${count})`);
+  B.line(`${slow} = call ptr @${getter}(ptr ${view}, double ${indexDouble(host, index)}) cold`);
+  B.line(`call void @scr_arr_release(ptr ${view})`);
+  const missEnd = B.currentLabel();
+  B.br(join);
+  B.startBlock(join);
+  const result = B.tmp();
+  B.line(`${result} = phi ptr [ ${value}, %${hitEnd} ], [ ${slow}, %${missEnd} ]`);
+  return result;
+}
+
+/** The index as a double (an integer snapshot's own double is its name). */
+function indexDouble(host: LlvmEmitterContext, index: LlValue): string {
+  if (index.type.kind === "f64") return index.name;
+  const d = host.B.tmp();
+  host.B.line(`${d} = sitofp i32 ${index.name} to double`);
+  return d;
+}
+
+/** `obj.field.length` for an inline array field. */
+export function emitInlineArrayLength(
+  host: LlvmEmitterContext,
+  obj: string,
+  className: string,
+  tail: LlInlineTail,
+  type: IrType,
+): LlValue {
+  const count = emitInlineCount(host, obj, className, tail);
+  const length = host.B.tmp();
+  host.B.line(`${length} = uitofp i32 ${count} to double`);
+  return { name: length, type, uint32: count };
+}

packages/compiler/src/backend/llvm/integer-clones.test.ts

@@ -341,3 +341,164 @@ test("borrowed number arrays: length and element reads qualify", () => {
   const withOther: IrModule = { ...mod, functions: [sum, other] };
   expect(findIntegerClones(withOther, () => true).members.has("other")).toBe(false);
 });
+
+// A tree record whose children are `Tree | null` (tag 0 null, tag 1 the
+// record), and count(t) = t.left === null || t.right === null ? 1 :
+// 1 + count(t.left!) + count(t.right!), the binary-trees check.
+const TREE: IrType = { kind: "record", shapeId: "rTree" };
+const CHILD: IrType = { kind: "union", unionId: "uChild" };
+const field = (obj: IrExpr, name: string, type: IrType): IrExpr => ({
+  kind: "recordGet",
+  obj,
+  shapeId: "rTree",
+  field: name,
+  type,
+  loc,
+});
+const isNull = (value: IrExpr): IrExpr => ({
+  kind: "unionIsTag",
+  unionId: "uChild",
+  tag: 0,
+  negated: false,
+  value,
+  type: BOOL,
+  loc,
+});
+const child = (name: string): IrExpr => ({
+  kind: "unionNarrow",
+  unionId: "uChild",
+  tag: 1,
+  value: field(ref("t", TREE), name, CHILD),
+  type: TREE,
+  loc,
+});
+const count = fn(
+  "count",
+  ["t"],
+  [
+    ret(
+      ternary(
+        {
+          kind: "logical",
+          op: "||",
+          left: isNull(field(ref("t", TREE), "left", CHILD)),
+          right: isNull(field(ref("t", TREE), "right", CHILD)),
+          type: BOOL,
+          loc,
+        },
+        num(1),
+        bin("+", bin("+", num(1), call("count", [child("left")])), call("count", [child("right")])),
+      ),
+    ),
+  ],
+  F64,
+  [TREE],
+);
+// leftmost(t) = t.left === null ? 0 : leftmost(t.left!), a plain tail call.
+const leftmost = fn(
+  "leftmost",
+  ["t"],
+  [
+    ret(
+      ternary(
+        isNull(field(ref("t", TREE), "left", CHILD)),
+        num(0),
+        call("leftmost", [child("left")]),
+      ),
+    ),
+  ],
+  F64,
+  [TREE],
+);
+function treeModule(workers: boolean): IrModule {
+  return {
+    irVersion: 17,
+    sourceFile: loc.file,
+    entry: "main",
+    records: [
+      {
+        id: "rTree",
+        fields: [
+          { name: "left", type: CHILD },
+          { name: "right", type: CHILD },
+          { name: "size", type: F64 },
+        ],
+      },
+    ],
+    unions: [{ id: "uChild", arms: [{ kind: "nullT" }, TREE] }],
+    ...(workers ? { workers: true } : {}),
+    functions: [
+      {
+        ...fn(
+          "main",
+          ["t"],
+          [stmt(call("count", [ref("t", TREE)])), stmt(call("leftmost", [ref("t", TREE)]))],
+          VOID,
+          [TREE],
+        ),
+        params: [],
+      },
+      count,
+      leftmost,
+    ],
+  };
+}
+
+test("borrowed records: field reads, null tests and narrows qualify", () => {
+  const mod = treeModule(false);
+  const refs = {
+    reference: (t: IrType) =>
+      (t.kind === "record" && t.shapeId === "rTree") ||
+      (t.kind === "union" && t.unionId === "uChild"),
+    arm: (unionId: string, tag: number) =>
+      unionId === "uChild" ? (tag === 0 ? ("null" as const) : ("record" as const)) : null,
+    field: (shapeId: string, name: string) =>
+      shapeId === "rTree" && name !== "size"
+        ? { struct: "sc_rs_rTree", index: name === "left" ? 1 : 2, type: CHILD }
+        : null,
+  };
+  // Records need the layout oracle and a borrowed parameter.
+  expect(findIntegerClones(mod, () => true).members.has("count")).toBe(false);
+  expect(findIntegerClones(mod, () => false, refs).members.has("count")).toBe(false);
+  expect(findIntegerClones(mod, () => true, refs).dispatched.has("count")).toBe(true);
+  // A field the oracle does not lay out in place disqualifies the reader.
+  const sized = fn(
+    "sized",
+    ["t"],
+    [ret(bin("+", field(ref("t", TREE), "size", F64), call("sized", [child("left")])))],
+    F64,
+    [TREE],
+  );
+  const withSized: IrModule = { ...mod, functions: [...mod.functions, sized] };
+  expect(findIntegerClones(withSized, () => true, refs).members.has("sized")).toBe(false);
+});
+
+test("record clones read fields in place; worker executables poll and use the thread's guard", () => {
+  for (const workers of [false, true]) {
+    const mod = treeModule(workers);
+    expect(validateModule(mod)).toEqual([]);
+    const llvm = emitLlvmModule(mod, { targetTriple: "x86_64-unknown-linux-gnu" });
+    const clone = define(llvm, "sc_f_count.int");
+    expect(clone).toContain("define internal { i32, i1 } @sc_f_count.int(ptr %a0)");
+    expect(clone).toMatch(/getelementptr inbounds %sc_rs_rTree, ptr %t\d+, i64 0, i32 1/);
+    // A NULL child bails before the call (the ordinary body reports it).
+    expect(clone).toMatch(/icmp ne ptr %t\d+, null/);
+    // count(t.right) is the accumulating loop's next iteration.
+    expect(clone.match(/= call \{ i32, i1 \} @sc_f_count\.int\(/g)!.length).toBeGreaterThan(0);
+    expect(clone).toMatch(/br label %tail\d+/);
+    expect(clone).not.toContain("retain");
+    const plain = define(llvm, "sc_f_leftmost.int");
+    if (workers) {
+      // One poll per call; the accumulating loop polls nowhere else, a loop
+      // over plain tail calls every POLL_INTERVAL iterations.
+      expect(clone.match(/load atomic i32, ptr @scr_exc_flag monotonic/g)).toHaveLength(1);
+      expect(clone).not.toContain("poll countdown");
+      expect(plain).toContain("poll countdown");
+      expect(plain.match(/load atomic i32, ptr @scr_exc_flag monotonic/g)).toHaveLength(2);
+      expect(llvm).toMatch(/@scr_stack_guard = external thread_local/);
+    } else {
+      expect(clone).not.toContain("@scr_exc_flag");
+      expect(plain).not.toContain("poll countdown");
+    }
+  }
+});

packages/compiler/src/backend/llvm/integer-clones.ts

@@ -83,24 +83,52 @@ const BAIL = `${CLONE_RESULT} { i32 poison, i1 true }`;
 /** The stack one replaced recursion level is charged, in bytes. Ordinary
  * frames of these functions are about that size. */
 const VIRTUAL_FRAME = 32;
+/** Iterations between the polls of a loop over plain tail calls in worker
+ * executables (emitLoopPoll). */
+const POLL_INTERVAL = 64;
 
-/** The single-threaded stack guard, declared as the emitter's ordinary
- * stack checks declare it (clones only exist without workers). */
-export const STACK_GUARD_DECL = "@scr_stack_guard = external global ptr";
+/** The stack guard and the termination word, declared as the emitter's
+ * ordinary stack checks and polls declare them: process-wide in programs
+ * without workers, per thread in ELF worker executables (the only worker
+ * programs with clones). */
+export function stackGuardDecl(workers: boolean): string {
+  return `@scr_stack_guard = external ${workers ? "thread_local " : ""}global ptr`;
+}
+const EXC_FLAG_DECL = "@scr_exc_flag = external thread_local(localexec) global i32, align 4";
 
 export const INTEGER_CLONE_SUFFIX = ".int";
 export const INTEGER_DISPATCH_SUFFIX = ".spec";
 
 type Scalar = "num" | "bool";
-/** A value kind: an int32 number, a boolean, or a borrowed number array. */
-type Kind = Scalar | "arr";
+/** A value kind: an int32 number, a boolean, a borrowed number array, or a
+ * reference into a borrowed record graph. */
+type Kind = Scalar | "arr" | "ref";
+
+/** The records a clone may read (from the emitter's layouts). A reference
+ * is a plain pointer: a record, or a union of one record arm and `null`
+ * (NULL). A clone only reads its records' fields, and it has no effect, so
+ * every record reachable from a borrowed parameter stays alive and
+ * unchanged while it runs: it reads them without retaining them. */
+export interface CloneReferences {
+  /** Whether values of `type` are such pointers. */
+  reference(type: IrType): boolean;
+  /** For a tag of a reference union: whether it names the record arm
+   * ("record") or the `null` arm ("null"). */
+  arm(unionId: string, tag: number): "record" | "null" | null;
+  /** Where record shape `shapeId` keeps `field` (its struct and element
+   * index), or null when the field is not a slot a clone reads in place:
+   * a number (a double), a boolean (an i8) or a reference. */
+  field(shapeId: string, field: string): { struct: string; index: number; type: IrType } | null;
+}
 
 function scalarKind(type: IrType): Scalar | null {
   return type.kind === "f64" ? "num" : type.kind === "bool" ? "bool" : null;
 }
 
-function valueKind(type: IrType): Kind | null {
-  return type.kind === "array" && type.elem.kind === "f64" ? "arr" : scalarKind(type);
+function valueKind(type: IrType, refs: CloneReferences | undefined): Kind | null {
+  if (type.kind === "array" && type.elem.kind === "f64") return "arr";
+  if (refs?.reference(type) === true) return "ref";
+  return scalarKind(type);
 }
 
 /** The LLVM type of a value of `kind` inside a clone. */
@@ -119,20 +147,26 @@ function isInt32Literal(value: number): boolean {
 }
 
 /** The call targets of `fn` when its body has the qualifying shape. */
-function cloneShape(fn: IrFunction, borrowed?: BorrowedParameter): Set<string> | null {
+function cloneShape(
+  fn: IrFunction,
+  borrowed?: BorrowedParameter,
+  refs?: CloneReferences,
+): Set<string> | null {
   if (fn.async || fn.generator !== undefined || fn.captures !== undefined) return null;
   if (fn.classCaptures !== undefined && fn.classCaptures.length > 0) return null;
   const result = scalarKind(fn.returnType);
   if (result === null) return null;
   const locals = new Map<string, IrLocal>(fn.locals.map((l) => [l.id, l]));
-  // Array parameters are only read: never assigned, and their locals are
-  // the parameters themselves.
+  // Array and reference parameters are only read: never assigned, and their
+  // locals are the parameters themselves.
   const arrays = new Set<string>();
-  for (const [index, p] of fn.params.entries())
-    if (valueKind(p.type) === "arr") {
-      if (borrowed?.(fn.name, index) !== true) return null;
-      arrays.add(p.localId);
-    }
+  const references = new Set<string>();
+  for (const [index, p] of fn.params.entries()) {
+    const kind = valueKind(p.type, refs);
+    if (kind !== "arr" && kind !== "ref") continue;
+    if (borrowed?.(fn.name, index) !== true) return null;
+    (kind === "arr" ? arrays : references).add(p.localId);
+  }
   const localKind = (id: string): Scalar | null => {
     const local = locals.get(id);
     if (!local || local.boxed || local.tdz) return null;
@@ -140,12 +174,38 @@ function cloneShape(fn: IrFunction, borrowed?: BorrowedParameter): Set<string> |
   };
   const arrayRef = (e: IrExpr): boolean => e.kind === "varRef" && arrays.has(e.localId);
   for (const p of fn.params)
-    if (!arrays.has(p.localId) && (localKind(p.localId) === null || scalarKind(p.type) === null))
+    if (
+      !arrays.has(p.localId) &&
+      !references.has(p.localId) &&
+      (localKind(p.localId) === null || scalarKind(p.type) === null)
+    )
       return null;
+  /** A field the clone reads in place, of a record a reference reads. */
+  const readsField = (e: Extract<IrExpr, { kind: "recordGet" }>): boolean =>
+    reference(e.obj) && refs?.field(e.shapeId, e.field) != null;
+  /** A reference parameter, a reference field of a record one reaches, or
+   * a narrow of a reference union to its record arm. */
+  const reference = (e: IrExpr): boolean => {
+    if (refs?.reference(e.type) !== true) return false;
+    switch (e.kind) {
+      case "varRef":
+        return references.has(e.localId);
+      case "recordGet":
+        return readsField(e);
+      case "unionNarrow":
+        return refs.arm(e.unionId, e.tag) === "record" && reference(e.value);
+      default:
+        return false;
+    }
+  };
   const callees = new Set<string>();
   const expr = (e: IrExpr, want: Scalar): boolean => {
     if (scalarKind(e.type) !== want) return false;
     switch (e.kind) {
+      case "recordGet":
+        return readsField(e);
+      case "unionIsTag":
+        return want === "bool" && refs?.arm(e.unionId, e.tag) != null && reference(e.value);
       case "numLit":
         return isInt32Literal(e.value);
       case "boolLit":
@@ -202,8 +262,10 @@ function cloneShape(fn: IrFunction, borrowed?: BorrowedParameter): Set<string> |
       case "call":
         callees.add(e.callee);
         return e.args.every((a) => {
-          const kind = valueKind(a.type);
-          return kind === "arr" ? arrayRef(a) : kind !== null && expr(a, kind);
+          const kind = valueKind(a.type, refs);
+          if (kind === "arr") return arrayRef(a);
+          if (kind === "ref") return reference(a);
+          return kind !== null && expr(a, kind);
         });
       default:
         return false;
@@ -238,7 +300,11 @@ function cloneShape(fn: IrFunction, borrowed?: BorrowedParameter): Set<string> |
 }
 
 /** Every call in `fn`'s body has arguments of its callee's parameter kinds. */
-function callsMatch(fn: IrFunction, byName: ReadonlyMap<string, IrFunction>): boolean {
+function callsMatch(
+  fn: IrFunction,
+  byName: ReadonlyMap<string, IrFunction>,
+  refs: CloneReferences | undefined,
+): boolean {
   let ok = true;
   const visit = (e: IrExpr): void => {
     switch (e.kind) {
@@ -248,7 +314,7 @@ function callsMatch(fn: IrFunction, byName: ReadonlyMap<string, IrFunction>): bo
           !callee ||
           callee.params.length !== e.args.length ||
           scalarKind(callee.returnType) !== scalarKind(e.type) ||
-          callee.params.some((p, i) => valueKind(p.type) !== valueKind(e.args[i]!.type))
+          callee.params.some((p, i) => valueKind(p.type, refs) !== valueKind(e.args[i]!.type, refs))
         )
           ok = false;
         e.args.forEach(visit);
@@ -295,12 +361,16 @@ function callsMatch(fn: IrFunction, byName: ReadonlyMap<string, IrFunction>): bo
   return ok;
 }
 
-export function findIntegerClones(mod: IrModule, borrowed?: BorrowedParameter): IntegerClones {
+export function findIntegerClones(
+  mod: IrModule,
+  borrowed?: BorrowedParameter,
+  refs?: CloneReferences,
+): IntegerClones {
   const byName = new Map(mod.functions.map((fn) => [fn.name, fn]));
   const edges = new Map<string, Set<string>>();
   for (const fn of mod.functions) {
-    const callees = cloneShape(fn, borrowed);
-    if (callees && callsMatch(fn, byName)) edges.set(fn.name, callees);
+    const callees = cloneShape(fn, borrowed, refs);
+    if (callees && callsMatch(fn, byName, refs)) edges.set(fn.name, callees);
   }
   // Drop functions that call a non-member until none does.
   for (let changed = true; changed;) {
@@ -414,6 +484,11 @@ export interface IntegerCloneHost {
   declare(decl: string): void;
   /** Lines setting the i1 `dest` to "an exception is pending". */
   pendingTestLines(dest: string): string[];
+  /** The records clones read (the same oracle findIntegerClones used). */
+  readonly cloneReferences: CloneReferences | undefined;
+  /** An ELF worker executable: the stack guard is per thread, and clones
+   * poll the thread's pending word, which a stop request also sets. */
+  readonly executableTls: boolean;
 }
 
 interface NumValue {
@@ -590,7 +665,9 @@ class CloneEmitter {
     };
     const params = new Set(fn.params.map((p) => p.localId));
     for (const local of fn.locals) {
-      const kind = params.has(local.id) ? valueKind(local.type) : scalarKind(local.type);
+      const kind = params.has(local.id)
+        ? valueKind(local.type, this.host.cloneReferences)
+        : scalarKind(local.type);
       if (kind === null || local.boxed) continue;
       const slot = `%l${this.slotCount++}`;
       B.entryAllocas.push(`${slot} = alloca ${kindType(kind)} ; ${local.name}`);
@@ -602,6 +679,7 @@ class CloneEmitter {
     });
     this.frame = frame;
     if (inline === null && this.stackCheck) this.emitStackCheck();
+    if (inline === null) this.emitPoll();
     const tail = this.tailShape();
     if (!tail.loop) {
       this.assumeParamRanges();
@@ -613,12 +691,57 @@ class CloneEmitter {
       B.entryAllocas.push(`${frame.accSlot} = alloca i32 ; accumulator`);
       B.line(`store i32 0, ptr ${frame.accSlot}`);
     }
+    const budget = tail.plain && this.host.executableTls ? `%l${this.slotCount++}` : null;
+    if (budget !== null) {
+      B.entryAllocas.push(`${budget} = alloca i32 ; poll countdown`);
+      B.line(`store i32 0, ptr ${budget}`);
+    }
     frame.loopLabel = B.newLabel("tail");
     B.br(frame.loopLabel);
     B.startBlock(frame.loopLabel);
+    if (budget !== null) this.emitLoopPoll(budget);
     this.assumeParamRanges();
   }
 
+  /** In an ELF worker executable, bails when the thread's pending word is
+   * set: an exception is pending, or the worker must stop. The dispatcher
+   * then forwards it. Ordinary bodies observe a stop through their pending
+   * tests after calls and in loops, which clones do not run, so a clone
+   * polls when it is called and every POLL_INTERVAL iterations of a loop
+   * over plain tail calls, which can run forever. An accumulating loop runs
+   * at most until its virtual stack pointer passes the guard. The load is
+   * ordered, so LLVM keeps it in loops it forms out of tail calls. */
+  private emitPoll(): void {
+    if (!this.host.executableTls) return;
+    const { B } = this;
+    this.host.declare(EXC_FLAG_DECL);
+    const word = B.tmp();
+    const clear = B.tmp();
+    B.line(`${word} = load atomic i32, ptr @scr_exc_flag monotonic, align 4`);
+    B.line(`${clear} = icmp eq i32 ${word}, 0`);
+    this.require(clear);
+  }
+
+  /** The poll of a loop iteration, every POLL_INTERVAL iterations. */
+  private emitLoopPoll(budget: string): void {
+    const { B } = this;
+    const left = B.tmp();
+    const rest = B.tmp();
+    const due = B.tmp();
+    B.line(`${left} = load i32, ptr ${budget}`);
+    B.line(`${rest} = sub i32 ${left}, 1`);
+    B.line(`store i32 ${rest}, ptr ${budget}`);
+    B.line(`${due} = icmp eq i32 ${left}, 0`);
+    const poll = B.newLabel("poll");
+    const next = B.newLabel("poll.next");
+    B.condBr(due, poll, next);
+    B.startBlock(poll);
+    B.line(`store i32 ${POLL_INTERVAL - 1}, ptr ${budget}`);
+    this.emitPoll();
+    B.br(next);
+    B.startBlock(next);
+  }
+
   /** Starts the virtual stack pointer of an activation with an accumulating
    * loop. */
   private openVirtualStack(frame: Frame, parent: Frame | null): void {
@@ -673,7 +796,9 @@ class CloneEmitter {
 
   emit(): string {
     const { B, fn } = this;
-    const params = fn.params.map((p, i) => `${kindType(valueKind(p.type)!)} %a${i}`);
+    const params = fn.params.map(
+      (p, i) => `${kindType(valueKind(p.type, this.host.cloneReferences)!)} %a${i}`,
+    );
     this.openFrame(
       fn.params.map((_, i) => `%a${i}`),
       null,
@@ -720,16 +845,17 @@ class CloneEmitter {
     return this.inlineSelf && this.frame.inline === null && this.isSelfCall(e);
   }
 
-  /** Whether the body has self tail calls, and whether one accumulates. */
-  private tailShape(): { loop: boolean; accumulates: boolean } {
-    let loop = false;
+  /** Whether the body has self tail calls, whether one accumulates, and
+   * whether one is a plain tail call. */
+  private tailShape(): { loop: boolean; accumulates: boolean; plain: boolean } {
+    let plain = false;
     let accumulates = false;
     const ret = (e: IrExpr): void => {
       if (e.kind === "ternary") {
         ret(e.then);
         ret(e.else_);
-      } else if (this.isSelfCall(e)) loop = true;
-      else if (this.accumulatingCall(e) !== null) loop = accumulates = true;
+      } else if (this.isSelfCall(e)) plain = true;
+      else if (this.accumulatingCall(e) !== null) accumulates = true;
     };
     const stmts = (list: readonly IrStmt[]): void => {
       for (const s of list) {
@@ -741,7 +867,7 @@ class CloneEmitter {
       }
     };
     stmts(this.fn.body);
-    return { loop, accumulates };
+    return { loop: plain || accumulates, accumulates, plain };
   }
 
   private isSelfCall(e: IrExpr): e is Extract<IrExpr, { kind: "call" }> {
@@ -779,7 +905,7 @@ class CloneEmitter {
     host.declare(`declare ptr @llvm.stacksave.p0()`);
     host.declare(`declare i1 @llvm.expect.i1(i1, i1)`);
     host.declare(`declare void @scr_stack_overflow()`);
-    host.declare(STACK_GUARD_DECL);
+    host.declare(stackGuardDecl(host.executableTls));
     const sp = B.tmp();
     const guard = B.tmp();
     const low = B.tmp();
@@ -809,7 +935,7 @@ class CloneEmitter {
    * continuing at the overflow block past the guard. */
   private consumeFrame(): void {
     const { B } = this;
-    this.host.declare(STACK_GUARD_DECL);
+    this.host.declare(stackGuardDecl(this.host.executableTls));
     const depth = B.tmp();
     const next = B.tmp();
     const guardPtr = B.tmp();
@@ -991,7 +1117,9 @@ class CloneEmitter {
     exprs: readonly IrExpr[],
     marker = "",
   ): string {
-    const list = args.map((a, i) => `${kindType(valueKind(exprs[i]!.type)!)} ${a}`);
+    const list = args.map(
+      (a, i) => `${kindType(valueKind(exprs[i]!.type, this.host.cloneReferences)!)} ${a}`,
+    );
     const result = this.B.tmp();
     this.B.line(
       `${result} = ${marker}call ${CLONE_RESULT} @${mangleFunction(callee)}${INTEGER_CLONE_SUFFIX}(${list.join(", ")})`,
@@ -1007,6 +1135,43 @@ class CloneEmitter {
     return value;
   }
 
+  /** The record pointer of a reference expression (cloneShape's
+   * `reference`): a parameter, a field read, or a narrow to the record arm,
+   * which bails on NULL so the ordinary body reports it. */
+  private emitRef(e: IrExpr): string {
+    const { B } = this;
+    switch (e.kind) {
+      case "varRef": {
+        const value = B.tmp();
+        B.line(`${value} = load ptr, ptr ${this.frame.slots.get(e.localId)!.slot}`);
+        return value;
+      }
+      case "recordGet": {
+        const value = B.tmp();
+        B.line(`${value} = load ptr, ptr ${this.emitFieldPtr(e)}`);
+        return value;
+      }
+      case "unionNarrow": {
+        const value = this.emitRef(e.value);
+        const present = B.tmp();
+        B.line(`${present} = icmp ne ptr ${value}, null`);
+        this.require(present);
+        return value;
+      }
+      default:
+        throw new Error(`integer clone: unexpected reference expression ${e.kind}`);
+    }
+  }
+
+  /** The address of the field a record read names. */
+  private emitFieldPtr(e: Extract<IrExpr, { kind: "recordGet" }>): string {
+    const obj = this.emitRef(e.obj);
+    const { struct, index } = this.host.cloneReferences!.field(e.shapeId, e.field)!;
+    const ptr = this.B.tmp();
+    this.B.line(`${ptr} = getelementptr inbounds %${struct}, ptr ${obj}, i64 0, i32 ${index}`);
+    return ptr;
+  }
+
   /** `a.length`, bailing beyond the int32 range. */
   private emitLength(array: string): string {
     const { B } = this;
@@ -1108,12 +1273,14 @@ class CloneEmitter {
 
   private emitArgs(call: Extract<IrExpr, { kind: "call" }>): string[] {
     return call.args.map((a) => {
-      const kind = valueKind(a.type);
+      const kind = valueKind(a.type, this.host.cloneReferences);
       return kind === "arr"
         ? this.emitArray(a)
-        : kind === "num"
-          ? this.narrow(this.emitNum(a))
-          : this.emitBool(a);
+        : kind === "ref"
+          ? this.emitRef(a)
+          : kind === "num"
+            ? this.narrow(this.emitNum(a))
+            : this.emitBool(a);
     });
   }
 
@@ -1202,6 +1369,12 @@ class CloneEmitter {
         return { name: this.emitLength(this.emitArray(e.receiver)), wide: false };
       case "arrayGet":
         return { name: this.emitElement(e.arr, e.index), wide: false };
+      case "recordGet": {
+        // A number field that is exactly an int32; any other number bails.
+        const number = B.tmp();
+        B.line(`${number} = load double, ptr ${this.emitFieldPtr(e)}`);
+        return { name: this.exactInt32(number), wide: false };
+      }
       case "unary": {
         const operand = this.narrow(this.emitNum(e.operand));
         const value = B.tmp();
@@ -1351,6 +1524,21 @@ class CloneEmitter {
         B.line(`${value} = xor i1 ${operand}, true`);
         return value;
       }
+      case "recordGet": {
+        const byte = B.tmp();
+        const value = B.tmp();
+        B.line(`${byte} = load i8, ptr ${this.emitFieldPtr(e)}`);
+        B.line(`${value} = trunc i8 ${byte} to i1`);
+        return value;
+      }
+      case "unionIsTag": {
+        // NULL is the `null` arm; any other pointer the record arm.
+        const pointer = this.emitRef(e.value);
+        const isNull = this.host.cloneReferences!.arm(e.unionId, e.tag) === "null";
+        const value = B.tmp();
+        B.line(`${value} = icmp ${isNull !== e.negated ? "eq" : "ne"} ptr ${pointer}, null`);
+        return value;
+      }
       case "toBool": {
         if (scalarKind(e.operand.type) === "bool") return this.emitBool(e.operand);
         const operand = this.narrow(this.emitNum(e.operand));
@@ -1441,9 +1629,10 @@ export function emitIntegerDispatcher(
   const cloneArgs: string[] = [];
   const exact: string[] = [];
   fn.params.forEach((p, i) => {
-    const kind = valueKind(p.type);
-    if (kind === "bool" || kind === "arr") {
-      // Borrowed arrays pass through to the clone and the ordinary entry.
+    const kind = valueKind(p.type, host.cloneReferences);
+    if (kind === "bool" || kind === "arr" || kind === "ref") {
+      // Borrowed arrays and records pass through to the clone and the
+      // ordinary entry.
       const type = kindType(kind);
       params.push(`${type} %p${i}`);
       entryArgs.push(`${type} %p${i}`);

packages/compiler/src/backend/llvm/lib-dispatch.ts

@@ -1274,7 +1274,7 @@ export function emitLibCall(host: LlvmEmitterContext, e: LibCallExpr): LlValue {
     case "webstream":
       // A stream's start, pull and cancel steps run as promise jobs, so
       // a program with the classes runs its event loop.
-      if (e.fn === "webstream.class") host.usesTimers = true;
+      if (e.fn === "webstream.class" || e.fn === "webstream.install") host.usesTimers = true;
       return host.emitGenericLibCall(e);
     case "test":
       return emitTestLibCall(host, e);

packages/compiler/src/backend/llvm/lib-shared.ts

@@ -120,6 +120,7 @@ export const LIB_FN_SYMS: Record<string, string> = {
   "util.throttle": "scr_util_throttle",
   "webstream.class": "scr_webstream_class",
   "webstream.instanceOf": "scr_webstream_instanceof",
+  "webstream.install": "scr_webstream_install",
   "mime.typeNew": "scr_mime_type_new",
   "mime.typeParse": "scr_mime_type_parse",
   "mime.paramsNew": "scr_mime_params_new",
@@ -556,6 +557,7 @@ export const LIB_FN_SYMS: Record<string, string> = {
   "fs.readFileSyncBuf": "scr_fs_read_file_bytes",
   "fs.readFileSyncBytes": "scr_fs_read_file_bytes",
   "fs.writeFileSyncBytes": "scr_fs_write_file_bytes",
+  "fs.writeFileSyncFd": "scr_fs_write_file_fd",
   "fs.appendFileSyncBytes": "scr_fs_append_file_bytes",
   "fs.readFdSyncBytes": "scr_fs_read_fd_bytes",
   // Stats snapshots (scr_lib.c): statSync/lstatSync throw like the other

packages/compiler/src/backend/llvm/local-array-reads.ts

@@ -8,6 +8,7 @@ import {
   type IrType,
   type IrUnionDef,
 } from "../../ir/ir.js";
+import { emitInlineArrayGet, inlineFieldTail } from "./inline-arrays.js";
 import { everyStmtList } from "../../ir/traverse.js";
 import { analyzeCallLifetimes, type CallLifetimes } from "./call-lifetimes.js";
 import type { LlValue, LlvmEmitterContext } from "./expr-context.js";
@@ -607,6 +608,25 @@ export function emitBorrowedArrayRead(
   slot: string,
 ): void {
   const B = host.B;
+  // An element of an inline array field, borrowed from the instance.
+  if (read.arr.kind === "fieldGet" && read.arr.type.kind === "array") {
+    const tail = inlineFieldTail(host, read.arr.className, read.arr.field);
+    if (tail !== undefined) {
+      const obj = host.emitStableReceiver(read.arr.obj, [read.index]);
+      const index = host.emitExpr(read.index);
+      const value = emitInlineArrayGet(
+        host,
+        obj.name,
+        read.arr.className,
+        tail,
+        index,
+        read.arr.type.elem,
+        false,
+      );
+      B.line(`store ptr ${value}, ptr ${slot}`);
+      return;
+    }
+  }
   const array = host.emitStableReceiver(read.arr, [read.index]);
   const loopIndex = host.emitIntegerLoopIndex(read.index);
   if (loopIndex !== null && host.provenDenseIndex(read.arr, read.index)) {

packages/compiler/src/backend/llvm/local-union-storage.ts

@@ -130,9 +130,10 @@ export function findLocalUnionStorage(
   const types = new Map(fn.locals.map((local) => [local.id, local.type]));
   const result = new Map<string, LocalUnionStorageProof>();
   const safe = lifetimes.projectedLocals.get(fn.name);
-  if (!safe?.size) return result;
+  const copied = lifetimes.copiedLocals.get(fn.name);
+  if (!safe?.size && !copied?.size) return result;
   for (const local of fn.locals) {
-    if (local.type.kind !== "union" || !safe.has(local.id)) continue;
+    if (local.type.kind !== "union" || !(safe?.has(local.id) || copied?.has(local.id))) continue;
     const arms = unions.get(local.type.unionId)?.arms;
     if (!arms) continue;
     const refs = arms.filter(isRefCounted);
@@ -149,6 +150,18 @@ export function findLocalUnionStorage(
       return true;
     },
   });
+  // A copied local's box stays private only while every local it is copied
+  // to snapshots the copy's tag and bits: one that keeps the box pointer
+  // makes the source a heap box again, which can cascade along the copies.
+  for (let changed = copied !== undefined; changed;) {
+    changed = false;
+    for (const [id, targets] of copied!) {
+      if (result.has(id) && !targets.every((target) => result.has(target))) {
+        result.delete(id);
+        changed = true;
+      }
+    }
+  }
   return result;
 }
 

packages/compiler/src/backend/llvm/nullable-unions.test.ts

@@ -3,6 +3,7 @@ import {
   F64,
   NULL_T,
   UNDEFINED_T,
+  type IrClassDef,
   type IrModule,
   type IrType,
   type IrUnionDef,
@@ -146,3 +147,73 @@ test("typed array arms are nullable pointers compared by identity", () => {
   expect(isObjectArm(unions.single("iu")!)).toBe(true);
   expect(isPointerIdentityArm(unions.single("iu")!)).toBe(true);
 });
+
+test("unions of two or more classes become class unions", () => {
+  const cls = (className: string): IrType => ({ kind: "object", className });
+  const def = (name: string, base?: string, runtime?: true): IrClassDef => ({
+    name,
+    fields: [],
+    loc,
+    ...(base === undefined ? {} : { base }),
+    ...(runtime === undefined ? {} : { runtime }),
+  });
+  const classes = [
+    def("%Error", undefined, true),
+    def("%TypeError", "%Error", true),
+    def("Leaf"),
+    def("Pair"),
+    def("Node"),
+    def("Name", "Node"),
+    def("Call", "Node"),
+    def("Sym"),
+    def("Type"),
+  ];
+  const unions = [
+    { id: "lp", arms: [cls("Leaf"), cls("Pair")] },
+    { id: "lpu", arms: [cls("Leaf"), UNDEFINED_T, cls("Pair")] },
+    { id: "lpnu", arms: [NULL_T, cls("Leaf"), cls("Pair"), UNDEFINED_T] },
+    { id: "siblings", arms: [cls("Name"), cls("Call")] },
+    { id: "target", arms: [cls("Sym"), cls("Node"), cls("Type")] },
+    { id: "ancestor", arms: [cls("Node"), cls("Name")] },
+    { id: "number", arms: [cls("Leaf"), F64] },
+    { id: "error", arms: [cls("Leaf"), cls("%TypeError")] },
+    { id: "mixed", arms: [cls("Leaf"), cls("Sym")] },
+    { id: "rp", arms: [cls("Pair"), cls("Leaf")] },
+  ];
+  const lifetime = { contracts: { programLifetime: ["Sym", "Type", "Node"] } };
+  const nullable = new NullableUnions(module(unions, { classes, ...lifetime }));
+  expect(nullable.classes("lp")).toMatchObject({ unitTag: -1, nullTag: -1 });
+  expect(nullable.classes("lpu")).toMatchObject({ unitTag: 1, nullTag: -1 });
+  expect(nullable.classes("lpnu")).toMatchObject({ unitTag: 3, nullTag: 0 });
+  expect(nullable.classes("lpnu")?.arms).toEqual([
+    { tag: 1, className: "Leaf" },
+    { tag: 2, className: "Pair" },
+  ]);
+  expect(nullable.classes("siblings")).not.toBeNull();
+  expect(nullable.classes("target")).not.toBeNull();
+  // An arm that extends another, a scalar arm, a runtime-rooted arm, and
+  // arms of which only some are ProgramLifetime keep the box.
+  expect(nullable.get("ancestor")).toBeNull();
+  expect(nullable.get("number")).toBeNull();
+  expect(nullable.get("error")).toBeNull();
+  expect(nullable.get("mixed")).toBeNull();
+  // Standalone arms keep their class id only because they are arms.
+  expect([...nullable.taggedClasses()].sort()).toEqual(["Leaf", "Pair", "Sym", "Type"]);
+  expect(nullable.helperOwner("rp")).toBe("lp");
+  expect(nullable.helperOwner("lpu")).toBe("lp");
+  expect(nullable.allClassUnions().map((u) => u.unionId)).toEqual([
+    "lp",
+    "lpu",
+    "lpnu",
+    "siblings",
+    "target",
+    "rp",
+  ]);
+  const plain = { classes, ...lifetime };
+  // 32-bit targets, debug builds and library builds keep boxes.
+  expect(new NullableUnions(module(unions, plain), true, undefined, false).get("lp")).toBeNull();
+  expect(new NullableUnions(module(unions, plain), false).get("lp")).toBeNull();
+  expect(
+    new NullableUnions(module(unions, plain), true, undefined, false).taggedClasses().size,
+  ).toBe(0);
+});

packages/compiler/src/backend/llvm/nullable-unions.ts

@@ -54,8 +54,31 @@ export interface ShapeUnion {
   arms: readonly { tag: number; shapeId: string }[];
 }
 
+/** A union of two or more class arms and at most two unit arms (`A | B`,
+ * `A | B | undefined`, sum types over classes) whose VALUES are the
+ * instance pointer itself, like a ShapeUnion: NULL is the unit arm
+ * (`undefined` when both exist, NULLABLE_NULL is then `null`). No arm is
+ * an ancestor of another, so the arm of an instance is the arm whose
+ * preorder interval holds its class id: every instance of an arm's
+ * hierarchy keeps its class id in its header (classes.ts LlClassMeta.idInRc
+ * and compactHeaderWord), which standalone arm classes do only because they
+ * are arms (taggedClasses). The arms are all compact classes or all keep
+ * the rc word, so one load reads the id of any value (the per-union
+ * `@sc_ctag.<id>` helper). */
+export interface ClassUnion {
+  repr: "classes";
+  unionId: string;
+  /** The unit arm NULL encodes; -1 when the union has no unit arm. */
+  unitTag: number;
+  /** The `null` arm of a two-unit-arm union, encoded by NULLABLE_NULL;
+   * -1 otherwise. */
+  nullTag: number;
+  /** The class arms in tag order. */
+  arms: readonly { tag: number; className: string }[];
+}
+
 /** A union whose values are plain pointers (no `%ScrUnion` box). */
-export type PointerUnion = NullableUnion | ShapeUnion;
+export type PointerUnion = NullableUnion | ShapeUnion | ClassUnion;
 
 /** The struct index of a tagged record shape's hidden shape-number word. */
 export const SHAPE_WORD_INDEX = 1;
@@ -79,6 +102,27 @@ export function shapeTraceFn(unionId: string): string {
   return `@sc_strace.${unionId.replace(/[^A-Za-z0-9_.]/g, "_")}`;
 }
 
+/** The per-union helper mapping a class-union value to its tag. */
+export function classTagFn(unionId: string): string {
+  return `@sc_ctag.${unionId.replace(/[^A-Za-z0-9_.]/g, "_")}`;
+}
+
+/** The per-union retain, release and trace entry points of a class union. */
+export function classRetainFn(unionId: string): string {
+  return `@sc_cretain.${unionId.replace(/[^A-Za-z0-9_.]/g, "_")}`;
+}
+export function classReleaseFn(unionId: string): string {
+  return `@sc_crelease.${unionId.replace(/[^A-Za-z0-9_.]/g, "_")}`;
+}
+export function classTraceFn(unionId: string): string {
+  return `@sc_ctrace.${unionId.replace(/[^A-Za-z0-9_.]/g, "_")}`;
+}
+
+/** The helper mapping a shape- or class-union value to its tag. */
+export function pointerTagFn(union: ShapeUnion | ClassUnion): string {
+  return union.repr === "shapes" ? shapeTagFn(union.unionId) : classTagFn(union.unionId);
+}
+
 /** The module-scope immortal sentinel standing for `null` in a
  * `C | null | undefined` union (NULL is `undefined` there). */
 export const NULLABLE_NULL = "@sc_nullable_null";
@@ -102,17 +146,25 @@ export type NullableRefArm =
 
 /** The arm is a JS object: always truthy. */
 export function isObjectArm(nullable: PointerUnion): boolean {
-  return nullable.repr === "shapes" || nullable.arm.kind !== "string";
+  return nullable.repr !== "single" || nullable.arm.kind !== "string";
 }
 
 /** The arm compares by pointer identity. Function arms compare identity
  * roots instead: a signature adapter is the function it adapts. */
 export function isPointerIdentityArm(nullable: PointerUnion): boolean {
   return (
-    nullable.repr === "shapes" || (nullable.arm.kind !== "string" && nullable.arm.kind !== "func")
+    nullable.repr !== "single" || (nullable.arm.kind !== "string" && nullable.arm.kind !== "func")
   );
 }
 
+/** The arm `tag` of a pointer union holds a reference: the value is that
+ * payload itself (not a unit encoding). */
+export function isPayloadTag(nullable: PointerUnion, tag: number): boolean {
+  if (nullable.repr === "single") return tag === nullable.refTag;
+  if (nullable.repr === "shapes") return nullable.arms.some((arm) => arm.tag === tag);
+  return nullable.arms.some((arm) => arm.tag === tag);
+}
+
 /** The representation decision for every union of one module. It is a
  * pure function of the module, so serialized-IR and in-memory emission
  * agree. */
@@ -128,12 +180,18 @@ export class NullableUnions {
   /** Some nullable union encodes `null` with NULLABLE_NULL. */
   readonly usesNullSentinel: boolean = false;
   /** The shape union whose release and trace helpers each shape union
-   * uses: the first one over the same record arms. */
+   * uses: the first one over the same record arms; likewise for class
+   * unions over the same classes. */
   private readonly helperOwners = new Map<string, string>();
+  /** The standalone classes that are arms of a class union, which keep
+   * their class id in their header only for that (taggedClasses). */
+  private readonly tagged = new Set<string>();
 
-  /** `shapes`: allow shape unions (debug builds keep boxes, whose
-   * debug-info layouts describe every arm). */
-  constructor(mod: IrModule, shapes = true, nodes?: IrNodeIndex) {
+  /** `shapes`: allow shape and class unions (debug builds keep boxes,
+   * whose debug-info layouts describe every arm). `classIds`: the target
+   * keeps class ids in instance headers (64-bit targets), which class
+   * unions read their arm from. */
+  constructor(mod: IrModule, shapes = true, nodes?: IrNodeIndex, classIds = true) {
     const emitted = new Set<string>();
     for (const cls of mod.classes ?? []) if (!cls.runtime) emitted.add(cls.name);
     const records = new Set<string>();
@@ -143,6 +201,7 @@ export class NullableUnions {
     // boxes) and of programs with lifetime regions (whose walks report a
     // reference slot by its type's release entry point).
     const allowShapes = shapes && mod.lib === undefined && mod.lifetimeRegions !== true;
+    const classes = allowShapes && classIds ? new ClassForest(mod) : null;
     // A record that is an arm of a boxed union never becomes a shape
     // union's arm: conversions between unions over the same records then
     // keep one representation.
@@ -154,7 +213,8 @@ export class NullableUnions {
       if (boxed.has(def.id)) continue;
       const nullable =
         nullableShape(def, emitted, records) ??
-        (allowShapes ? shapeUnion(def, records, boxedRecords) : null);
+        (allowShapes ? shapeUnion(def, records, boxedRecords) : null) ??
+        (classes !== null ? classUnion(def, classes) : null);
       if (!nullable) continue;
       this.byId.set(def.id, nullable);
       if (nullable.nullTag >= 0) this.usesNullSentinel = true;
@@ -162,22 +222,34 @@ export class NullableUnions {
         for (const arm of nullable.arms)
           if (!this.shapeNumbers.has(arm.shapeId))
             this.shapeNumbers.set(arm.shapeId, this.shapeNumbers.size + 2);
+      if (nullable.repr === "classes")
+        for (const arm of nullable.arms)
+          if (classes!.standalone(arm.className)) this.tagged.add(arm.className);
     }
     // Release and trace dispatch on the shape word, which names a record
-    // shape module-wide, so unions over the same records share them.
+    // shape module-wide, so unions over the same records share them; class
+    // unions dispatch on the class id, so the same holds for their arms.
     const owners = new Map<string, string>();
     for (const nullable of this.byId.values()) {
-      if (nullable.repr !== "shapes") continue;
-      const key = nullable.arms
-        .map((arm) => arm.shapeId)
-        .sort()
-        .join(" ");
+      if (nullable.repr === "single") continue;
+      const names =
+        nullable.repr === "shapes"
+          ? nullable.arms.map((arm) => arm.shapeId)
+          : nullable.arms.map((arm) => arm.className);
+      const key = `${nullable.repr}:${names.sort().join(" ")}`;
       const owner = owners.get(key) ?? nullable.unionId;
       owners.set(key, owner);
       this.helperOwners.set(nullable.unionId, owner);
     }
   }
 
+  /** Standalone classes (no base, no subclass) that are arms of a class
+   * union: their instances keep their class id in their header like
+   * hierarchy members do (classes.ts buildClassGraph). */
+  taggedClasses(): ReadonlySet<string> {
+    return this.tagged;
+  }
+
   /** The shape union whose release and trace helpers this one uses. */
   helperOwner(unionId: string): string {
     return this.helperOwners.get(unionId) ?? unionId;
@@ -203,6 +275,12 @@ export class NullableUnions {
     return nullable?.repr === "shapes" ? nullable : null;
   }
 
+  /** The class-union form only. */
+  classes(unionId: string): ClassUnion | null {
+    const nullable = this.byId.get(unionId);
+    return nullable?.repr === "classes" ? nullable : null;
+  }
+
   /** Every shape union, in module order. */
   allShapeUnions(): ShapeUnion[] {
     const out: ShapeUnion[] = [];
@@ -210,6 +288,13 @@ export class NullableUnions {
     return out;
   }
 
+  /** Every class union, in module order. */
+  allClassUnions(): ClassUnion[] {
+    const out: ClassUnion[] = [];
+    for (const nullable of this.byId.values()) if (nullable.repr === "classes") out.push(nullable);
+    return out;
+  }
+
   /** The module-wide number a tagged record shape stores in its shape
    * word, or 0 for shapes that carry none. */
   shapeNumber(shapeId: string): number {
@@ -312,6 +397,107 @@ function shapeUnion(
   return { repr: "shapes", unionId: def.id, unitTag, nullTag, arms };
 }
 
+/** The class forest of a module, by base links: what classUnion needs
+ * before the emitter builds its class graph. */
+class ClassForest {
+  private readonly byName = new Map<string, IrClassDef>();
+  private readonly hasChildren = new Set<string>();
+  private readonly lifetimeRoots: ReadonlySet<string>;
+  private readonly slotRoots = new Set<string>();
+
+  constructor(mod: IrModule) {
+    for (const cls of mod.classes ?? []) {
+      this.byName.set(cls.name, cls);
+      if (cls.base !== undefined) this.hasChildren.add(cls.base);
+    }
+    this.lifetimeRoots = new Set(mod.contracts?.programLifetime ?? []);
+    for (const key of mod.threadSlotKeys ?? []) {
+      const root = this.root(key);
+      if (root !== undefined) this.slotRoots.add(root.name);
+    }
+  }
+
+  root(className: string): IrClassDef | undefined {
+    let cls = this.byName.get(className);
+    while (cls?.base !== undefined) cls = this.byName.get(cls.base);
+    return cls;
+  }
+
+  /** `ancestor` is `className` or one of its bases. */
+  extends(className: string, ancestor: string): boolean {
+    let cls = this.byName.get(className);
+    while (cls !== undefined) {
+      if (cls.name === ancestor) return true;
+      cls = cls.base === undefined ? undefined : this.byName.get(cls.base);
+    }
+    return false;
+  }
+
+  standalone(className: string): boolean {
+    return this.byName.get(className)?.base === undefined && !this.hasChildren.has(className);
+  }
+
+  /** The class is in a ProgramLifetime hierarchy, whose instances count
+   * nothing. */
+  lifetime(className: string): boolean {
+    const root = this.root(className);
+    return root !== undefined && this.lifetimeRoots.has(root.name);
+  }
+
+  /** Instances of the class keep their class id in a compact header
+   * (classes.ts assignCompactLayouts): ProgramLifetime hierarchies without
+   * a slot word. */
+  compact(className: string): boolean {
+    return this.lifetime(className) && !this.slotRoots.has(this.root(className)!.name);
+  }
+
+  /** An emitted class in a program-rooted hierarchy that is no Array
+   * subclass: one whose instances can carry a class id. */
+  identifiable(className: string): boolean {
+    const cls = this.byName.get(className);
+    const root = this.root(className);
+    return (
+      cls !== undefined &&
+      cls.runtime !== true &&
+      root !== undefined &&
+      root.runtime !== true &&
+      root.arrayElem === undefined
+    );
+  }
+}
+
+/** Two or more class arms none of which extends another, all compact or
+ * all keeping the rc word, all ProgramLifetime classes or none, and at most
+ * two unit arms (both: undefined and null). */
+function classUnion(def: IrUnionDef, classes: ClassForest): ClassUnion | null {
+  const units = def.arms.filter(isUnitType).length;
+  if (units > 2 || def.arms.length - units < 2) return null;
+  const arms: { tag: number; className: string }[] = [];
+  for (let tag = 0; tag < def.arms.length; tag++) {
+    const arm = def.arms[tag]!;
+    if (isUnitType(arm)) continue;
+    if (arm.kind !== "object" || !classes.identifiable(arm.className)) return null;
+    arms.push({ tag, className: arm.className });
+  }
+  const compact = classes.compact(arms[0]!.className);
+  const lifetime = classes.lifetime(arms[0]!.className);
+  for (const arm of arms) {
+    if (classes.compact(arm.className) !== compact) return null;
+    if (classes.lifetime(arm.className) !== lifetime) return null;
+    for (const other of arms)
+      if (other !== arm && classes.extends(arm.className, other.className)) return null;
+  }
+  let unitTag = -1,
+    nullTag = -1;
+  if (units === 1) unitTag = def.arms.findIndex(isUnitType);
+  else if (units === 2) {
+    unitTag = def.arms.findIndex((arm) => arm.kind === "undefinedT");
+    nullTag = def.arms.findIndex((arm) => arm.kind === "nullT");
+    if (unitTag < 0 || nullTag < 0) return null;
+  }
+  return { repr: "classes", unionId: def.id, unitTag, nullTag, arms };
+}
+
 /** Unions the runtime itself inspects as tagged boxes keep that
  * representation: Map keys and Set elements hash and compare union keys by
  * tag and payload inside the C map. A union of records alone is the

packages/compiler/src/backend/llvm/publish.ts

@@ -26,7 +26,6 @@ import { mapOf, STRING, typeKey } from "../../ir/ir.js";
 import { mangleClassStruct, mangleRecordStruct } from "../mangle.js";
 import { FN_ATTRS } from "./shapes.js";
 import { COMPACT_ID_MASK, compactRefLoadLines } from "./classes.js";
-import { shapeTagFn } from "./nullable-unions.js";
 import type { InlineUnionFields } from "./inline-unions.js";
 import { LlvmUnsupportedError } from "./unsupported.js";
 
@@ -53,8 +52,9 @@ export interface PublishHost {
   readonly inlineUnions: InlineUnionFields;
   /** The nullable-pointer representation of a union, when it has one. */
   nullableArm(t: IrType): IrType | null;
-  /** The union's values are tagged record pointers (nullable-unions.ts). */
-  shapeUnion(t: IrType): boolean;
+  /** The tag helper of a union whose values are tagged record pointers or
+   * class instance pointers (nullable-unions.ts), or null. */
+  pointerTagFn(t: IrType): string | null;
   /** The struct index of a record shape's first field. */
   fieldBase(shapeId: string): number;
   /** Class layout: hierarchy membership, preorder interval, field slots
@@ -80,6 +80,10 @@ export interface PublishClassInfo {
    * `compressed` when the field holds a 32-bit reference (classes.ts
    * LlClassMeta.compressed), which the walker decodes. */
   fields: { name: string; index: number; type: IrType; compressed?: boolean }[];
+  /** An inline array field (classes.ts LlClassMeta.inlineTail), which
+   * `fields` leaves out: the GEP index of its i32 element count, the byte
+   * offset of its 32-bit references and their type. */
+  tail?: { name: string; countIndex: number; offset: number; elem: IrType };
 }
 
 /** scr_pub_mark's object kinds (SCR_PUB_KIND_* in scr_runtime.h), for the RC audit. */
@@ -258,12 +262,13 @@ export class PublishWalkers {
             ? []
             : [{ tag: a.tag, fn: a.fn, compact: this.compactRef(def.arms[a.tag]!) }],
         );
-        if (this.host.shapeUnion(t)) {
-          // The value is the record itself (or a unit encoding): no box to
-          // mark; the arm's walker marks and walks the record.
+        const tagFn = this.host.pointerTagFn(t);
+        if (tagFn !== null) {
+          // The value is the record or instance itself (or a unit
+          // encoding): no box to mark; the arm's walker marks and walks it.
           const lines = [head, `entry:`];
           lines.push(
-            `  %tag = call i32 ${shapeTagFn(t.unionId)}(ptr %o)`,
+            `  %tag = call i32 ${tagFn}(ptr %o)`,
             `  switch i32 %tag, label %done [ ${live.map((a) => `i32 ${a.tag}, label %arm${a.tag}`).join(" ")} ]`,
           );
           for (const a of live)
@@ -405,6 +410,7 @@ export class PublishWalkers {
       loc,
       info.idInRc,
       info.compact,
+      info.tail,
     );
   }
 
@@ -563,6 +569,7 @@ export class PublishWalkers {
     loc: SrcLoc,
     classId = false,
     compact = false,
+    tail?: PublishClassInfo["tail"],
   ): string[] {
     const lines = [
       head,
@@ -591,7 +598,33 @@ export class PublishWalkers {
       );
       n++;
     }
-    lines.push(`  br label %done`, `done:`, `  ret void`, `}`);
+    const tailFn =
+      tail === undefined ? null : this.walker(tail.elem, `${path}.${tail.name}[]`, loc);
+    if (tail !== undefined && tailFn !== null) {
+      // Each element of an inline array field, as a 32-bit reference.
+      lines.push(
+        `  br label %tail.pre`,
+        `tail.pre:`,
+        `  %tail.cp = getelementptr inbounds %${struct}, ptr %o, i64 0, i32 ${tail.countIndex} ; ${llvmComment(tail.name)}`,
+        `  %tail.n = load i32, ptr %tail.cp`,
+        `  %tail.at = getelementptr inbounds i8, ptr %o, i64 ${tail.offset}`,
+        `  br label %tail.head`,
+        `tail.head:`,
+        `  %tail.i = phi i32 [ 0, %tail.pre ], [ %tail.next, %tail.latch ]`,
+        `  %tail.end = icmp uge i32 %tail.i, %tail.n`,
+        `  br i1 %tail.end, label %done, label %tail.body`,
+        `tail.body:`,
+        `  %tail.p = getelementptr inbounds i32, ptr %tail.at, i32 %tail.i`,
+        `  %tail.r = load i32, ptr %tail.p`,
+        ...compactRefLoadLines(`%tail.r`, `%tail.v`, `%tail.d`),
+        ...this.push(`%tail.v`, tailFn, `tail`, this.compactRef(tail.elem)),
+        `  br label %tail.latch`,
+        `tail.latch:`,
+        `  %tail.next = add nuw i32 %tail.i, 1`,
+        `  br label %tail.head`,
+      );
+    } else lines.push(`  br label %done`);
+    lines.push(`done:`, `  ret void`, `}`);
     return lines;
   }
 }

packages/compiler/src/backend/llvm/receivers.ts

@@ -1,4 +1,5 @@
 /* LlEmitter's receivers: receiver reads, and the analyses that decide when calls, receivers and returns borrow. */
+import { inlineFieldTail } from "./inline-arrays.js";
 import type { LlEmitter } from "./emitter.js";
 import {
   type IrExpr,
@@ -133,6 +134,9 @@ export function emitReadReceiver(emitter: LlEmitter, e: IrExpr): LlValue {
   // A nullable-pointer field has no box to borrow. Projections use
   // emitUnionProjection; other borrowers get an owned heap box.
   if (emitter.nullableFieldGet(e)) return emitter.emitExpr(e);
+  // An inline array field has no array to borrow: its read makes a view.
+  if (e.kind === "fieldGet" && inlineFieldTail(emitter, e.className, e.field) !== undefined)
+    return emitter.emitExpr(e);
   if (e.kind === "recordGet" || e.kind === "fieldGet") {
     const receiver = emitter.emitReadReceiver(e.obj);
     if (e.kind === "recordGet") {
@@ -224,8 +228,9 @@ export function canBorrowCallArgument(emitter: LlEmitter, value: IrExpr): boolea
   // An exact capsule owns its instance while the capsule's binding holds it.
   if (value.kind === "dynCheck" && value.exact)
     return value.value.kind === "varRef" && emitter.canBorrowCallArgument(value.value);
-  // An inline union field has no box to lend.
-  if (emitter.inlineFieldGet(value)) return false;
+  // An inline union field has no box to lend, an inline array field no
+  // array.
+  if (emitter.inlineFieldGet(value) || emitter.inlineArrayRead(value)) return false;
   // Class casts reinterpret the same pointer (prefix layouts).
   if ((value.kind === "upcast" || value.kind === "downcast") && value.value.type.kind === "object")
     return emitter.canBorrowCallArgument(value.value);
@@ -308,10 +313,15 @@ export function canBorrowReceiver(emitter: LlEmitter, e: IrExpr): boolean {
     case "upcast":
       return emitter.canBorrowReceiver(e.value);
     // An inline union field has no box to lend: projections read it
-    // through emitReadReceiver's private stack box.
+    // through emitReadReceiver's private stack box. An inline array field
+    // has no array to lend.
     case "fieldGet":
     case "recordGet":
-      return !emitter.inlineFieldGet(e) && emitter.canBorrowReceiver(e.obj);
+      return (
+        !emitter.inlineFieldGet(e) &&
+        !emitter.inlineArrayRead(e) &&
+        emitter.canBorrowReceiver(e.obj)
+      );
     case "ternary":
       return emitter.canBorrowReceiver(e.then) && emitter.canBorrowReceiver(e.else_);
     case "seqExpr": {

packages/compiler/src/backend/llvm/shapes.ts

@@ -47,10 +47,14 @@ import { SCR_ARR_FIELD, llvmCommentText } from "./common.js";
 import {
   NULLABLE_NULL,
   SHAPE_WORD_INDEX,
+  classReleaseFn,
+  classRetainFn,
+  classTraceFn,
   shapeReleaseFn,
   shapeTagFn,
   shapeTraceFn,
   shapeWordHeader,
+  type ClassUnion,
   type NullableUnions,
   type ShapeUnion,
 } from "./nullable-unions.js";
@@ -257,6 +261,7 @@ export { computeTraced } from "../cycle-analysis.js";
 export function vAdapters(host: ShapeHost, t: IrType): { retain: string; release: string } {
   const nullable = host.nullableUnions.of(t);
   if (nullable?.repr === "shapes") return shapeUnionRc(host, nullable);
+  if (nullable?.repr === "classes") return classUnionRc(host, nullable);
   if (nullable) {
     const arm = vAdapters(host, nullable.arm);
     return { retain: nullableRetainSym(host, nullable.arm), release: arm.release };
@@ -314,6 +319,7 @@ export function retainSym(host: ShapeHost, t: IrType): string {
   if (t.kind === "jsval") return jsvalRcSym(host, "retain");
   const nullable = host.nullableUnions.of(t);
   if (nullable?.repr === "shapes") return shapeUnionRc(host, nullable).retain;
+  if (nullable?.repr === "classes") return classUnionRc(host, nullable).retain;
   if (nullable) return nullableRetainSym(host, nullable.arm);
   const stem = runtimeRcStem(t);
   if (host.rcHelpers !== null && stem !== null && inlineRcFamily(host, stem, "retain") !== null) {
@@ -335,6 +341,13 @@ function shapeUnionRc(host: ShapeHost, union: ShapeUnion): { retain: string; rel
   };
 }
 
+/** A class union's RC entry points (class-unions.ts), shared by the unions
+ * over the same classes. */
+function classUnionRc(host: ShapeHost, union: ClassUnion): { retain: string; release: string } {
+  const owner = host.nullableUnions.helperOwner(union.unionId);
+  return { retain: classRetainFn(owner), release: classReleaseFn(owner) };
+}
+
 /** The widest span of shape numbers a union's arms may cover for its tag
  * helper to switch on the shape word (emitShapeUnionHelpers). */
 const SHAPE_TAG_SWITCH_SPAN = 32;
@@ -552,6 +565,7 @@ export function releaseSym(host: ShapeHost, t: IrType): string {
   if (t.kind === "jsval") return jsvalRcSym(host, "release");
   const nullable = host.nullableUnions.of(t);
   if (nullable?.repr === "shapes") return shapeUnionRc(host, nullable).release;
+  if (nullable?.repr === "classes") return classUnionRc(host, nullable).release;
   if (nullable) return releaseSym(host, nullable.arm);
   const stem = runtimeRcStem(t);
   if (host.rcHelpers !== null && stem !== null && inlineRcFamily(host, stem, "release") !== null) {
@@ -1111,6 +1125,11 @@ export function traceAdapter(host: ShapeHost, t: IrType): string | null {
         return shapeUnionTraced(host, nullable)
           ? shapeTraceFn(host.nullableUnions.helperOwner(nullable.unionId))
           : null;
+      // Every arm of a class union carries a collector header when one does.
+      if (nullable?.repr === "classes")
+        return host.tracedShapes.has(`object:${nullable.arms[0]!.className}`)
+          ? classTraceFn(host.nullableUnions.helperOwner(nullable.unionId))
+          : null;
       if (nullable) return traceAdapter(host, nullable.arm);
       if (!host.tracedUnions.has(t.unionId)) return null;
       host.declare(`declare void @scr_union_trace_v(ptr, ptr, ptr)`);
@@ -1754,7 +1773,7 @@ function directDestroyKey(host: ShapeHost, type: IrType): string | null {
   const nullable = host.nullableUnions.of(type);
   let t = type;
   if (nullable !== null) {
-    if (nullable.repr === "shapes") return null;
+    if (nullable.repr !== "single") return null;
     t = nullable.arm;
   }
   if (t.kind === "record") return `record:${t.shapeId}`;
@@ -1977,8 +1996,9 @@ export function releaseBody(
   segmented = false,
   frozen = false,
   destroyDefs: readonly string[] | null = null,
+  count: RcCount = host.sizeType,
 ): string[] {
-  const S = host.sizeType;
+  const S = count;
   if (bounded) host.declare(`declare void @scr_rc_destroy(ptr, ptr)`);
   const destroy = `${fnName}_destroy`;
   // A freezable record's count is the word without its frozen bit.
@@ -2367,8 +2387,20 @@ export function emitRecordShapes(
           `  %shp = getelementptr inbounds %${struct}, ptr %o, i64 0, i32 ${SHAPE_WORD_INDEX}`,
           `  store i32 ${shapeNumber}, ptr %shp ; shape number`,
         );
+      // A record literal covers every field of its shape (ir.ts recordLit),
+      // so without dispose entries, which read the fields at the drop, no
+      // field is read before the literal stores it.
       defs.push(
-        ...arenaNewLines(host, newSym, sizeOf, traced ? cycType : null, dispose, init, "%o"),
+        ...arenaNewLines(
+          host,
+          newSym,
+          sizeOf,
+          traced ? cycType : null,
+          dispose,
+          init,
+          "%o",
+          dispose === null ? "header" : "object",
+        ),
       );
       if (dispose !== null)
         defs.push(

packages/compiler/src/backend/llvm/stack-objects.ts

@@ -16,7 +16,7 @@ import {
 } from "./record-results.js";
 import { everyStmtList } from "../../ir/traverse.js";
 import { mangleClassStruct, mangleRecordStruct } from "../mangle.js";
-import { classFieldIndex, classUndefinedFieldInits } from "./classes.js";
+import { classFieldIndex, classIdImmortalRcWord, classUndefinedFieldInits } from "./classes.js";
 import { CYC_BUF, CYC_HDR } from "./alloc.js";
 import type { CallLifetimes } from "./call-lifetimes.js";
 import { emitCallExpr, emitConstruction } from "./expr-calls.js";
@@ -396,7 +396,10 @@ export function emitStackObject(
         B.line(line.trim());
     }
   }
-  B.line(`store ${host.sizeType} -1, ptr ${slot}`);
+  const tagged = type.kind === "object" ? host.classMeta.get(type.className) : undefined;
+  // A class-union arm keeps its class id beside the immortal count.
+  if (tagged?.idInRc === true) B.line(`store i64 ${classIdImmortalRcWord(tagged)}, ptr ${slot}`);
+  else B.line(`store ${host.sizeType} -1, ptr ${slot}`);
   if (type.kind === "record") {
     // A tagged shape's word names it to every shape union's tag read.
     const shapeNumber = host.nullableUnions.shapeNumber(type.shapeId);

packages/compiler/src/backend/llvm/stack-unions.test.ts

@@ -172,8 +172,11 @@ test.each(["return", "alias", "capture", "store"])(
       work.body.push(ret(ref("item")));
     }
     if (reason === "alias") {
+      // The alias escapes, so it holds a heap box and so does its source.
+      work.returnType = optional;
       work.locals.push({ id: "alias", name: "alias", type: optional, mutable: false });
       work.body.push({ kind: "varDecl", localId: "alias", init: ref("item"), loc });
+      work.body.push(ret(ref("alias")));
     }
     if (reason === "capture") {
       work.locals[0]!.boxed = true;
@@ -204,6 +207,27 @@ test.each(["return", "alias", "capture", "store"])(
   },
 );
 
+test("copies to a projection-only alias keep both locals in private storage", () => {
+  const mod = fixture();
+  const work: IrFunction = {
+    name: "work",
+    params: [],
+    locals: [],
+    body: [],
+    returnType: VOID,
+    loc,
+  };
+  mod.functions.unshift(work);
+  declareLocal(mod, wrap(fresh()));
+  work.locals.push({ id: "alias", name: "alias", type: optional, mutable: true });
+  work.body.push({ kind: "varDecl", localId: "alias", init: ref("item"), loc });
+  work.body.push({ kind: "assign", localId: "alias", value: ref("item"), loc });
+  work.body.push(effect(call("read", [ref("alias")], record)));
+  const text = body(emit(mod), "sc_f_work");
+  expect(text).toContain("alloca %ScrUnion");
+  expect(text).not.toMatch(/@scr_union_(?:new|retain|release)/);
+});
+
 test("forwarded parameters never retain or release a borrowed stack box", () => {
   const mod = fixture();
   const outer: IrFunction = {

packages/compiler/src/backend/llvm/union-repr.ts

@@ -3,10 +3,11 @@
  * nullable union (nullable-unions.ts) is the arm pointer itself: NULL for
  * its NULL-encoded unit arm and, in `C | null | undefined`, the immortal
  * NULLABLE_NULL sentinel for `null`. A shape union's value is likewise the
- * record pointer; its tag comes from the record's shape word. Every structural read of a union value
- * goes through here. */
+ * record pointer; its tag comes from the record's shape word. A class
+ * union's value is the instance pointer; its tag comes from the class id.
+ * Every structural read of a union value goes through here. */
 import type { BlockBuilder } from "./blocks.js";
-import { NULLABLE_NULL, shapeTagFn, type PointerUnion } from "./nullable-unions.js";
+import { NULLABLE_NULL, pointerTagFn, type PointerUnion } from "./nullable-unions.js";
 
 /** The i32 tag of a union value. `alias` is the tag word's alias
  * attachment (", !tbaa !N"), when the caller has one. */
@@ -17,8 +18,8 @@ export function emitUnionTag(
   alias = "",
 ): string {
   const t = B.tmp();
-  if (nullable?.repr === "shapes") {
-    B.line(`${t} = call i32 ${shapeTagFn(nullable.unionId)}(ptr ${u})`);
+  if (nullable !== null && nullable.repr !== "single") {
+    B.line(`${t} = call i32 ${pointerTagFn(nullable)}(ptr ${u})`);
     return t;
   }
   if (nullable) {
@@ -45,8 +46,9 @@ export function emitUnionTag(
  * the `null` sentinel). */
 export function emitNullablePresent(B: BlockBuilder, nullable: PointerUnion, u: string): string {
   const present = B.tmp();
-  if (nullable.repr === "shapes" && nullable.unitTag < 0) {
-    // No unit arm: every value is a record (NULL only as a pending dummy).
+  if (nullable.repr !== "single" && nullable.unitTag < 0) {
+    // No unit arm: every value is a record or an instance (NULL only as a
+    // pending dummy).
     B.line(`${present} = icmp ne ptr ${u}, null`);
     return present;
   }
@@ -67,7 +69,7 @@ export function emitNullableIsTag(
   tag: number,
   negated: boolean,
 ): string {
-  if (nullable.repr === "shapes" && tag !== nullable.unitTag && tag !== nullable.nullTag) {
+  if (nullable.repr !== "single" && tag !== nullable.unitTag && tag !== nullable.nullTag) {
     const actual = emitUnionTag(B, nullable, u),
       t = B.tmp();
     B.line(`${t} = icmp ${negated ? "ne" : "eq"} i32 ${actual}, ${tag}`);
@@ -104,8 +106,8 @@ export function emitUnionPeek(
 /** Text-template forms for helpers written as raw LLVM lines: `%out`
  * receives the tag / payload of `u`. */
 export function unionTagLines(nullable: PointerUnion | null, u: string, out: string): string[] {
-  if (nullable?.repr === "shapes")
-    return [`  ${out} = call i32 ${shapeTagFn(nullable.unionId)}(ptr ${u})`];
+  if (nullable !== null && nullable.repr !== "single")
+    return [`  ${out} = call i32 ${pointerTagFn(nullable)}(ptr ${u})`];
   if (nullable && nullable.nullTag < 0)
     return [
       `  ${out}.isnull = icmp eq ptr ${u}, null`,

packages/compiler/src/backend/llvm/unwind-abi.test.ts

@@ -93,7 +93,9 @@ test("a method of a class outside any hierarchy is on the unwinding ABI; closure
   // closure over mid is a pointer target and returns with the flag.
   const add = body(ir, "Counter_add");
   expect(add).toMatch(/invoke double @sc_b?f_[^(]*mid/);
-  expect(add).toMatch(/resume \{ ptr, i32 \} %[\w.]+/);
+  expect(add).toMatch(/call void @scr_unwind_resume\(ptr %[\w.]+\)\n\s+unreachable/);
+  expect(add).not.toMatch(/\bresume \{/);
+  expect(ir).toContain("declare void @scr_unwind_resume(ptr) cold");
   expect(add).not.toContain("@scr_unwind_raise");
   const closures = ir
     .split(/\n(?=define )/)
@@ -212,7 +214,10 @@ test("an invoked wrapper of a raising function resumes the unwind; its address m
   expect(out).toContain(
     "%r = invoke double @sc_bf_body.i32(i32 %i) to label %ub.ok.0 unwind label %ub.lp.0",
   );
-  expect(out).toContain("resume { ptr, i32 } %ub.v.0");
+  expect(out).toContain(
+    "%ub.e.0 = extractvalue { ptr, i32 } %ub.v.0, 0\n  call void @scr_unwind_resume(ptr %ub.e.0)\n  unreachable",
+  );
+  expect(out).toContain("declare void @scr_unwind_resume(ptr) cold");
   expect(() =>
     bridgePlainUnwindCalls([
       `${raising}\n@table = internal constant [1 x ptr] [ptr @sc_bf_body.i32]`,
@@ -277,11 +282,27 @@ test("calls that cannot unwind are marked nounwind", () => {
     "  unreachable",
     "}",
   ].join("\n");
-  const [out] = bridgePlainUnwindCalls([raising]);
+  const resuming = [
+    "define internal void @sc_bf_resumes(ptr %b) #0 personality ptr @__gcc_personality_v0 {",
+    "entry:",
+    "  invoke void @sc_bf_raises(ptr %b) to label %ok unwind label %lp",
+    "ok:",
+    "  ret void",
+    "lp:",
+    "  %e = landingpad { ptr, i32 } cleanup",
+    "  %t = extractvalue { ptr, i32 } %e, 0",
+    "  call void @scr_unwind_resume(ptr %t)",
+    "  unreachable",
+    "}",
+  ].join("\n");
+  const [out] = bridgePlainUnwindCalls([[raising, resuming].join("\n")]);
   expect(out).toContain("%n = call i64 @scr_bytes_length(ptr %b) nounwind ; length");
   expect(out).toContain("call void @scr_bytes_release(ptr %b) nounwind, !dbg !4");
   expect(out).toContain("%r = call ptr %b(ptr %b) nounwind");
   expect(out).toContain("  call void @scr_unwind_raise()\n");
+  // The resume unwinds: where LLVM inlines the function at an invoke, the
+  // call becomes an invoke of the caller's landing pad.
+  expect(out).toContain("  call void @scr_unwind_resume(ptr %t)\n");
 });
 
 test("functions without a plain call of a raising function change only their call lines", () => {

packages/compiler/src/backend/llvm/unwind-abi.ts

@@ -76,9 +76,11 @@ function recursiveFunctions(edges: ReadonlyMap<string, readonly string[]>): Set<
  * than a `call`'s callee, and not called from an async or generator body
  * (coroutine bodies keep plain calls).
  *
- * A native unwind costs about 2 µs a raise and 0.1 µs a frame, against a
- * few nanoseconds for a flag test, so the functions an exception is likely to
- * cross many of keep the flag: those with a `throw` or rethrow of their own
+ * A native unwind costs about 0.05 µs a raise and 10 to 25 ns a frame on
+ * x86-64 (scr_unwind.c's cached plans) and about 2 µs a raise and 0.1 µs a
+ * frame through libgcc's unwinder, against a few nanoseconds for a flag
+ * test, so the functions an exception is likely to cross many of keep the
+ * flag: those with a `throw` or rethrow of their own
  * and those up to THROW_LEVELS calls above one (a handler near the throw
  * takes it without a native unwind), and the recursive ones (deep stacks
  * are recursions). */
@@ -254,8 +256,8 @@ export function unwindAbiFunctions(
  * its own takes the exception. */
 export const UNWIND_RAISE = "call void @scr_unwind_raise()\n  unreachable";
 /** What the unwind of a landing pad ends with when no handler of the
- * function takes the exception: the unwind goes on (`resume`) with the
- * exception the landing pad stored in UNWIND_SLOT. The block builder
+ * function takes the exception: the unwind goes on (scr_unwind_resume) with
+ * the exception the landing pad stored in UNWIND_SLOT. The block builder
  * expands it (BlockBuilder.terminate). */
 export const UNWIND_RESUME = "resume { ptr, i32 } <unwind slot>";
 /** The function's slot for the exception its landing pads take. */
@@ -264,13 +266,17 @@ export const UNWIND_SLOT = "%unw.exc";
  * would call the sanitizer's no-return hook before it, which fails on fiber
  * stacks; scr_unwind_raise clears the unwound frames' shadow itself. */
 export const UNWIND_RAISE_DECLARATION = "declare void @scr_unwind_raise() cold";
+/** scr_unwind_resume's declaration. Landing pads call it with the
+ * exception they took to continue the unwind (BlockBuilder.resumeUnwind);
+ * like scr_unwind_raise, it is not `noreturn`. */
+export const UNWIND_RESUME_DECLARATION = "declare void @scr_unwind_resume(ptr) cold";
 
 /** The personality of functions with landing pads: the landing pads are
- * cleanups that scr_unwind_raise's forced unwind runs. */
+ * cleanups that scr_unwind_raise's unwind runs. */
 export const UNWIND_PERSONALITY = "personality ptr @__gcc_personality_v0";
 
-/** An unwind leaving the function: a raise, or a landing pad's `resume`. */
-const UNWINDS = /call void @scr_unwind_raise\(\)|\n\s+resume /;
+/** An unwind leaving the function: a raise, or a landing pad's resume. */
+const UNWINDS = /call void @scr_unwind_raise\(\)|call void @scr_unwind_resume\(/;
 /** A pending-flag test: the thread's exception word, the active cell, or the runtime query. */
 const PENDING_TEST = /@scr_exc_flag\b|%exc\.cell\b|@scr_exc_active\b|@scr_exc_pending\(/;
 
@@ -307,15 +313,16 @@ function argumentsEnd(line: string, open: number): number {
   return close;
 }
 
-/** A call that cannot unwind, marked so: only scr_unwind_raise and the
- * raising functions propagate an unwind (runtime code and every function
- * reached through a pointer return normally), so every other call is
- * `nounwind`, and the inliner keeps it a call where it inlines a raising
- * function at an invoke. */
+/** A call that cannot unwind, marked so: only scr_unwind_raise,
+ * scr_unwind_resume and the raising functions propagate an unwind (runtime
+ * code and every function reached through a pointer return normally), so
+ * every other call is `nounwind`, and the inliner keeps it a call where it
+ * inlines a raising function at an invoke. */
 function markNounwind(line: string): string {
   const call = line.indexOf("call ");
   const callee = CALLEE.exec(line.slice(call));
-  if (callee === null || callee[0] === "@scr_unwind_raise(") return line;
+  if (callee === null || callee[0] === "@scr_unwind_raise(" || callee[0] === "@scr_unwind_resume(")
+    return line;
   const close = argumentsEnd(line, call + callee.index + callee[0].length - 1);
   return close >= line.length
     ? line
@@ -446,6 +453,7 @@ export function bridgePlainUnwindCalls(parts: string[]): string[] {
       }
   if (violations.size > 0) throw new UnwindAbiViolation(violations);
   let personality = false;
+  let resumes = false;
   let counter = 0;
   const out = split.map((chunks) =>
     chunks
@@ -525,8 +533,11 @@ export function bridgePlainUnwindCalls(parts: string[]): string[] {
             result.push(
               `${pad}:`,
               `${indent}%ub.v.${n} = landingpad { ptr, i32 } cleanup`,
-              `${indent}resume { ptr, i32 } %ub.v.${n}`,
+              `${indent}%ub.e.${n} = extractvalue { ptr, i32 } %ub.v.${n}, 0`,
+              `${indent}call void @scr_unwind_resume(ptr %ub.e.${n})`,
+              `${indent}unreachable`,
             );
+            resumes = true;
             result.push(`${ok}:`);
           } else if (m[2] !== undefined) {
             const type = (m[3] ?? "").replace(RETURN_ATTRIBUTES, "").trim();
@@ -585,5 +596,7 @@ export function bridgePlainUnwindCalls(parts: string[]): string[] {
   );
   if (personality && !out.some((part) => part.includes("@__gcc_personality_v0(...)")))
     out[0] = `declare i32 @__gcc_personality_v0(...)\n${out[0]}`;
+  if (resumes && !out.some((part) => part.includes(UNWIND_RESUME_DECLARATION)))
+    out[0] = `${UNWIND_RESUME_DECLARATION}\n${out[0]}`;
   return out;
 }

packages/compiler/src/backend/native-codegen-core.ts

@@ -9,60 +9,41 @@ export type NativeCodegenOutputKind = "asm" | "obj";
 /** LLVM text that stays one module, LLVM text per partition, the fewest
  * partitions (MIN_PROGRAM_PARTITIONS, of at least MIN_PARTITION_BYTES each)
  * a divided module gets, and the most one executable uses. */
-const SINGLE_PARTITION_BYTES = 1024 * 1024;
+const SINGLE_PARTITION_BYTES = 128 * 1024;
 const PROGRAM_PARTITION_BYTES = 4 * 1024 * 1024;
-const MIN_PARTITION_BYTES = 512 * 1024;
+const MIN_PARTITION_BYTES = 32 * 1024;
 const MIN_PROGRAM_PARTITIONS = 8;
 const MAX_PROGRAM_PARTITIONS = 64;
-/** SINGLE_PARTITION_BYTES and MIN_PARTITION_BYTES of unoptimized (dev)
- * executables. */
+/** SINGLE_PARTITION_BYTES of unoptimized (dev) executables. */
 const UNOPTIMIZED_SINGLE_PARTITION_BYTES = 64 * 1024;
-const UNOPTIMIZED_MIN_PARTITION_BYTES = 32 * 1024;
 
 /** Executable objects larger than SINGLE_PARTITION_BYTES are divided into
  * partitions that the helper simplifies, optimizes and compiles concurrently,
  * importing small functions across partitions for inlining, and reuses
- * individually across builds. A divided module gets at least
- * MIN_PROGRAM_PARTITIONS so that mid-size programs use several cores. The
- * count is a power of two that depends only on the module's size and
- * whether it is optimized, on every target, so artifacts never vary with the
- * host and ordinary edits keep every function in its partition.
+ * individually across builds, so an edit optimizes again only the partitions
+ * it affects. A divided module gets at least MIN_PROGRAM_PARTITIONS so that
+ * small and mid-size programs use several cores. The count is a power of two
+ * that depends only on the module's size and whether it is optimized, on
+ * every target, so artifacts never vary with the host and ordinary edits keep
+ * every function in its partition. An optimized module whose largest
+ * partition would hold more than two thirds of its function text is
+ * optimized whole by the helper instead (as for one partition), and its code
+ * is generated in parts.
  *
  * Unoptimized (dev) partitions are compiled on their own with nothing
  * imported, so they cost no run time and an edit compiles only the partitions
  * whose functions it changes: dev executables are divided from
- * UNOPTIMIZED_SINGLE_PARTITION_BYTES, at least
- * UNOPTIMIZED_MIN_PARTITION_BYTES each. */
+ * UNOPTIMIZED_SINGLE_PARTITION_BYTES. */
 export function nativeProgramPartitions(llvmBytes: number, unoptimized = false): number {
-  const single = unoptimized ? UNOPTIMIZED_SINGLE_PARTITION_BYTES : SINGLE_PARTITION_BYTES;
-  const least = unoptimized ? UNOPTIMIZED_MIN_PARTITION_BYTES : MIN_PARTITION_BYTES;
-  if (llvmBytes <= single) return 1;
+  if (llvmBytes <= (unoptimized ? UNOPTIMIZED_SINGLE_PARTITION_BYTES : SINGLE_PARTITION_BYTES))
+    return 1;
   const required = Math.max(
     Math.ceil(llvmBytes / PROGRAM_PARTITION_BYTES),
-    Math.min(MIN_PROGRAM_PARTITIONS, Math.ceil(llvmBytes / least)),
+    Math.min(MIN_PROGRAM_PARTITIONS, Math.ceil(llvmBytes / MIN_PARTITION_BYTES)),
   );
   return Math.min(MAX_PROGRAM_PARTITIONS, 2 ** Math.ceil(Math.log2(required)));
 }
 
-/** LLVM text that an optimized one-partition program generates code for on
- * one thread, the LLVM text per code generation part, and the most parts. */
-const SINGLE_CODEGEN_BYTES = 128 * 1024;
-const CODEGEN_PART_BYTES = 64 * 1024;
-const MAX_CODEGEN_PARTS = 8;
-
-/** Code generation parts for an optimized executable object that
- * nativeProgramPartitions keeps as one partition. The helper optimizes the
- * module whole, as for a single output, and then generates code for the parts
- * concurrently (`--split codegen`), so the generated code does not change.
- * Returns 1 for modules of at most SINGLE_CODEGEN_BYTES and for partitioned
- * ones. The count is a power of two that depends only on the module's size,
- * so artifacts never vary with the host. */
-export function nativeCodegenParts(llvmBytes: number): number {
-  if (llvmBytes <= SINGLE_CODEGEN_BYTES || llvmBytes > SINGLE_PARTITION_BYTES) return 1;
-  const required = Math.ceil(llvmBytes / CODEGEN_PART_BYTES);
-  return Math.min(MAX_CODEGEN_PARTS, 2 ** Math.ceil(Math.log2(required)));
-}
-
 /** Output paths for each partition; the first is the requested path. */
 export function nativePartitionPaths(path: string, count: number): string[] {
   const dot = path.lastIndexOf(".");

packages/compiler/src/backend/native-codegen.test.ts

@@ -8,11 +8,7 @@ import {
   NativeCodegenError,
   validateNativeCodegenVersion,
 } from "./native-codegen.js";
-import {
-  nativeCodegenParts,
-  nativePartitionPaths,
-  nativeProgramPartitions,
-} from "./native-codegen-core.js";
+import { nativePartitionPaths, nativeProgramPartitions } from "./native-codegen-core.js";
 import { nativeArtifactDependenciesStillMatch } from "./native-toolchain.js";
 import { LINUX_X64_GNU_TARGET, MACOS_ARM64_TARGET, WASM32_WASI_TARGET } from "./targets.js";
 import { compilerReleaseVersion } from "../library/sidecar.js";
@@ -206,6 +202,36 @@ test("profiles reach the helper and key the native cache by their bytes", async
   expect(calls[2]).toContain("--profile-use /profiles/a.profdata");
 });
 
+test("block counts reach the helper, and a build that records them reuses no artifact", async () => {
+  const pkg = await fakePackage();
+  const use = { path: "/profiles/a.profdata", digest: "a".repeat(64), contextSensitive: false };
+  const base = request(pkg.root, pkg.packageJson, join(pkg.root, "used.o"));
+  await emitNativeArtifact({ ...base, profile: { use } });
+  const blocks = { path: "/profiles/a.blocks", digest: "c".repeat(64) };
+  const annotated = {
+    ...base,
+    outputPath: join(pkg.root, "blocks.o"),
+    profile: { use: { ...use, blocks } },
+  };
+  await emitNativeArtifact(annotated);
+  await emitNativeArtifact({ ...annotated, outputPath: join(pkg.root, "blocks-again.o") });
+  const recording = {
+    ...base,
+    outputPath: join(pkg.root, "recorded.o"),
+    profile: { use, recordBlocks: "/work/a.blocks" },
+  };
+  await emitNativeArtifact(recording);
+  await emitNativeArtifact(recording);
+  const calls = (await readFile(pkg.log, "utf8")).trim().split("\n");
+  // The block counts key the cache; each recording build runs the helper.
+  expect(calls).toHaveLength(4);
+  expect(calls[1]).toContain(
+    "--profile-use /profiles/a.profdata --profile-blocks /profiles/a.blocks",
+  );
+  expect(calls[2]).toContain("--record-profile-blocks /work/a.blocks");
+  expect(calls[3]).toContain("--record-profile-blocks /work/a.blocks");
+});
+
 test("a cached artifact keeps the profile match of the build that produced it", async () => {
   const pkg = await fakePackage({
     profileReport:
@@ -217,7 +243,8 @@ test("a cached artifact keeps the profile match of the build that produced it",
       use: { path: "/profiles/a.profdata", digest: "a".repeat(64), contextSensitive: false },
     },
   };
-  const match = { annotated: 120, mismatched: 7, missing: 40, unknownPartitions: 0 };
+  // A helper that predates block-count files reports no recovered count.
+  const match = { annotated: 120, mismatched: 7, missing: 40, recovered: 0, unknownPartitions: 0 };
   expect((await emitNativeArtifact(used)).profileMatch).toEqual(match);
   const again = await emitNativeArtifact({ ...used, outputPath: join(pkg.root, "again.o") });
   expect(again.profileMatch).toEqual(match);
@@ -232,6 +259,7 @@ test("a cached artifact keeps the profile match of the build that produced it",
     annotated: 0,
     mismatched: 0,
     missing: 0,
+    recovered: 0,
     unknownPartitions: 1,
   });
   // Builds without a profile carry no match, from the helper or the cache.
@@ -398,18 +426,16 @@ test("sanitized native artifacts fail before helper resolution", async () => {
 });
 
 test("program partitions depend only on module size", () => {
-  const megabyte = 1024 * 1024;
-  expect(nativeProgramPartitions(megabyte)).toBe(1);
-  expect(nativeProgramPartitions(megabyte + 1)).toBe(4);
-  expect(nativeProgramPartitions(2 * megabyte)).toBe(4);
-  expect(nativeProgramPartitions(2 * megabyte + 1)).toBe(8);
+  const kilobyte = 1024;
+  const megabyte = 1024 * kilobyte;
+  expect(nativeProgramPartitions(128 * kilobyte)).toBe(1);
+  expect(nativeProgramPartitions(128 * kilobyte + 1)).toBe(8);
   expect(nativeProgramPartitions(32 * megabyte)).toBe(8);
   // Powers of two keep every function in its partition across small edits.
   expect(nativeProgramPartitions(32 * megabyte + 1)).toBe(16);
   expect(nativeProgramPartitions(100 * megabyte)).toBe(32);
   expect(nativeProgramPartitions(1024 * megabyte)).toBe(64);
   // Unoptimized (dev) programs are divided from 64 KiB.
-  const kilobyte = 1024;
   expect(nativeProgramPartitions(64 * kilobyte, true)).toBe(1);
   expect(nativeProgramPartitions(64 * kilobyte + 1, true)).toBe(4);
   expect(nativeProgramPartitions(128 * kilobyte, true)).toBe(4);
@@ -427,41 +453,6 @@ test("program partitions depend only on module size", () => {
   ]);
 });
 
-test("code generation parts depend only on module size", () => {
-  const kilobyte = 1024;
-  expect(nativeCodegenParts(128 * kilobyte)).toBe(1);
-  expect(nativeCodegenParts(128 * kilobyte + 1)).toBe(4);
-  expect(nativeCodegenParts(256 * kilobyte)).toBe(4);
-  expect(nativeCodegenParts(256 * kilobyte + 1)).toBe(8);
-  expect(nativeCodegenParts(1024 * kilobyte)).toBe(8);
-  // Partitioned modules already compile their partitions concurrently.
-  expect(nativeCodegenParts(1024 * kilobyte + 1)).toBe(1);
-});
-
-test("split code generation passes --split codegen and no partition cache", async () => {
-  const pkg = await fakePackage();
-  const output = join(pkg.root, "split", "program.o");
-  const artifact = await emitNativeArtifact({
-    ...request(pkg.root, pkg.packageJson, output),
-    partitions: 4,
-    splitCodegen: true,
-  });
-  expect(artifact.outputPaths).toEqual(nativePartitionPaths(output, 4));
-  const args = (await readFile(pkg.log, "utf8")).trim().split(" ");
-  expect(args.filter((arg) => arg === "--output")).toHaveLength(4);
-  expect(args.join(" ")).toContain("--split codegen");
-  expect(args).not.toContain("--cache-dir");
-
-  // Partitions of the same module are a distinct artifact.
-  await emitNativeArtifact({
-    ...request(pkg.root, pkg.packageJson, join(pkg.root, "partitions", "program.o")),
-    partitions: 4,
-  });
-  const calls = (await readFile(pkg.log, "utf8")).trim().split("\n");
-  expect(calls).toHaveLength(2);
-  expect(calls[1]).not.toContain("--split");
-});
-
 test("partitioned objects are emitted, cached and restored together", async () => {
   const pkg = await fakePackage();
   const first = join(pkg.root, "first", "program.o");

packages/compiler/src/backend/native-codegen.ts

@@ -125,9 +125,6 @@ export interface NativeCodegenOptions {
   cacheRoot?: string | null;
   /** Program partitions, written to nativePartitionPaths(outputPath). */
   partitions?: number;
-  /** With several partitions: optimize the module whole and divide only its
-   * code generation among them. */
-  splitCodegen?: boolean;
   /** Profile-guided optimization: instrumentation or an indexed profile. */
   profile?: NativeProfileOptions;
 }
@@ -354,7 +351,6 @@ function cacheKey(
     .update("\0")
     .update(options.sourcePath)
     .update((options.partitions ?? 1) > 1 ? `\0partitions:${options.partitions}` : "")
-    .update(options.splitCodegen === true ? "\0split:codegen" : "")
     .update(options.profile === undefined ? "" : `\0${nativeProfileKey(options.profile)}`)
     .digest("hex");
 }
@@ -429,8 +425,13 @@ export async function emitNativeArtifact(
     );
   }
   const helper = await resolveHelper(target, options.resolvePackageJson, options.helperHost);
+  // A build that records block counts runs the helper on every partition.
   const root = await prepareBuildCacheRoot(
-    options.cacheRoot === undefined ? buildCacheRoot() : options.cacheRoot,
+    options.profile?.recordBlocks !== undefined
+      ? null
+      : options.cacheRoot === undefined
+        ? buildCacheRoot()
+        : options.cacheRoot,
   );
   const key = cacheKey(options, target, helper.identity);
   const outputs = nativePartitionPaths(options.outputPath, options.partitions ?? 1);
@@ -458,7 +459,7 @@ export async function emitNativeArtifact(
   // edit recompiles only the partitions it affects. The directory names the
   // exact helper build and target; the helper keys entries by content.
   const partitionCache =
-    root === null || outputs.length < 2 || options.splitCodegen === true
+    root === null || outputs.length < 2
       ? null
       : join(
           root,
@@ -498,7 +499,6 @@ export async function emitNativeArtifact(
       "--source-path",
       options.sourcePath,
       ...(partitionCache === null ? [] : ["--cache-dir", partitionCache]),
-      ...(options.splitCodegen === true && outputs.length > 1 ? ["--split", "codegen"] : []),
       ...nativeProfileHelperArgs(options.profile),
     ]);
     for (const stage of stages) {
@@ -561,6 +561,7 @@ function helperProfileMatch(match: NativeProfileMatch): Record<string, number> {
     annotated: match.annotated,
     mismatched: match.mismatched,
     missing: match.missing,
+    recovered: match.recovered,
     unknown_partitions: match.unknownPartitions,
   };
 }
@@ -578,6 +579,7 @@ async function cachedProfileMatch(
       annotated: 0,
       mismatched: 0,
       missing: 0,
+      recovered: 0,
       unknownPartitions: partitions,
     }
   );

packages/compiler/src/backend/native-profile.test.ts

@@ -10,6 +10,8 @@ import {
   profileGenerateLinkerArgs,
   profileHasContextCounts,
   profileLayoutLinkerArgs,
+  profileBlocksPath,
+  recordProfileBlocksPath,
 } from "./native-profile.js";
 import { NATIVE_TARGETS } from "./targets.js";
 
@@ -127,11 +129,22 @@ test("only the instrumented link adds the profile runtime", () => {
 });
 
 test("the helper's profile match is read from its output", () => {
+  expect(
+    parseNativeProfileMatch(
+      '{"ok":true,"profile":{"annotated":120,"mismatched":7,"missing":40,"recovered":6,"unknown_partitions":1}}\n',
+    ),
+  ).toEqual({ annotated: 120, mismatched: 7, missing: 40, recovered: 6, unknownPartitions: 1 });
+  // A helper that predates block-count files reports no recovered count.
   expect(
     parseNativeProfileMatch(
       '{"ok":true,"profile":{"annotated":120,"mismatched":7,"missing":40,"unknown_partitions":1}}\n',
     ),
-  ).toEqual({ annotated: 120, mismatched: 7, missing: 40, unknownPartitions: 1 });
+  ).toEqual({ annotated: 120, mismatched: 7, missing: 40, recovered: 0, unknownPartitions: 1 });
+  expect(
+    parseNativeProfileMatch(
+      '{"ok":true,"profile":{"annotated":1,"mismatched":0,"missing":0,"recovered":-1,"unknown_partitions":0}}\n',
+    ),
+  ).toBeNull();
   // A helper that predates the report prints nothing.
   expect(parseNativeProfileMatch("")).toBeNull();
   expect(parseNativeProfileMatch('{"ok":true}\n')).toBeNull();
@@ -144,6 +157,7 @@ test("the build reports the match, and warns when the profile is stale", () => {
       annotated: 9170,
       mismatched: 0,
       missing: 28636,
+      recovered: 0,
       unknownPartitions: 0,
     }),
   ).toBe(
@@ -155,6 +169,7 @@ test("the build reports the match, and warns when the profile is stale", () => {
       annotated: 90,
       mismatched: 10,
       missing: 5,
+      recovered: 0,
       unknownPartitions: 1,
     }),
   ).toBe(
@@ -165,6 +180,7 @@ test("the build reports the match, and warns when the profile is stale", () => {
       annotated: 5208,
       mismatched: 3739,
       missing: 28859,
+      recovered: 0,
       unknownPartitions: 2,
     }),
   ).toBe(
@@ -177,9 +193,69 @@ test("the build reports the match, and warns when the profile is stale", () => {
       annotated: 0,
       mismatched: 0,
       missing: 12,
+      recovered: 0,
       unknownPartitions: 0,
     }),
   ).toBe(
     "scriptc: profile app.profdata: 0 functions annotated, 0 mismatched, 12 without a record\n",
   );
+  // Functions the block-count file annotated have counts, so they leave the warning's share.
+  expect(
+    renderNativeProfileMatch("app.profdata", {
+      annotated: 5208,
+      mismatched: 3739,
+      missing: 28859,
+      recovered: 3600,
+      unknownPartitions: 0,
+    }),
+  ).toBe(
+    "scriptc: profile app.profdata: 5208 functions annotated, 3739 mismatched, 28859 without a record\n" +
+      "scriptc: profile app.profdata: 3600 functions without profile counts annotated from app.blocks\n",
+  );
+  expect(
+    renderNativeProfileMatch("app.profdata", {
+      annotated: 5208,
+      mismatched: 3739,
+      missing: 28859,
+      recovered: 1,
+      unknownPartitions: 0,
+    }),
+  ).toBe(
+    "scriptc: profile app.profdata: 5208 functions annotated, 3739 mismatched, 28859 without a record\n" +
+      "scriptc: profile app.profdata: 1 function without profile counts annotated from app.blocks\n" +
+      "scriptc: warning: 3738 of the 8947 functions app.profdata describes (41.8%) changed since it was collected and are optimized without their counts; collect the profile again\n",
+  );
+});
+
+test("the block-count file sits beside the profile", () => {
+  expect(profileBlocksPath("/p/tsc-ts.cs.profdata")).toBe("/p/tsc-ts.cs.blocks");
+  expect(profileBlocksPath("/p/cli.profile")).toBe("/p/cli.profile.blocks");
+  expect(recordProfileBlocksPath({})).toBeUndefined();
+  expect(recordProfileBlocksPath({ SCRIPTC_RECORD_PROFILE_BLOCKS: "" })).toBeUndefined();
+  expect(recordProfileBlocksPath({ SCRIPTC_RECORD_PROFILE_BLOCKS: "/p/a.blocks" })).toBe(
+    "/p/a.blocks",
+  );
+});
+
+test("the block-count file reaches the helper and keys the caches", () => {
+  const use = { path: "/p/a.profdata", digest: "d", contextSensitive: false };
+  const blocks = { path: "/p/a.blocks", digest: "b" };
+  expect(nativeProfileHelperArgs({ use: { ...use, blocks } })).toEqual([
+    "--profile-use",
+    "/p/a.profdata",
+    "--profile-blocks",
+    "/p/a.blocks",
+  ]);
+  expect(nativeProfileHelperArgs({ use, recordBlocks: "/w/a.blocks" })).toEqual([
+    "--profile-use",
+    "/p/a.profdata",
+    "--record-profile-blocks",
+    "/w/a.blocks",
+  ]);
+  // Recording needs a profile to record from.
+  expect(
+    nativeProfileHelperArgs({ generate: "/r/%p.profraw", recordBlocks: "/w/a.blocks" }),
+  ).toEqual(["--profile-generate", "/r/%p.profraw"]);
+  expect(nativeProfileKey({ use })).toBe("profile-use:d");
+  expect(nativeProfileKey({ use: { ...use, blocks } })).toBe("profile-use:d\0profile-blocks:b");
 });

packages/compiler/src/backend/native-profile.ts

@@ -11,6 +11,14 @@
  * and merging its raw profiles into the first profile adds the counts of the
  * inlined code. A profile with those counts also orders each function's
  * blocks by them and moves cold blocks to separate sections.
+ *
+ * A function whose control flow changed since the profile was collected
+ * takes none of its counts. A block-count file beside the profile
+ * (profileBlocksPath) describes each function by its blocks' labels instead,
+ * and annotates those functions: blocks that keep their place take their
+ * recorded counts, and the counts of the others follow from flow
+ * conservation. A `--profile-use` build writes that file for the functions
+ * its profile annotates when SCRIPTC_RECORD_PROFILE_BLOCKS names its path.
  */
 import type { NativeOptimization } from "./optimization.js";
 
@@ -19,9 +27,40 @@ export interface NativeProfileOptions {
    * `use`, the instrumentation runs after inlining and records
    * context-sensitive counts. */
   generate?: string;
-  /** Indexed profile, the digest of its bytes (which key every cache), and
-   * whether it holds context-sensitive counts. */
-  use?: { path: string; digest: string; contextSensitive: boolean };
+  /** Indexed profile, the digest of its bytes (which key every cache),
+   * whether it holds context-sensitive counts, and its block-count file
+   * with the digest of its bytes when there is one. */
+  use?: {
+    path: string;
+    digest: string;
+    contextSensitive: boolean;
+    blocks?: { path: string; digest: string };
+  };
+  /** With `use`: the path the build writes the block counts of the
+   * functions the profile annotates to. Such a build reuses no cached
+   * artifacts. */
+  recordBlocks?: string;
+}
+
+/** The environment variable that names the block-count file a
+ * `--profile-use` build writes. */
+export const RECORD_PROFILE_BLOCKS_ENV = "SCRIPTC_RECORD_PROFILE_BLOCKS";
+
+/** The block-count file beside a profile: its path with `.blocks` in place
+ * of a `.profdata` extension, or with `.blocks` appended. */
+export function profileBlocksPath(profilePath: string): string {
+  return profilePath.endsWith(".profdata")
+    ? `${profilePath.slice(0, -".profdata".length)}.blocks`
+    : `${profilePath}.blocks`;
+}
+
+/** The block-count file a `--profile-use` build writes, from the
+ * environment, or undefined. */
+export function recordProfileBlocksPath(
+  env: Record<string, string | undefined>,
+): string | undefined {
+  const path = env[RECORD_PROFILE_BLOCKS_ENV];
+  return path === undefined || path === "" ? undefined : path;
 }
 
 /** Whether an indexed profile holds context-sensitive counts: the header's
@@ -59,18 +98,25 @@ export function nativeProfileHelperArgs(profile: NativeProfileOptions | undefine
   return [
     ...(profile?.generate === undefined ? [] : ["--profile-generate", profile.generate]),
     ...(profile?.use === undefined ? [] : ["--profile-use", profile.use.path]),
+    ...(profile?.use?.blocks === undefined ? [] : ["--profile-blocks", profile.use.blocks.path]),
+    ...(profile?.use === undefined || profile.recordBlocks === undefined
+      ? []
+      : ["--record-profile-blocks", profile.recordBlocks]),
   ];
 }
 
 /** How an optimized build's profile matched the program's functions, as the
  * helper reports it: functions that took the profile's counts, functions
  * whose control flow changed since the profile was collected, functions the
- * profile has no record of (new, or never run while it was collected), and
- * partitions reused from the cache without their counts. */
+ * profile has no record of (new, or never run while it was collected),
+ * mismatched functions that the block-count file annotated (never more
+ * than the mismatched ones), and partitions reused from the cache without
+ * their counts. */
 export interface NativeProfileMatch {
   annotated: number;
   mismatched: number;
   missing: number;
+  recovered: number;
   unknownPartitions: number;
 }
 
@@ -93,10 +139,18 @@ export function parseNativeProfileMatch(stdout: string): NativeProfileMatch | nu
     const annotated = count("annotated");
     const mismatched = count("mismatched");
     const missing = count("missing");
+    // A helper that predates block-count files reports no recovered count.
+    const recovered = profile["recovered"] === undefined ? 0 : count("recovered");
     const unknownPartitions = count("unknown_partitions");
-    if (annotated === null || mismatched === null || missing === null || unknownPartitions === null)
+    if (
+      annotated === null ||
+      mismatched === null ||
+      missing === null ||
+      recovered === null ||
+      unknownPartitions === null
+    )
       continue;
-    return { annotated, mismatched, missing, unknownPartitions };
+    return { annotated, mismatched, missing, recovered, unknownPartitions };
   }
   return null;
 }
@@ -107,9 +161,15 @@ export function parseNativeProfileMatch(stdout: string): NativeProfileMatch | nu
 export const STALE_PROFILE_SHARE = 0.1;
 
 /** `scriptc build --profile-use`'s report on stderr: one line with the match,
- * and a warning line when more than STALE_PROFILE_SHARE of the functions the
- * profile describes changed since it was collected. */
-export function renderNativeProfileMatch(profile: string, match: NativeProfileMatch): string {
+ * a line with the functions the block-count file annotated when there are
+ * any, and a warning line when more than STALE_PROFILE_SHARE of the
+ * functions the profile describes changed since it was collected and took
+ * no counts. */
+export function renderNativeProfileMatch(
+  profile: string,
+  match: NativeProfileMatch,
+  blocks: string = profileBlocksPath(profile),
+): string {
   const unknown =
     match.unknownPartitions === 0
       ? ""
@@ -117,11 +177,16 @@ export function renderNativeProfileMatch(profile: string, match: NativeProfileMa
   const lines = [
     `scriptc: profile ${profile}: ${match.annotated} functions annotated, ${match.mismatched} mismatched, ${match.missing} without a record${unknown}`,
   ];
-  const described = match.annotated + match.mismatched;
-  if (described > 0 && match.mismatched / described > STALE_PROFILE_SHARE) {
-    const percent = ((match.mismatched / described) * 100).toFixed(1);
+  if (match.recovered > 0)
     lines.push(
-      `scriptc: warning: ${match.mismatched} of the ${described} functions ${profile} describes (${percent}%) changed since it was collected and are optimized without their counts; collect the profile again`,
+      `scriptc: profile ${profile}: ${match.recovered} function${match.recovered === 1 ? "" : "s"} without profile counts annotated from ${blocks}`,
+    );
+  const described = match.annotated + match.mismatched;
+  const uncounted = Math.max(0, match.mismatched - match.recovered);
+  if (described > 0 && uncounted / described > STALE_PROFILE_SHARE) {
+    const percent = ((uncounted / described) * 100).toFixed(1);
+    lines.push(
+      `scriptc: warning: ${uncounted} of the ${described} functions ${profile} describes (${percent}%) changed since it was collected and are optimized without their counts; collect the profile again`,
     );
   }
   return `${lines.join("\n")}\n`;
@@ -132,6 +197,7 @@ export function nativeProfileKey(profile: NativeProfileOptions | undefined): str
   return [
     ...(profile?.generate === undefined ? [] : [`profile-generate:${profile.generate}`]),
     ...(profile?.use === undefined ? [] : [`profile-use:${profile.use.digest}`]),
+    ...(profile?.use?.blocks === undefined ? [] : [`profile-blocks:${profile.use.blocks.digest}`]),
   ].join("\0");
 }
 

packages/compiler/src/backend/native-tools.ts

@@ -49,9 +49,6 @@ export function emitNativeObject(options: {
   outputKind?: NativeCodegenOutputKind;
   /** Program partitions, written to nativePartitionPaths(outputPath). */
   partitions?: number;
-  /** With several partitions: optimize the module whole and divide only its
-   * code generation among them. */
-  splitCodegen?: boolean;
   /** Private directory where the helper reuses partition artifacts. */
   partitionCache?: string;
   /** Profile-guided optimization: instrumentation or an indexed profile. */
@@ -77,11 +74,9 @@ export function emitNativeObject(options: {
     "json",
     "--source-path",
     options.sourcePath,
-    ...(options.splitCodegen === true && outputs.length > 1
-      ? ["--split", "codegen"]
-      : options.partitionCache === undefined || outputs.length < 2
-        ? []
-        : ["--cache-dir", options.partitionCache]),
+    ...(options.partitionCache === undefined || outputs.length < 2
+      ? []
+      : ["--cache-dir", options.partitionCache]),
     ...nativeProfileHelperArgs(options.profile),
   ]);
   for (const output of outputs) requireNativeArtifact(output);

packages/compiler/src/cli/usage.ts

@@ -102,7 +102,8 @@ Options:
                      code is laid out, inlined and split by how the profiled
                      runs used it. Code the profile does not describe, or
                      describes from an older version of the program, is
-                     optimized as without one. build prints how the profile
+                     optimized as without one, unless a .blocks file beside
+                     the profile describes it. build prints how the profile
                      matched on stderr, and warns when more than 10% of the
                      functions it describes changed (release or speed;
                      --emit=exe only)

packages/compiler/src/compat/fetch-profile.ts

@@ -938,6 +938,9 @@ export interface StaticFetchMembers {
   headersCalls: ReadonlySet<string>;
   readableStreamReads: ReadonlySet<string>;
   readableStreamCalls: ReadonlySet<string>;
+  /** ReadableStream prototype members the native stream does not
+   * implement, symbol-keyed ones as `[Symbol.name]`. */
+  readableStreamUnsupported: ReadonlySet<string>;
 }
 
 let staticFetchMemberSets: StaticFetchMembers | undefined;
@@ -961,6 +964,16 @@ export function staticFetchMembers(): StaticFetchMembers {
     ),
     readableStreamReads: new Set(profile.members.readableStreamReads),
     readableStreamCalls: new Set(profile.members.readableStreamCalls),
+    readableStreamUnsupported: new Set(
+      profile.inventory.entries
+        .filter(
+          (entry) =>
+            entry.owner === "ReadableStream" &&
+            (entry.placement === "prototype" || entry.placement === "prototype-symbol") &&
+            entry.status === "unsupported",
+        )
+        .map((entry) => entry.member),
+    ),
   };
   return staticFetchMemberSets;
 }

packages/compiler/src/compile-types.ts

@@ -86,7 +86,8 @@ export interface CompileBaseOptions {
   /** Eligible packages compile automatically. Explicit names override automatic selection heuristics. */
   npmStatic?: readonly string[] | "auto";
   /** Shape inference's typed overlay for opted-in packages' JavaScript.
-   * False disables it (`--no-shape-overlay`); a package opts out with
+   * True enables it, false disables it (`--no-shape-overlay`); unset
+   * follows SCRIPTC_SHAPE_OVERLAY. A package opts out with
    * `"scriptc": { "packages": { "<name>": { "shapes": false } } }`. */
   shapeOverlay?: boolean;
   /** Outbound native FFI manifest. Its signature-only TypeScript bindings

packages/compiler/src/contracts/check.ts

@@ -67,6 +67,8 @@ import {
   type ContractAnnotations,
   type ContractNote,
 } from "../frontend/contract-types.js";
+import { planInlineArrays } from "../ir/inline-arrays.js";
+import { checkInlineArraySources } from "../ir/inline-array-sources.js";
 
 /** The largest array index: 2^32 - 2 (ECMA-262 array index). */
 const MAX_ARRAY_INDEX = 2 ** 32 - 2;
@@ -255,6 +257,7 @@ class ContractChecker {
 
   run(): ContractCheckResult {
     this.checkProgramLifetime();
+    this.checkInlineArrays();
     this.checkNoThrow();
     this.checkAcyclic();
     const elements = new Int32ElementContracts(this);
@@ -323,6 +326,77 @@ class ContractChecker {
     this.record("programLifetime", roots);
   }
 
+  /* ── InlineArray ── */
+
+  /** `readonly f: InlineArray<C>` (experimental, ir/inline-arrays.ts): the
+   * field's elements may live in the instance itself, which only a field
+   * written once from a constructor argument and never again can have.
+   * Checked: the class and C are ProgramLifetime classes, the constructor of
+   * the declaring class writes the field once from one of its parameters,
+   * every subclass constructor passes it one of its own through super(...),
+   * nothing else writes the field, no read of it is compared by identity
+   * (two reads need not give the same array), and no instance of the
+   * hierarchy is constructed through a class value or becomes a dynamic
+   * value. */
+  private checkInlineArrays(): void {
+    if (this.notes.inlineArrays.size === 0) return;
+    const roots = new Set(this.contracts.programLifetime ?? []);
+    const lifetime = (className: string): boolean => {
+      let cls = this.classes.get(className);
+      while (cls?.base !== undefined) cls = this.classes.get(cls.base);
+      return cls !== undefined && roots.has(cls.name);
+    };
+    const entries: string[] = [];
+    const notes = new Map<string, ContractNote>();
+    for (const [key, note] of this.notes.inlineArrays) {
+      const at = key.lastIndexOf("\0");
+      const field = key.slice(at + 1);
+      for (const cls of this.classesAt(key.slice(0, at))) {
+        if (!lifetime(cls.name)) {
+          this.report(
+            "SC7009",
+            note.loc,
+            `'${note.name}' is an InlineArray field of a class that is not ProgramLifetime`,
+            "declare the root class of the hierarchy `implements ProgramLifetime`: inline elements live in the instance, which is never freed",
+          );
+          continue;
+        }
+        const entry = `${cls.name}\0${field}`;
+        entries.push(entry);
+        notes.set(entry, note);
+      }
+    }
+    if (entries.length === 0) return;
+    const plan = planInlineArrays(
+      { ...this.mod, contracts: { ...this.mod.contracts, inlineArrays: entries } },
+      lifetime,
+    );
+    for (const r of plan.rejected) {
+      const note = notes.get(`${r.declaringClass}\0${r.field}`)!;
+      const at = r.loc === undefined ? "" : ` (at ${this.where(r.loc)})`;
+      this.report(
+        "SC7009",
+        note.loc,
+        `'${note.name}' cannot keep its InlineArray contract: ${r.reason}${at}`,
+        "write the field once in the constructor from one of its parameters (subclasses pass theirs through super(...)), never again, and keep the class's instances out of class values and dynamic values",
+      );
+    }
+    // Where the arrays come from and who writes them (ir/inline-array-sources.ts).
+    for (const issue of checkInlineArraySources(this.mod, plan)) {
+      const note = notes.get(`${issue.declaringClass}\0${issue.field}`)!;
+      const from = issue.origin === undefined ? "" : ` (from ${this.where(issue.origin)})`;
+      this.report(
+        "SC7009",
+        issue.loc,
+        `'${note.name}' (InlineArray): ${issue.reason}${from}`,
+        "an InlineArray field holds its array for good: build a new array for each constructor call, and never write to an array a field holds",
+      );
+    }
+    const kept = new Set<string>();
+    for (const cls of plan.classes.values()) kept.add(`${cls.declaringClass}\0${cls.field}`);
+    this.record("inlineArrays", [...kept]);
+  }
+
   /* ── Acyclic ── */
 
   private checkAcyclic(): void {

packages/compiler/src/diagnostics/codes.ts

@@ -107,6 +107,7 @@ export const DIAGNOSTIC_CODES = {
   SC7006: "Acyclic contract in an unchecked position",
   SC7007: "Acyclic type can lie on a reference cycle",
   SC7008: "ProgramLifetime class extends another class",
+  SC7009: "InlineArray field contract not kept",
   SC8001: "package or module runs in a Node island (note)",
   SC8002: "Node islands off for a package or module outside the native subset",
   SC8003: "package marked native is outside the native subset",

packages/compiler/src/diagnostics/diagnostic.ts

@@ -71,7 +71,7 @@
  *            build checks (SC6103, frontend/shape-inference)
  *   SC7xxx  contract types (@scriptc/types): a NoThrow, Int32, Immutable,
  *            Acyclic or ProgramLifetime contract that does not hold or
- *            cannot be checked (SC7001–SC7008, contracts/check.ts)
+ *            cannot be checked (SC7001–SC7009, contracts/check.ts)
  *   SC8xxx  Node islands: packages and modules outside the native subset
  *            that run (or, in report-only mode, would run) in a Node
  *            island — unknown project configuration (SC8004), a package

packages/compiler/src/executable/executable-cache.ts

@@ -254,7 +254,7 @@ export interface EarlyExecutableCachePublish extends EarlyExecutableCacheHit {
 function isProfileMatch(value: unknown): value is NativeProfileMatch {
   if (typeof value !== "object" || value === null) return false;
   const match = value as Record<string, unknown>;
-  return ["annotated", "mismatched", "missing", "unknownPartitions"].every((key) => {
+  return ["annotated", "mismatched", "missing", "recovered", "unknownPartitions"].every((key) => {
     const count = match[key];
     return typeof count === "number" && Number.isInteger(count) && count >= 0;
   });

packages/compiler/src/executable/prepare.ts

@@ -121,12 +121,15 @@ export function prepareExecutableModule(
     }
 
     try {
-      lowered = fe.lower({
-        ...(islandDecision !== null ? { nodeIslands: islandDecision } : {}),
-        targetPlatform: buildPlatform,
-        ...(ffi !== null ? { ffiImports: ffi.functions } : {}),
-        ...(opts.tier !== undefined ? { tierMode: opts.tier } : {}),
-      });
+      lowered = fe.lower(
+        {
+          ...(islandDecision !== null ? { nodeIslands: islandDecision } : {}),
+          targetPlatform: buildPlatform,
+          ...(ffi !== null ? { ffiImports: ffi.functions } : {}),
+          ...(opts.tier !== undefined ? { tierMode: opts.tier } : {}),
+        },
+        (module) => validateModule(module).length === 0,
+      );
       timing("lower");
     } catch (e) {
       // The last-resort panic fence: an upstream tsgo panic that crossed a

packages/compiler/src/frontend/contract-types.ts

@@ -16,15 +16,33 @@ import { type HeritageElement, isHeritageElement } from "./heritage-elements.js"
 
 export const CONTRACT_TYPES_MODULE = "@scriptc/types";
 
-export type ContractMarker = "Int32" | "NoThrow" | "Immutable" | "Acyclic" | "ProgramLifetime";
+/** The module and name of the inline array field marker, which is
+ * experimental: `readonly f: InlineArray<C>` on an instance field of a
+ * ProgramLifetime class (ir/inline-arrays.ts). */
+export const INLINE_ARRAY_MODULE = "@scriptc/experimental";
+export const INLINE_ARRAY_MARKER = "InlineArray";
+
+export type ContractMarker =
+  | "Int32"
+  | "NoThrow"
+  | "Immutable"
+  | "Acyclic"
+  | "ProgramLifetime"
+  | typeof INLINE_ARRAY_MARKER;
 const MARKERS: ReadonlySet<string> = new Set([
   "Int32",
   "NoThrow",
   "Immutable",
   "Acyclic",
   "ProgramLifetime",
+  INLINE_ARRAY_MARKER,
 ]);
 
+/** The module a marker is imported from. */
+function markerModule(marker: string): string {
+  return marker === INLINE_ARRAY_MARKER ? INLINE_ARRAY_MODULE : CONTRACT_TYPES_MODULE;
+}
+
 /** One written contract: what it promises and where the annotation is. */
 export interface ContractNote {
   marker: ContractMarker;
@@ -67,6 +85,9 @@ export interface ContractAnnotations {
   acyclic: AcyclicNote[];
   /** ProgramLifetime classes (`implements ProgramLifetime`) by class key. */
   classes: Map<string, ContractNote>;
+  /** Inline array fields (`readonly f: InlineArray<C>`) by
+   * `${class key}\0${field name}`. */
+  inlineArrays: Map<string, ContractNote>;
   diagnostics: ScrDiagnostic[];
   /** Number of annotations found (for timing and tests). */
   count: number;
@@ -123,7 +144,7 @@ function markerNamed(
     importDecl === undefined ||
     !ts.isImportDeclaration(importDecl) ||
     !ts.isStringLiteral(importDecl.moduleSpecifier) ||
-    importDecl.moduleSpecifier.text !== CONTRACT_TYPES_MODULE
+    importDecl.moduleSpecifier.text !== markerModule(member)
   )
     return null;
   return MARKERS.has(member) ? (member as ContractMarker) : null;
@@ -158,6 +179,7 @@ export function collectContractAnnotations(
     fields: new Map(),
     acyclic: [],
     classes: new Map(),
+    inlineArrays: new Map(),
     diagnostics: [],
     count: 0,
   };
@@ -166,7 +188,11 @@ export function collectContractAnnotations(
     ts.SyntaxKind.ExpressionWithTypeArguments,
   ]);
   for (const sf of files) {
-    if (sf.isDeclarationFile || !sf.text.includes(CONTRACT_TYPES_MODULE)) continue;
+    if (
+      sf.isDeclarationFile ||
+      (!sf.text.includes(CONTRACT_TYPES_MODULE) && !sf.text.includes(INLINE_ARRAY_MODULE))
+    )
+      continue;
     ts.walkPreorder(
       sf,
       (node) => {
@@ -183,6 +209,7 @@ export function collectContractAnnotations(
         out.count++;
         if (marker === "Acyclic") placeAcyclicAlias(out, checker, node, resolveUnit);
         else if (marker === "ProgramLifetime") unsupported(out, node, marker);
+        else if (marker === INLINE_ARRAY_MARKER) placeInlineArray(out, node);
         else place(out, checker, node, marker);
       },
       kinds,
@@ -203,6 +230,17 @@ function unsupported(out: ContractAnnotations, node: ts.Node, marker: ContractMa
     );
     return;
   }
+  if (marker === INLINE_ARRAY_MARKER) {
+    out.diagnostics.push(
+      contractDiag(
+        "SC7006",
+        locOf(node),
+        "the InlineArray contract is not checked in this position (InlineArray is checked on instance fields of ProgramLifetime classes)",
+        "write it directly as the type of an instance field: readonly f: InlineArray<C>",
+      ),
+    );
+    return;
+  }
   const where =
     marker === "NoThrow"
       ? "NoThrow wraps a function's written return type"
@@ -282,6 +320,28 @@ function placeHeritage(
   unsupported(out, node, marker);
 }
 
+/** `readonly f: InlineArray<C>`: an instance field of a class. */
+function placeInlineArray(out: ContractAnnotations, ref: ts.TypeReferenceNode): void {
+  const parent = ref.parent;
+  if (
+    parent === undefined ||
+    !ts.isPropertyDeclaration(parent) ||
+    parent.type !== ref ||
+    ref.typeArguments?.nodes.length !== 1 ||
+    ts.getModifiers(parent)?.some((m) => m.kind === ts.SyntaxKind.StaticKeyword) === true ||
+    !(ts.isIdentifier(parent.name) || ts.isPrivateIdentifier(parent.name))
+  )
+    return unsupported(out, ref, INLINE_ARRAY_MARKER);
+  const cls = parent.parent;
+  if (cls === undefined || !(ts.isClassDeclaration(cls) || ts.isClassExpression(cls)))
+    return unsupported(out, ref, INLINE_ARRAY_MARKER);
+  out.inlineArrays.set(`${contractLocKey(locOf(cls))}\0${parent.name.text}`, {
+    marker: INLINE_ARRAY_MARKER,
+    loc: locOf(ref),
+    name: `${nameText(cls.name)}.${parent.name.text}`,
+  });
+}
+
 /** `type T = Acyclic<{ ... }>`: the aliased object type. */
 function placeAcyclicAlias(
   out: ContractAnnotations,

packages/compiler/src/frontend/input-tracker.ts

@@ -173,6 +173,16 @@ export class FrontendInputTracker {
     }
   }
 
+  /** Raw file names whose source text this tracker recorded. */
+  private readonly notedTexts = new Set<string>();
+
+  /** Records a raw read of a source file's text, once per file name. */
+  noteText(fileName: string): void {
+    if (this.notedTexts.has(fileName)) return;
+    this.notedTexts.add(fileName);
+    this.record({ op: "text", path: resolve(fileName) });
+  }
+
   record(probe: FrontendInputProbe): void {
     const key = `${probe.op}\0${probe.path}`;
     if (probe.op === "text" && this.probes.has(key)) return;
@@ -238,7 +248,7 @@ function record(probe: FrontendInputProbe): void {
 // callbacks, line numbers) changes with edits that keep the tokens.
 observeSourceText((fileName) => {
   const tracker = synchronousTracker ?? activeTracker.getStore();
-  if (tracker !== undefined) tracker.record({ op: "text", path: resolve(fileName) });
+  if (tracker !== undefined) tracker.noteText(fileName);
 });
 
 /** Mark the active frontend as unsafe to persist because an exact host query

packages/compiler/src/frontend/lowering/array-values.ts

@@ -260,6 +260,9 @@ const PURE_KINDS = new Set<IrExpr["kind"]>([
   "unionIsTag",
   "unionNarrow",
   "unionWrap",
+  // Its statements are checked on their own (declarations and assignments
+  // are recorded writes).
+  "seqExpr",
 ]);
 
 /** The locals evaluating `value` may assign, and whether it may run other
@@ -273,7 +276,11 @@ function valueEffects(lowerer: Lowerer, value: IrExpr): { writes: Set<string>; o
   everyExpr(value, {
     expr: (e) => {
       if (e.kind === "assignExpr" || e.kind === "incDec") writes.add(e.localId);
-      else if (e.kind === "call" ? !readers.has(e.callee) : !PURE_KINDS.has(e.kind)) opaque = true;
+      else if (e.kind === "libCall") {
+        // The "undefined" key of an undefined index is a constant.
+        if (e.fn !== "array.undefinedKey") opaque = true;
+      } else if (e.kind === "call" ? !readers.has(e.callee) : !PURE_KINDS.has(e.kind))
+        opaque = true;
       return true;
     },
     stmt: (s) => {
@@ -314,6 +321,16 @@ function readsMutableLocal(lowerer: Lowerer, e: IrExpr): boolean {
   });
 }
 
+/** Whether `receiver` still reads the same value after `index` runs: the
+ * index assigns none of the bindings it reads, and runs no code that could
+ * assign them through a closure. */
+export function unaffectedByIndex(lowerer: Lowerer, receiver: IrExpr, index: IrExpr): boolean {
+  const effects = valueEffects(lowerer, index);
+  return (
+    unaffectedBy(receiver, effects) && !(effects.opaque && readsMutableLocal(lowerer, receiver))
+  );
+}
+
 /** A number, string or boolean element type. Arrays of it hold a present
  * undefined in the element's state, so `(T | undefined)[]` stores T
  * elements (the type mapper) and a T[] seen as one stays the same array. */

packages/compiler/src/frontend/lowering/builtins/array-classes.ts

@@ -647,26 +647,27 @@ export function lowerWithArraySpecies(
     // instance is created (with that length for map, 0 otherwise), then
     // the method runs to that length and its elements move into the
     // instance.
-    const bounded =
-      result.kind === "call" &&
-      result.args[0]?.kind === "varRef" &&
-      result.args[0].localId === local.id
-        ? boundedArrayHofHelper(lowerer, result.callee)
-        : null;
-    if (bounded === null || result.kind !== "call")
+    const stmts: IrStmt[] = [{ kind: "varDecl", localId: local.id, init: receiver, loc }];
+    let bound: IrLocal | undefined;
+    const rebound = rebuildSpeciesCall(result, local.id, (inner) => {
+      const args: IrExpr[] = [receiverRef];
+      for (const arg of inner.args.slice(1)) {
+        const held = lowerer.declareHiddenLocal("%speciesArgument", arg.type);
+        stmts.push({ kind: "varDecl", localId: held.id, init: arg, loc });
+        args.push({ kind: "varRef", localId: held.id, type: arg.type, loc });
+      }
+      const length = lowerer.declareHiddenLocal("%speciesLength", F64);
+      bound = length;
+      args.push({ kind: "varRef", localId: length.id, type: F64, loc });
+      const helper = boundedArrayHofHelper(lowerer, inner.callee);
+      return helper === null ? null : { ...inner, callee: helper, args };
+    });
+    if (rebound === null || bound === undefined)
       lowerer.unsupported(
         "SC1090",
         call,
         `'${access.name.text}' in this form when the program declares a class that extends Array`,
       );
-    const stmts: IrStmt[] = [{ kind: "varDecl", localId: local.id, init: receiver, loc }];
-    const args: IrExpr[] = [receiverRef];
-    for (const arg of result.args.slice(1)) {
-      const held = lowerer.declareHiddenLocal("%speciesArgument", arg.type);
-      stmts.push({ kind: "varDecl", localId: held.id, init: arg, loc });
-      args.push({ kind: "varRef", localId: held.id, type: arg.type, loc });
-    }
-    const bound = lowerer.declareHiddenLocal("%speciesLength", F64);
     stmts.push({
       kind: "varDecl",
       localId: bound.id,
@@ -681,7 +682,6 @@ export function lowerWithArraySpecies(
       loc,
     });
     const boundRef: IrExpr = { kind: "varRef", localId: bound.id, type: F64, loc };
-    args.push(boundRef);
     const instance = lowerer.declareHiddenLocal("%speciesInstance", receiver.type);
     const length: IrExpr = mode.resultLength
       ? boundRef
@@ -706,7 +706,7 @@ export function lowerWithArraySpecies(
         fn: "array.speciesFinish",
         args: [
           { kind: "varRef", localId: instance.id, type: receiver.type, loc },
-          { ...result, callee: bounded, args },
+          rebound,
           { kind: "boolLit", value: mode.setLength, type: BOOL, loc },
         ],
         type: result.type,
@@ -736,6 +736,31 @@ export function lowerWithArraySpecies(
   };
 }
 
+/** `e` with the `%arr.<method>` helper call over `receiverLocal` rebuilt by
+ * `rebuild`, or null when `e` holds no such call. The helper call is the
+ * one taking the receiver as its first argument; a filter whose callback
+ * narrows converts the filtered array after the helper ran, so the call
+ * can sit behind a wrapper (the checked-narrowing call), which this
+ * rebuild leaves in place. */
+function rebuildSpeciesCall(
+  e: IrExpr,
+  receiverLocal: string,
+  rebuild: (inner: IrExpr & { kind: "call" }) => IrExpr | null,
+): IrExpr | null {
+  if (e.kind !== "call") return null;
+  const first = e.args[0];
+  if (first !== undefined && first.kind === "varRef" && first.localId === receiverLocal)
+    return rebuild(e);
+  for (let i = 0; i < e.args.length; i++) {
+    const replaced = rebuildSpeciesCall(e.args[i]!, receiverLocal, rebuild);
+    if (replaced === null) continue;
+    const args = e.args.slice();
+    args[i] = replaced;
+    return { ...e, args };
+  }
+  return null;
+}
+
 /** `C.of(...items)` and `C.from(iterable)` on a class declared `extends
  * Array` that declares neither: Array.of / Array.from's elements, in an
  * instance the class constructs (`new C(items.length)` for of, `new C()`

packages/compiler/src/frontend/lowering/builtins/filesystem-calls.ts

@@ -997,6 +997,42 @@ export function lowerFsWriteBytesCall(
   return null;
 }
 
+// The writeFileSync(fd, data[, "utf8"]) form — Node accepts a file
+// descriptor where it accepts a path (write to a descriptor openSync
+// returned, the descriptor staying open: closeSync owns its lifetime).
+// Routed by the ARGUMENT's static type, like readFileSync(fd).
+export function lowerFsWriteDescriptorCall(
+  lowerer: Lowerer,
+  expr: ts.CallExpression,
+  loc: SrcLoc,
+): IrExpr {
+  const supported = 'only string data is supported: writeFileSync(fd, data, "utf8")';
+  const dataNode = expr.arguments.nodes[1]!;
+  const dataIr = lowerer.mapTypeOf(lowerer.typeOf(dataNode));
+  if (dataIr?.kind !== "string") {
+    lowerer.noLowering(
+      `writeFileSync(fd) of '${dataIr ? lowerer.fmt(dataIr) : lowerer.checker.typeToString(lowerer.typeOf(dataNode))}' data`,
+      dataNode,
+      supported,
+    );
+  }
+  if (expr.arguments.nodes.length > 3) {
+    lowerer.noLowering(
+      `writeFileSync(fd) with ${expr.arguments.nodes.length} arguments`,
+      expr,
+      supported,
+    );
+  }
+  const fd = lowerer.lowerExprExpecting(expr.arguments.nodes[0]!, F64);
+  return {
+    kind: "libCall",
+    fn: "fs.writeFileSyncFd",
+    args: [fd, lowerer.lowerExprExpecting(dataNode, STRING)],
+    type: VOID,
+    loc,
+  };
+}
+
 export function lowerFsWriteOptionsCall(
   lowerer: Lowerer,
   expr: ts.CallExpression,

packages/compiler/src/frontend/lowering/builtins/module-calls.ts

@@ -14,6 +14,7 @@ import {
   lowerFsReadSyncCall,
   lowerFsWriteSyncCall,
   lowerFsReadDescriptorCall,
+  lowerFsWriteDescriptorCall,
   lowerFsEncodedRead,
   lowerFsPromisesEncodedRead,
   lowerFsMkdirSyncCall,
@@ -331,6 +332,17 @@ export function lowerBuiltinModuleCall(
     const lowered = lowerFsWriteBytesCall(lowerer, expr, bi, loc);
     if (lowered) return lowered;
   }
+  // writeFileSync(fd, data): Node accepts an open descriptor where it
+  // accepts a path. Routed by the FIRST ARGUMENT's static type, like
+  // readFileSync(fd) (before the path forms complete the data type).
+  if (
+    bi.module === "fs" &&
+    bi.member === "writeFileSync" &&
+    expr.arguments.nodes.length >= 2 &&
+    lowerer.mapTypeOf(lowerer.typeOf(expr.arguments.nodes[0]!))?.kind === "f64"
+  ) {
+    return lowerFsWriteDescriptorCall(lowerer, expr, loc);
+  }
   if (
     bi.module === "fs" &&
     (bi.member === "writeFileSync" || bi.member === "appendFileSync") &&

packages/compiler/src/frontend/lowering/builtins/promises.ts

@@ -5,7 +5,11 @@ import * as ts from "../../ts7/adapter.js";
 import type { Lowerer } from "../lowerer.js";
 import { countedFor, varRef } from "../../../ir/build.js";
 import { arrayValueStore, optionalElementValue, primitiveElement } from "../array-values.js";
-import { maybePromiseResolve, maybePromiseUnionOf } from "../maybe-promise.js";
+import {
+  maybePromiseResolve,
+  maybePromiseUnionOf,
+  promiseOrUnitUnionOf,
+} from "../maybe-promise.js";
 import { isJsSourceFile, locOf } from "../../program.js";
 import {
   BOOL,
@@ -670,11 +674,17 @@ export function lowerPromiseStaticCall(
         if (v.type.kind !== "promise") lowerer.badType(argNode, argT);
         return v;
       }
-      const maybePromise = argIr ? maybePromiseUnionOf(lowerer, argIr) : null;
+      // A promise-or-unit union (`Promise<void> | void`,
+      // `Promise<T> | undefined`) resolves exactly like a maybe-promise
+      // one: Node adopts a promise operand and fulfills every other value
+      // as itself (a nullish arm fulfills undefined).
+      const maybePromise = argIr
+        ? (maybePromiseUnionOf(lowerer, argIr) ?? promiseOrUnitUnionOf(lowerer, argIr))
+        : null;
       if (maybePromise) {
         const resultT = lowerer.mapTypeOf(lowerer.typeOf(call));
         const v = lowerer.lowerExpr(argNode);
-        const mp = maybePromiseUnionOf(lowerer, v.type);
+        const mp = maybePromiseUnionOf(lowerer, v.type) ?? promiseOrUnitUnionOf(lowerer, v.type);
         const resolved =
           resultT?.kind === "promise" && mp
             ? maybePromiseResolve(lowerer, v, mp, resultT, loc)

packages/compiler/src/frontend/lowering/builtins/web-streams-dyn.ts

@@ -1,9 +1,19 @@
+import { staticFetchMembers } from "../../../compat/fetch-profile.js";
 import { numLit } from "../../../ir/build.js";
-import { BOOL, DYN, STRING, type IrExpr, type SrcLoc } from "../../../ir/ir.js";
+import { everyStmt } from "../../../ir/traverse.js";
+import {
+  BOOL,
+  DYN,
+  STRING,
+  VOID,
+  type IrExpr,
+  type IrFunction,
+  type SrcLoc,
+} from "../../../ir/ir.js";
 import * as ts from "../../ts7/adapter.js";
 import { isJsSourceFile, locOf } from "../../program.js";
 import { constituentTypes } from "../../ts7/checker.js";
-import type { Lowerer } from "../lowerer.js";
+import { ENTRY_NAME, type Lowerer } from "../lowerer.js";
 import { stdlibGlobalNameOf } from "../surfaces.js";
 import { streamWebImportOf } from "./module-bindings.js";
 
@@ -221,3 +231,98 @@ export function isWebStreamValueInJavaScript(lowerer: Lowerer, node: ts.Expressi
     return true;
   return typeInvolvesWebStream(lowerer, lowerer.typeOf(node));
 }
+
+/** Whether a checker type is the stdlib ReadableStream, or a union with it. */
+function isReadableStreamType(lowerer: Lowerer, type: ts.Type): boolean {
+  if ((type.flags & ts.TypeFlags.Union) !== 0)
+    return constituentTypes(type).some((t) => isReadableStreamType(lowerer, t));
+  const symbol = type.getSymbol();
+  return (
+    symbol !== undefined &&
+    ts.symbolName(symbol) === "ReadableStream" &&
+    lowerer.isStdlibSymbol(symbol)
+  );
+}
+
+/** Whether a member access resolves to a member of the stdlib
+ * ReadableStream interface. */
+function isReadableStreamMember(lowerer: Lowerer, access: ts.PropertyAccessExpression): boolean {
+  const member = lowerer.checker.getSymbolAtLocation(access.name);
+  if (member === undefined || !lowerer.isStdlibSymbol(member)) return false;
+  let owner: ts.Node | undefined = member.declarations[0]?.parent;
+  while (owner !== undefined && !ts.isInterfaceDeclaration(owner)) owner = owner.parent;
+  return owner !== undefined && owner.name.text === "ReadableStream";
+}
+
+/** Whether a checker type is any or unknown, which a fetch body can be. */
+function isUntypedType(type: ts.Type): boolean {
+  return (type.flags & (ts.TypeFlags.Any | ts.TypeFlags.Unknown)) !== 0;
+}
+
+/** Whether the program links the fetch unit (a fetch. entry point), whose
+ * Response and Request bodies can reach JavaScript as untyped values. */
+function programLinksFetch(functions: IrFunction[]): boolean {
+  let found = false;
+  const visitor = {
+    expr: (e: IrExpr): boolean => {
+      if (e.kind === "libCall" && e.fn.startsWith("fetch.")) found = true;
+      return !found;
+    },
+    stmt: (): boolean => !found,
+  };
+  for (const fn of functions) {
+    for (const s of fn.body) everyStmt(s, visitor);
+    if (found) return true;
+  }
+  return false;
+}
+
+/** Whether a JavaScript source of the program uses a ReadableStream member
+ * that the fetch unit's own stream does not implement (the fetch profile's
+ * readableStreamUnsupported): a member access the checker resolves to the
+ * interface, or `for await` over a ReadableStream value. The same member
+ * names on an untyped receiver, or `for await` over an untyped value,
+ * count in a program that links the fetch unit, since the value can be a
+ * fetch body. */
+function programUsesWebStreamOnlyMembers(lowerer: Lowerer, functions: IrFunction[]): boolean {
+  const members = staticFetchMembers().readableStreamUnsupported;
+  const iterates = members.has("[Symbol.asyncIterator]");
+  let untyped = false;
+  for (const file of lowerer.program.getSourceFiles()) {
+    if (file.isDeclarationFile || !isJsSourceFile(file)) continue;
+    let found = false;
+    ts.walkPreorder(file, (node) => {
+      let receiver: ts.Type | null = null;
+      if (ts.isPropertyAccessExpression(node) && members.has(node.name.text)) {
+        if (isReadableStreamMember(lowerer, node)) found = true;
+        else receiver = lowerer.typeOf(node.expression);
+      } else if (iterates && ts.isForOfStatement(node) && node.awaitModifier !== undefined) {
+        receiver = lowerer.typeOf(node.expression);
+        if (isReadableStreamType(lowerer, receiver)) found = true;
+      }
+      if (found) return "stop";
+      if (receiver !== null && isUntypedType(receiver)) untyped = true;
+      return undefined;
+    });
+    if (found) return true;
+  }
+  return untyped && programLinksFetch(functions);
+}
+
+/** Links the web stream classes into a program whose JavaScript uses a
+ * ReadableStream member the fetch unit's own stream lacks: a fetch body's
+ * handle stands for a web ReadableStream once such a member is used
+ * (scr_fetch.c, sf_stream_web_view). %main installs the classes first. A
+ * program that names one of the classes links them already. */
+export function installFetchBodyWebStreams(lowerer: Lowerer, functions: IrFunction[]): void {
+  if (programNamesWebStreamClass(lowerer) || !programUsesWebStreamOnlyMembers(lowerer, functions))
+    return;
+  const main = functions.find((fn) => fn.name === ENTRY_NAME);
+  if (!main) return;
+  const loc = main.loc;
+  main.body.unshift({
+    kind: "exprStmt",
+    expr: { kind: "libCall", fn: "webstream.install", args: [], type: VOID, loc },
+    loc,
+  });
+}

packages/compiler/src/frontend/lowering/builtins/x509.ts

@@ -62,6 +62,22 @@ export function refuseX509DynValues(lowerer: Lowerer, functions: readonly IrFunc
   )
     return;
   const seen = new Map<string, boolean>();
+  // A type is recursive when it reaches itself without naming a shape or a
+  // union: `type Rec = string | Rec[]` passes through the array, set, map
+  // or promise case below. Those kinds have no id to memoize by, so a type
+  // met again while it is still being visited answers false: what a cycle
+  // adds to `holds` (a certificate exists somewhere in the type) is
+  // reachability, and nothing outside the cycle makes it reachable.
+  const visiting = new Set<IrType>();
+  const descend = (t: IrType, visit: () => boolean): boolean => {
+    if (visiting.has(t)) return false;
+    visiting.add(t);
+    try {
+      return visit();
+    } finally {
+      visiting.delete(t);
+    }
+  };
   const holds = (t: IrType): boolean => {
     switch (t.kind) {
       case "record": {
@@ -79,13 +95,24 @@ export function refuseX509DynValues(lowerer: Lowerer, functions: readonly IrFunc
       }
       case "array":
       case "set":
-        return holds(t.elem);
+        return descend(t, () => holds(t.elem));
       case "map":
-        return holds(t.key) || holds(t.value);
+        return descend(t, () => holds(t.key) || holds(t.value));
       case "promise":
-        return holds(t.inner);
-      case "union":
-        return lowerer.unions.get(t.unionId)?.arms.some(holds) === true;
+        return descend(t, () => holds(t.inner));
+      case "union": {
+        // Unions have an id, so a cycle that names one is answered from the
+        // memo above; the identity guard covers a cycle that does not.
+        const key = `union:${t.unionId}`;
+        const known = seen.get(key);
+        if (known !== undefined) return known;
+        return descend(t, () => {
+          seen.set(key, false);
+          const result = lowerer.unions.get(t.unionId)?.arms.some(holds) === true;
+          seen.set(key, result);
+          return result;
+        });
+      }
       default:
         return false;
     }

packages/compiler/src/frontend/lowering/class-protocols.ts

@@ -589,6 +589,14 @@ export class ClassProtocols {
     if (facts.observed.has(symbol) || facts.written.has(symbol)) return true;
     const target = facts.targets.get(symbol);
     if (target === undefined) return true;
+    // A type that exposes a program class's method can never be a record
+    // copy: the copy would drop the method, which Node reads from the
+    // instance's prototype, and a copied receiver would lose overrides.
+    // The mark exists because an identity-carrying value flows into the
+    // type, so keep the view even when nothing compares the value (pi's
+    // `{ handleInput?(data: string): void } | undefined` field receives
+    // component instances directly and is only ever called through).
+    if (this.methodViewSources(target).length > 0) return true;
     const names = new Set(this.checker.getPropertiesOfType(target).map((p) => ts.symbolName(p)));
     const origins = facts.origins.get(symbol) ?? new Set<ts.Symbol>();
     const writesNames = (written: Set<string> | "all"): boolean =>

packages/compiler/src/frontend/lowering/class-to-primitive.ts

@@ -41,6 +41,7 @@ interface ToStringTag {
 }
 
 const PRIMITIVE_KINDS = new Set(["f64", "string", "bool", "bigint", "undefinedT", "nullT"]);
+const OBJECT_ARMS_TO_STRING = new Set(["url", "bytes", "date", "regex"]);
 
 /** ToString(ToPrimitive(value, hint)) of a program class instance, as Node
  * computes it: `[Symbol.toPrimitive](hint)` when the class defines it, then
@@ -391,8 +392,9 @@ function refuse(lowerer: Lowerer, node: ts.Node, info: ClassInfo, what: string):
 }
 
 /** ToString(ToPrimitive(value, hint)) of a union whose arms are program
- * class instances, URLs and primitives (`Box | undefined`, `string | URL`):
- * each arm converts as it would alone. Null for other unions. */
+ * class instances, URLs, typed arrays, dates, regular expressions and
+ * primitives (`Box | undefined`, `string | URL`, `URL | Uint8Array`): each
+ * arm converts as it would alone. Null for other unions. */
 export function classUnionToString(
   lowerer: Lowerer,
   value: IrExpr,
@@ -402,11 +404,19 @@ export function classUnionToString(
   if (value.type.kind !== "union") return null;
   const unionId = value.type.unionId;
   const arms = lowerer.unions.get(unionId)?.arms;
-  if (!arms || !arms.some((arm) => arm.kind === "object" || arm.kind === "url")) return null;
+  if (!arms || arms.every((arm) => PRIMITIVE_KINDS.has(arm.kind))) return null;
+  // Without class or URL arms both hints give the same text, and
+  // ensureString, which tries the checker's narrowed arm first, converts
+  // the union with the string hint.
+  if (hint === "default" && !arms.some((arm) => arm.kind === "object" || arm.kind === "url"))
+    return null;
   const convertible = (arm: IrType): boolean => {
     if (PRIMITIVE_KINDS.has(arm.kind)) return true;
-    // A URL arm converts through its href, like String(url).
-    if (arm.kind === "url") return true;
+    // These arms convert as ensureString converts them alone, with the
+    // same result for both hints: a URL through its href, a typed array
+    // through its join, a date through toString, a regular expression
+    // through its source and flags.
+    if (OBJECT_ARMS_TO_STRING.has(arm.kind)) return true;
     if (arm.kind !== "object") return false;
     const info = lowerer.classes.get(arm.className);
     return info !== undefined && plainProgramClass(info);

packages/compiler/src/frontend/lowering/coercions/structural-plans.ts

@@ -15,6 +15,12 @@ export type FieldLift =
  * and conversions that can only throw are not structural conversions. */
 export function widthLiftPlan(lowerer: Lowerer, src: IrType, dst: IrType): WidthLift | null {
   if (typeEquals(src, dst)) return { how: "copy" };
+  // A promise whose payload widens (`Promise<void>` into `Promise<unknown>`,
+  // a settlement type gaining optional members): the checked payload
+  // widening coerceToExpected applies at a promise slot. The promise keeps
+  // its identity and settlement turn; the payload converts when it settles.
+  if (dst.kind === "promise" && src.kind === "promise" && lowerer.promisePayloadWidens(src, dst))
+    return { how: "promiseWiden" };
   // An 'unknown' (dyn) DESTINATION slot: the static→dyn conversion —
   // dynFrom, a DEEP COPY (`{ v: 5 }` into `{ v: unknown }`, `number[]`
   // into `unknown[]` — tsc's top type over the width family's copies).
@@ -72,6 +78,7 @@ export function widthLiftPlan(lowerer: Lowerer, src: IrType, dst: IrType): Width
         (src.kind === "record" && arm.kind === "record") ||
         (src.kind === "array" && arm.kind === "array") ||
         (src.kind === "func" && arm.kind === "func") ||
+        (src.kind === "promise" && arm.kind === "promise") ||
         (src.kind === "object" && arm.kind === "object") ||
         // Tuples already lift into ordinary array slots. Consider that
         // same conversion when the array is an arm of a union too.
@@ -170,6 +177,25 @@ export function widthLiftPlan(lowerer: Lowerer, src: IrType, dst: IrType): Width
   ) {
     return { how: "narrow" };
   }
+  // A UNION of tuples/arrays into ONE array slot that is not already one of
+  // its arms: each arm converts on its own tag (a tuple rebuilds through
+  // tupleArr, an array copies or lifts through arr), and a unit arm takes
+  // the stranded TypeError — the slot holds no unit. This sits AFTER the
+  // checked-extraction rule above, so a union that already contains the
+  // destination array arm keeps compiling to the same narrow it always did;
+  // only sources with no conversion at all (the `as const` key-table read)
+  // gain one.
+  if (dst.kind === "array" && src.kind === "union") {
+    const from = lowerer.unions.get(src.unionId);
+    if (
+      from &&
+      !from.arms.some((arm) => typeEquals(arm, dst)) &&
+      from.arms.some((arm) => !isUnitType(arm))
+    ) {
+      if (from.arms.every((arm) => isUnitType(arm) || lowerer.widthLiftPlan(arm, dst) !== null))
+        return { how: "unionArr" };
+    }
+  }
   // A DERIVED instance into a BASE-typed slot (`{ p: Q }` copying into
   // `{ p: P }`): the same implicit upcast coerceToExpected performs at
   // top level — prefix layout, a pointer reinterpret, no copy.
@@ -190,6 +216,16 @@ export function widthLiftPlan(lowerer: Lowerer, src: IrType, dst: IrType): Width
     return { how: "funcAdapt" };
   }
   if (dst.kind === "record" && src.kind === "record") {
+    // An INDEX-SIGNATURE destination takes the overflow CAPTURE reshape:
+    // declared fields copy off the source, every other source field keys
+    // into the destination's overflow (the same helper widthCoerce uses
+    // at top level). Without this the pair had no width plan at all, so a
+    // union arm, field or element of this shape refused.
+    if (lowerer.shapes.get(dst.shapeId)?.indexValue !== undefined) {
+      return lowerer.recordOvfCapturePlan(src.shapeId, dst.shapeId) !== null
+        ? { how: "ovf" }
+        : null;
+    }
     return lowerer.recordWidthPlan(src.shapeId, dst.shapeId) !== null ? { how: "width" } : null;
   }
   if (dst.kind === "record" && src.kind === "union") {

packages/compiler/src/frontend/lowering/coercions/structural.ts

@@ -8,7 +8,7 @@ import { arrayValueRead, arrayValueStore, primitiveElement } from "../array-valu
 import type { WidthLift } from "../width-lift.js";
 import { InternalCompilerError } from "../../../errors.js";
 import type { IrExpr, IrFunction, IrRecordShape, IrStmt, IrType, SrcLoc } from "../../../ir/ir.js";
-import { BOOL, DYN, F64, STRING, UNDEFINED_T, typeEquals } from "../../../ir/ir.js";
+import { BOOL, DYN, F64, STRING, UNDEFINED_T, isUnitType, typeEquals } from "../../../ir/ir.js";
 import { typeKey } from "../../type-mapper.js";
 import { findStaticOn, findGenericStaticOn } from "../lower-classes.js";
 import { lowerRecordOvfCaptureHelper } from "../containers/indexed-objects.js";
@@ -238,12 +238,26 @@ export function applyWidthLift(
         throw new InternalCompilerError("lowerer bug: planned width lift failed to intern");
       return { kind: "call", callee: helper, args: [value], type: dst, loc };
     }
+    case "ovf": {
+      if (dst.kind !== "record" || value.type.kind !== "record")
+        throw new InternalCompilerError("lowerer bug: ovf lift shape");
+      const helper = lowerRecordOvfCaptureHelper(lowerer, value.type.shapeId, dst.shapeId, loc);
+      if (!helper)
+        throw new InternalCompilerError("lowerer bug: planned ovf lift failed to intern");
+      return { kind: "call", callee: helper, args: [value], type: dst, loc };
+    }
     case "unionWidth": {
       if (dst.kind !== "record" || value.type.kind !== "union")
         throw new InternalCompilerError("lowerer bug: union width lift shape");
       const helper = lowerer.unionRecordWidthHelper(value.type.unionId, dst, loc);
       return { kind: "call", callee: helper, args: [value], type: dst, loc };
     }
+    case "unionArr": {
+      if (dst.kind !== "array" || value.type.kind !== "union")
+        throw new InternalCompilerError("lowerer bug: union array lift shape");
+      const helper = lowerer.unionArrayWidthHelper(value.type.unionId, dst, loc);
+      return { kind: "call", callee: helper, args: [value], type: dst, loc };
+    }
     case "arr": {
       if (dst.kind !== "array" || value.type.kind !== "array")
         throw new InternalCompilerError("lowerer bug: arr lift shape");
@@ -318,6 +332,16 @@ export function applyWidthLift(
         throw new InternalCompilerError("lowerer bug: planned funcAdapt lift failed to intern");
       return { kind: "call", callee: adapter, args: [value], type: dst, loc };
     }
+    case "promiseWiden": {
+      if (dst.kind !== "promise" || value.type.kind !== "promise")
+        throw new InternalCompilerError("lowerer bug: promiseWiden lift shape");
+      const converted = lowerer.coerceToExpected(value, dst);
+      if (!typeEquals(converted.type, dst))
+        throw new InternalCompilerError(
+          "lowerer bug: planned promise payload widening stopped converting",
+        );
+      return converted;
+    }
     default: {
       const _exhaustive: never = lift;
       void _exhaustive;
@@ -381,6 +405,92 @@ export function unionRecordWidthHelper(
   return name;
 }
 
+/** Dispatch a union of tuples/arrays into one array slot from its runtime
+ * tag (widthLiftPlan's unionArr): each arm rebuilds or copies on its own
+ * tag through the ordinary width lifts, the same copy stance
+ * tupleArrayWidthHelper takes for one tuple. A unit arm takes the stranded
+ * TypeError: the slot holds no unit, the checker excluded that arm, and
+ * only a lying `!`/assertion still carries it. Intern before descending so
+ * a recursive arm lift can reuse it. */
+export function unionArrayWidthHelper(
+  lowerer: Lowerer,
+  fromId: string,
+  target: IrType & { kind: "array" },
+  loc: SrcLoc,
+): string {
+  const key = `unionArr:${fromId}:${typeKey(target)}`;
+  const existing = lowerer.valueHelpers.get(key);
+  if (existing) return existing;
+  const name = keyedHelperName(lowerer, "%union.arr", key);
+  lowerer.valueHelpers.set(key, name);
+  const type: IrType = { kind: "union", unionId: fromId };
+  const value = varRef("value.0", type, loc);
+  const from = lowerer.unions.get(fromId);
+  if (!from) throw new InternalCompilerError("lowerer bug: missing array union");
+  const body: IrStmt[] = [];
+  for (let tag = 0; tag < from.arms.length; tag++) {
+    const arm = from.arms[tag]!;
+    const then: IrStmt[] = [];
+    if (isUnitType(arm)) {
+      const what = arm.kind === "nullT" ? "null" : "undefined";
+      then.push({
+        kind: "throw",
+        value: {
+          kind: "libCall",
+          fn: "error.new",
+          args: [
+            {
+              kind: "strLit",
+              value: `${what} is not representable in a '${lowerer.fmt(target)}' slot (a value narrowed or asserted past the type still held it)`,
+              type: STRING,
+              loc,
+            },
+          ],
+          type: { kind: "object", className: "%TypeError" },
+          loc,
+        },
+        loc,
+      });
+    } else {
+      const lift = lowerer.widthLiftPlan(arm, target);
+      if (!lift) throw new InternalCompilerError("lowerer bug: invalid union array plan");
+      const narrowed: IrExpr = { kind: "unionNarrow", unionId: fromId, tag, value, type: arm, loc };
+      then.push({
+        kind: "return",
+        value: lowerer.applyWidthLift(lift, narrowed, target, loc),
+        loc,
+      });
+    }
+    body.push({
+      kind: "if",
+      cond: { kind: "unionIsTag", unionId: fromId, tag, negated: false, value, type: BOOL, loc },
+      then,
+      else_: null,
+      loc,
+    });
+  }
+  body.push({
+    kind: "throw",
+    value: {
+      kind: "libCall",
+      fn: "error.new",
+      args: [{ kind: "strLit", value: "invalid array union tag", type: STRING, loc }],
+      type: { kind: "object", className: "%TypeError" },
+      loc,
+    },
+    loc,
+  });
+  lowerer.liftedFns.push({
+    name,
+    params: [{ localId: "value.0", name: "value", type }],
+    returnType: target,
+    locals: [{ id: "value.0", name: "value", type, mutable: false }],
+    body,
+    loc,
+  });
+  return name;
+}
+
 export function recordWidthHelper(
   lowerer: Lowerer,
   fromId: string,

packages/compiler/src/frontend/lowering/containers/array-callbacks.ts

@@ -3382,10 +3382,13 @@ export function lowerTuplePredicateCall(
  * of the receiver's length when it starts. ArraySpeciesCreate reads the
  * receiver's length before the species constructor runs, so a constructor
  * that appends to the receiver does not extend the loop
- * (lowerWithArraySpecies). Null when `name` is no such helper. */
+ * (lowerWithArraySpecies). Every helper that reads a length is one, the
+ * flatMap whose callback returns a union included, whether or not a
+ * pipeline can fuse it (arrHofInfo holds the fusable ones); forEach makes
+ * no array, so it stays out. Null when `name` is no such helper. */
 export function boundedArrayHofHelper(lowerer: Lowerer, name: string): string | null {
-  const info = lowerer.arrHofInfo.get(name);
-  if (!info || info.method === "forEach") return null;
+  if (!name.startsWith("%arr.")) return null;
+  if (lowerer.arrHofInfo.get(name)?.method === "forEach") return null;
   const bounded = `${name}.bounded`;
   if (lowerer.liftedFnNamed(bounded)) return bounded;
   const fn = lowerer.liftedFnNamed(name);

packages/compiler/src/frontend/lowering/containers/array-concat.ts

@@ -4,6 +4,7 @@ import { elementMayBeForeign, foreignElementThrow } from "../array-values.js";
 import { countedFor, numLit, varRef } from "../../../ir/build.js";
 import {
   BOOL,
+  DYN,
   F64,
   arrayOf,
   typeEquals,
@@ -19,15 +20,18 @@ import type { Lowerer } from "../lowerer.js";
 
 /** Copy into fresh storage after every argument has evaluated. Array
  * arguments retain their own element ABI; only copied values are widened.
- * Missing slots stay holes and present undefined stays present. */
+ * Missing slots stay holes and present undefined stays present. A shape
+ * entry is an array argument's element type, null for an element value, or
+ * "dyn" for a checked-dynamic argument, which JS spreads when it is an
+ * array at run time (IsConcatSpreadable) and appends otherwise. */
 export function arrayConcatHelper(
   lowerer: Lowerer,
   elem: IrType,
-  shape: (IrType | null)[],
+  shape: (IrType | null | "dyn")[],
   node: ts.CallExpression,
   loc: SrcLoc,
 ): string {
-  const key = `concat:${typeKey(elem)}:${shape.map((type) => (type ? `a:${typeKey(type)}` : "e")).join(";")}`;
+  const key = `concat:${typeKey(elem)}:${shape.map((type) => (type === "dyn" ? "d" : type ? `a:${typeKey(type)}` : "e")).join(";")}`;
   const existing = lowerer.arrHofHelpers.get(key);
   if (existing) return existing;
   const name = keyedHelperName(lowerer, "%arr.concat", key);
@@ -37,7 +41,7 @@ export function arrayConcatHelper(
     ...shape.map((type, i) => ({
       localId: `x.${i}`,
       name: `x${i}`,
-      type: type ? arrayOf(type) : elem,
+      type: type === "dyn" ? DYN : type ? arrayOf(type) : elem,
     })),
   ];
   const locals: IrLocal[] = params.map((p) => ({
@@ -173,6 +177,43 @@ export function arrayConcatHelper(
       },
     ];
   };
+  // A checked-dynamic argument: an array spreads its elements, converted
+  // to the element type; any other value is appended as one element.
+  const appendDyn = (source: IrExpr, site: ts.Node): IrStmt[] => [
+    {
+      kind: "if",
+      cond: { kind: "dynTest", test: "array", value: source, type: BOOL, loc },
+      then: [
+        {
+          kind: "exprStmt",
+          expr: {
+            kind: "arrIntrinsic",
+            method: "concatSpread",
+            receiver: out,
+            args: [lowerer.coerceInto(site, source, arrT)],
+            type: F64,
+            loc,
+          },
+          loc,
+        },
+      ],
+      else_: [
+        {
+          kind: "exprStmt",
+          expr: {
+            kind: "arrIntrinsic",
+            method: "push",
+            receiver: out,
+            args: [lowerer.coerceInto(site, source, elem)],
+            type: F64,
+            loc,
+          },
+          loc,
+        },
+      ],
+      loc,
+    },
+  ];
   const body: IrStmt[] = [
     {
       kind: "varDecl",
@@ -182,11 +223,17 @@ export function arrayConcatHelper(
     },
     ...append(varRef("a.0", arrT, loc), elem, node.expression),
     ...shape.flatMap((type, i) =>
-      append(varRef(`x.${i}`, type ? arrayOf(type) : elem, loc), type, node.arguments.nodes[i]!),
+      type === "dyn"
+        ? appendDyn(varRef(`x.${i}`, DYN, loc), node.arguments.nodes[i]!)
+        : append(
+            varRef(`x.${i}`, type ? arrayOf(type) : elem, loc),
+            type,
+            node.arguments.nodes[i]!,
+          ),
     ),
     { kind: "return", value: out, loc },
   ];
-  if (shape.some((type) => type && !typeEquals(type, elem))) {
+  if (shape.some((type) => type !== null && type !== "dyn" && !typeEquals(type, elem))) {
     for (const id of ["i.0", "offset.0", "count.0"])
       locals.push({ id, name: id, type: F64, mutable: true });
   }

packages/compiler/src/frontend/lowering/containers/array-construction.ts

@@ -1158,6 +1158,37 @@ export function lowerArrayFromCall(
         loc,
       };
     }
+    // `Array.from(typedArray, mapper)`: the element read is live (a
+    // mapper that writes the source changes later reads, as in Node),
+    // so this is a loop over the typed array, not a toArray snapshot.
+    if (src.type.kind === "bytes" && args.length === 2) {
+      const callback = lowerArrayCallback(lowerer, args[1]!, [F64], arrayOf(F64));
+      if (callback.arity > 2)
+        lowerer.noLowering("Array.from mapper with an array parameter", args[1]!);
+      const fnRet = callback.fnArg.type.ret;
+      if (fnRet.kind === "void" || fnRet.kind === "func")
+        lowerer.badType(call, lowerer.typeOf(call));
+      requireProducedArrayElement(lowerer, call, "'Array.from(typedArray, mapper)'", fnRet);
+      const outElem = callbackArrayElement(lowerer, call, fnRet);
+      // The element kind rides in the key because the lifted helper's
+      // source parameter carries it (the reads themselves are always f64).
+      const key = `fromBytes:${src.type.elem}:${typeKey(outElem)}:${typeKey(fnRet)}:${callback.arity}`;
+      let helper = lowerer.arrHofHelpers.get(key);
+      if (!helper) {
+        helper = keyedHelperName(lowerer, "%arr.fromBytes", key);
+        lowerer.arrHofHelpers.set(key, helper);
+        lowerer.liftedFns.push(
+          buildArrayFromBytesFn(lowerer, helper, src.type, outElem, fnRet, callback.arity, loc),
+        );
+      }
+      return {
+        kind: "call",
+        callee: helper,
+        args: [src, callback.fnArg],
+        type: arrayOf(outElem),
+        loc,
+      };
+    }
     // A union of typed arrays and number arrays (indexed-unions.ts).
     const indexedArms = indexedUnionArms(lowerer, src.type);
     if (indexedArms && args.length === 1)
@@ -1320,6 +1351,96 @@ function buildArrayFromArrayFn(
   };
 }
 
+/** The `Array.from(typedArray, mapper)` generation loop: every index is
+ * read through the typed array itself (live, like Node's iteration), so
+ * a mapper that writes the source is visible to the indexes it has not
+ * visited yet. */
+function buildArrayFromBytesFn(
+  lowerer: Lowerer,
+  name: string,
+  bytesT: IrType & { kind: "bytes" },
+  outElem: IrType,
+  fnRet: IrType,
+  arity: number,
+  loc: SrcLoc,
+): IrFunction {
+  const outT = arrayOf(outElem);
+  const fnT = funcOf([F64, F64].slice(0, arity), fnRet);
+  const a = varRef("a.0", bytesT, loc);
+  const i = varRef("i.0", F64, loc);
+  const out = varRef("out.0", outT, loc);
+  // The index parameter is the second one, never read for an index the
+  // element covers, so an element-only mapper drops it.
+  const reads: IrExpr[] = [
+    { kind: "bytesIntrinsic", method: "get", receiver: a, args: [i], type: F64, loc },
+    i,
+  ];
+  const mapped: IrExpr = {
+    kind: "callValue",
+    callee: varRef("f.0", fnT, loc),
+    args: reads.slice(0, arity),
+    type: fnRet,
+    loc,
+  };
+  const held = !typeEquals(fnRet, outElem);
+  const value: IrExpr = held ? varRef("v.0", fnRet, loc) : mapped;
+  return {
+    name,
+    params: [
+      { localId: "a.0", name: "a", type: bytesT },
+      { localId: "f.0", name: "f", type: fnT },
+    ],
+    returnType: outT,
+    locals: [
+      { id: "a.0", name: "a", type: bytesT, mutable: true },
+      { id: "f.0", name: "f", type: fnT, mutable: true },
+      { id: "out.0", name: "out", type: outT, mutable: false },
+      { id: "i.0", name: "i", type: F64, mutable: true },
+      ...(held ? [{ id: "v.0", name: "v", type: fnRet, mutable: false }] : []),
+    ],
+    body: [
+      {
+        kind: "varDecl",
+        localId: "out.0",
+        init: { kind: "arrayLit", elems: [], type: outT, loc },
+        loc,
+      },
+      { kind: "varDecl", localId: "i.0", init: numLit(0, loc), loc },
+      {
+        kind: "while",
+        cond: {
+          kind: "bin",
+          op: "<",
+          left: i,
+          right: {
+            kind: "bytesIntrinsic",
+            method: "length",
+            receiver: a,
+            args: [],
+            type: F64,
+            loc,
+          },
+          type: BOOL,
+          loc,
+        },
+        body: [
+          ...(held ? [{ kind: "varDecl", localId: "v.0", init: mapped, loc } as const] : []),
+          arrayValueStore(lowerer, out, i, value, outElem, loc),
+          {
+            kind: "assign",
+            localId: "i.0",
+            value: { kind: "bin", op: "+", left: i, right: numLit(1, loc), type: F64, loc },
+            loc,
+          },
+        ],
+        loc,
+      },
+      { kind: "return", value: out, loc },
+    ],
+    loc,
+  };
+}
+
 /** `Array.from(s)` / `[...s]` on a STRING: the code-point split into a
  * fresh string[] (scr_str_code_points). */
 export function strCharsCall(src: IrExpr, loc: SrcLoc): IrExpr {

packages/compiler/src/frontend/lowering/containers/array-methods.ts

@@ -998,7 +998,7 @@ export function lowerArrayMethodCall(
     // an array-of-arrays receiver given a bare inner array, where TS
     // types it as an element but JS would SPREAD it — is fenced.
     const receiver = lowerer.lowerExpr(access.expression);
-    const shape: (IrType | null)[] = [];
+    const shape: (IrType | null | "dyn")[] = [];
     const args: IrExpr[] = [];
     for (const argNode of call.arguments.nodes) {
       if (ts.isSpreadElement(argNode)) {
@@ -1032,6 +1032,16 @@ export function lowerArrayMethodCall(
         args.push(arg);
         continue;
       }
+      // A checked-dynamic value: whether JS spreads it is decided at run
+      // time (an array spreads one level, anything else is one element).
+      if (
+        (argIr === null || argIr.kind === "dyn") &&
+        (probedArg ?? tryLowerExpression(lowerer, argNode))?.type.kind === "dyn"
+      ) {
+        shape.push("dyn");
+        args.push(lowerer.lowerExpr(argNode));
+        continue;
+      }
       // An element value — union elements wrap exactly like a push.
       shape.push(null);
       args.push(lowerer.lowerExprExpecting(argNode, elem));

packages/compiler/src/frontend/lowering/containers/indexed-objects.ts

@@ -869,6 +869,143 @@ function lowerFromEntriesStringRows(
   return { kind: "call", callee: helper, args: [receiver], type: resultT, loc };
 }
 
+/** One destination-field initializer of the overflow capture reshape: a
+ * directly-lifted source field, or the target field's undefined arm (left
+ * absent for a runtime keyed write). */
+export type RecordOvfFieldInit =
+  | { name: string; kind: "undef"; unionId: string; utag: number }
+  | { name: string; kind: "direct"; src: IrType; lift: WidthLift };
+
+/** The planning half of lowerRecordOvfCaptureHelper (its doc is the
+ * contract): the same gates, with no interning and no observable state (a
+ * speculative widthLiftPlan call must not mark the source's key order
+ * observed — the builder does that), so the width relation can promise
+ * the reshape before any helper exists. */
+export interface RecordOvfCapturePlan {
+  inits: RecordOvfFieldInit[];
+  fieldLifts: Map<string, WidthLift | "dyn">;
+  orderedFields: IrRecordShape["fields"];
+  consumed: Set<string>;
+  /** How the source's own overflow slot enters the target's value slot. */
+  ovfLift: WidthLift | "dyn" | null;
+}
+
+export function recordOvfCapturePlan(
+  lowerer: Lowerer,
+  fromId: string,
+  toId: string,
+): RecordOvfCapturePlan | null {
+  const from = lowerer.shapes.get(fromId);
+  const to = lowerer.shapes.get(toId);
+  if (!from || !to?.indexValue || from.tuple || to.tuple) return null;
+  const planningKey = `ovfp:${fromId}:${toId}`;
+  // Recursive shapes re-enter their own pair through a field: assume the
+  // pair is coercible while the outer plan checks its other fields (the
+  // same rule recordWidthPlan applies).
+  if (lowerer.coercions.planning.has(planningKey))
+    return {
+      inits: [],
+      fieldLifts: new Map(),
+      orderedFields: [],
+      consumed: new Set(),
+      ovfLift: null,
+    };
+  lowerer.coercions.planning.add(planningKey);
+  try {
+    const fIv = from.indexValue ?? null;
+    const tIv = to.indexValue;
+    // How a source value enters the target's value slot: identity, the
+    // dyn conversion, or a width lift (wrap/retag/nested reshape).
+    const slotLift = (t: IrType): WidthLift | "dyn" | null => {
+      if (typeEquals(t, tIv)) return { how: "copy" };
+      if (tIv.kind === "dyn" && (t.kind === "dyn" || lowerer.dynConvertible(t))) return "dyn";
+      return lowerer.widthLiftPlan(t, tIv);
+    };
+    // The overflow value slot must line up (sources without an index
+    // signature have no overflow to carry over).
+    if (fIv && slotLift(fIv) === null) return null;
+    // Target declared fields initialize one of two ways. A same-named
+    // SOURCE declared field whose type lifts initializes the slot DIRECTLY
+    // (consumed — no keyed write below): the declared half of a width flow
+    // into a hybrid target (`{ a: 1, b: 2 }` into `{ a: number;
+    // [k: string]: number }`), which is also what admits REQUIRED declared
+    // target fields — tsc guarantees the source declares them (an index
+    // signature alone never satisfies a required member), and an
+    // index-signature source's OVERFLOW can never collide with them (its
+    // own declared name owns the key). Every other target field defaults
+    // to its undefined arm (fresh record), so it must be optional-flavored
+    // and must accept a runtime collision (dyn slots validate via dynCheck
+    // — fields must be dyn-convertible; typed slots write through
+    // directly).
+    const inits: RecordOvfFieldInit[] = [];
+    const consumed = new Set<string>();
+    for (const tf of to.fields) {
+      const sf = from.fields.find((f) => f.name === tf.name);
+      const directLift = sf ? lowerer.widthLiftPlan(sf.type, tf.type) : null;
+      if (sf && directLift) {
+        inits.push({ name: tf.name, kind: "direct", src: sf.type, lift: directLift });
+        consumed.add(tf.name);
+        continue;
+      }
+      if (tf.type.kind !== "union") return null;
+      const utag = lowerer.armTag(tf.type.unionId, UNDEFINED_T);
+      if (utag < 0) return null;
+      if (tIv.kind === "dyn" ? !lowerer.dynConvertible(tf.type) : !typeEquals(tf.type, tIv))
+        return null;
+      inits.push({ name: tf.name, kind: "undef", unionId: tf.type.unionId, utag });
+    }
+    // Every source declared field must flow into the keyed-write slot —
+    // in DECLARATION order (insertion order below; declaredOrder omits
+    // internal '%'-fields, which never enter the copy).
+    const orderedFields = from.declaredOrder
+      ? from.declaredOrder.flatMap((n) => {
+          const f = from.fields.find((x) => x.name === n);
+          return f ? [f] : [];
+        })
+      : from.fields;
+    const fieldLifts = new Map<string, WidthLift | "dyn">();
+    for (const ff of orderedFields) {
+      if (consumed.has(ff.name)) continue; // direct-initialized above
+      const lift = slotLift(ff.type);
+      if (lift === null) return null;
+      fieldLifts.set(ff.name, lift);
+    }
+    // DISPATCH writes — keys that can hit a declared target slot: the
+    // overflow loop's runtime keys, and a literal source-field name the
+    // target also declares. Non-dyn slots store THROUGH on a collision, so
+    // such writes need every declared field to BE the slot type (the
+    // validator's recordKeySet rule); dyn slots validate per field
+    // (dynCheck), so each field must be dyn-convertible. Writes to
+    // literal names the target does NOT declare are overflowOnly and
+    // exempt — which is what lets a direct-initialized required field
+    // (`id: number` beside a `number | undefined` slot) coexist with a
+    // declared-only source, while an index-signature source declines.
+    const dispatchWrites =
+      fIv !== undefined ||
+      orderedFields.some(
+        (ff) => !consumed.has(ff.name) && to.fields.some((f) => f.name === ff.name),
+      );
+    if (dispatchWrites) {
+      if (
+        tIv.kind === "dyn"
+          ? !to.fields.every((f) => lowerer.dynConvertible(f.type))
+          : !to.fields.every((f) => typeEquals(f.type, tIv))
+      ) {
+        return null;
+      }
+    }
+    return {
+      inits,
+      fieldLifts,
+      orderedFields,
+      consumed,
+      ovfLift: fIv ? slotLift(fIv) : null,
+    };
+  } finally {
+    lowerer.coercions.planning.delete(planningKey);
+  }
+}
+
 /** Interned `%rec.capture.<n>(s)` — the INDEX-SIGNATURE reshape behind
  * `Object.fromEntries(entries) as ModelPricing` (hybrid→hybrid width
  * coercion) and `const r: Record<string, number> = { a: 1, b: 2 }`
@@ -891,100 +1028,30 @@ function lowerFromEntriesStringRows(
  * slot), and every target field must either be initialized directly by
  * a same-named source declared field (the rule that admits REQUIRED
  * declared members) or be optional-flavored (the fresh record's
- * default) and writable at runtime. */
+ * default) and writable at runtime.
+ *
+ * The width relation's `{ how: "ovf" }` lift (a record into an
+ * index-signature record arm) builds this same helper through
+ * recordOvfCapturePlan. */
 export function lowerRecordOvfCaptureHelper(
   lowerer: Lowerer,
   fromId: string,
   toId: string,
   loc: SrcLoc,
 ): string | null {
-  const from = lowerer.shapes.get(fromId);
+  const plan = recordOvfCapturePlan(lowerer, fromId, toId);
+  if (!plan) return null;
   const to = lowerer.shapes.get(toId);
-  if (!from || !to?.indexValue || from.tuple || to.tuple) return null;
+  const from = lowerer.shapes.get(fromId);
+  if (!to || !from) return null;
   const fIv = from.indexValue ?? null;
   const tIv = to.indexValue;
-  // How a source value enters the target's value slot: identity, the
-  // dyn conversion, or a width lift (wrap/retag/nested reshape).
-  const slotLift = (t: IrType): WidthLift | "dyn" | null => {
-    if (typeEquals(t, tIv)) return { how: "copy" };
-    if (tIv.kind === "dyn" && (t.kind === "dyn" || lowerer.dynConvertible(t))) return "dyn";
-    return lowerer.widthLiftPlan(t, tIv);
-  };
-  // The overflow value slot must line up (sources without an index
-  // signature have no overflow to carry over).
-  if (fIv && slotLift(fIv) === null) return null;
-  // Target declared fields initialize one of two ways. A same-named
-  // SOURCE declared field whose type lifts initializes the slot DIRECTLY
-  // (consumed — no keyed write below): the declared half of a width flow
-  // into a hybrid target (`{ a: 1, b: 2 }` into `{ a: number;
-  // [k: string]: number }`), which is also what admits REQUIRED declared
-  // target fields — tsc guarantees the source declares them (an index
-  // signature alone never satisfies a required member), and an
-  // index-signature source's OVERFLOW can never collide with them (its
-  // own declared name owns the key). Every other target field defaults
-  // to its undefined arm (fresh record), so it must be optional-flavored
-  // and must accept a runtime collision (dyn slots validate via dynCheck
-  // — fields must be dyn-convertible; typed slots write through
-  // directly).
-  type FieldInit =
-    | { name: string; kind: "undef"; unionId: string; utag: number }
-    | { name: string; kind: "direct"; src: IrType; lift: WidthLift };
-  const inits: FieldInit[] = [];
-  const consumed = new Set<string>();
-  for (const tf of to.fields) {
-    const sf = from.fields.find((f) => f.name === tf.name);
-    const directLift = sf ? lowerer.widthLiftPlan(sf.type, tf.type) : null;
-    if (sf && directLift) {
-      inits.push({ name: tf.name, kind: "direct", src: sf.type, lift: directLift });
-      consumed.add(tf.name);
-      continue;
-    }
-    if (tf.type.kind !== "union") return null;
-    const utag = lowerer.armTag(tf.type.unionId, UNDEFINED_T);
-    if (utag < 0) return null;
-    if (tIv.kind === "dyn" ? !lowerer.dynConvertible(tf.type) : !typeEquals(tf.type, tIv))
-      return null;
-    inits.push({ name: tf.name, kind: "undef", unionId: tf.type.unionId, utag });
-  }
-  // Every source declared field must flow into the keyed-write slot —
-  // in DECLARATION order (insertion order below; declaredOrder omits
-  // internal '%'-fields, which never enter the copy).
+  if (tIv === undefined) return null;
+  // The copy below writes the source's declared fields in order, so a
+  // program that reaches it observes that order (the plan itself is pure:
+  // a speculative widthLiftPlan call must not mark a shape observed).
   lowerer.keyOrder.observed.add(from.id);
-  const orderedFields = from.declaredOrder
-    ? from.declaredOrder.flatMap((n) => {
-        const f = from.fields.find((x) => x.name === n);
-        return f ? [f] : [];
-      })
-    : from.fields;
-  const fieldLifts = new Map<string, WidthLift | "dyn">();
-  for (const ff of orderedFields) {
-    if (consumed.has(ff.name)) continue; // direct-initialized above
-    const lift = slotLift(ff.type);
-    if (lift === null) return null;
-    fieldLifts.set(ff.name, lift);
-  }
-  // DISPATCH writes — keys that can hit a declared target slot: the
-  // overflow loop's runtime keys, and a literal source-field name the
-  // target also declares. Non-dyn slots store THROUGH on a collision, so
-  // such writes need every declared field to BE the slot type (the
-  // validator's recordKeySet rule); dyn slots validate per field
-  // (dynCheck), so each field must be dyn-convertible. Writes to
-  // literal names the target does NOT declare are overflowOnly and
-  // exempt — which is what lets a direct-initialized required field
-  // (`id: number` beside a `number | undefined` slot) coexist with a
-  // declared-only source, while an index-signature source declines.
-  const dispatchWrites =
-    fIv !== undefined ||
-    orderedFields.some((ff) => !consumed.has(ff.name) && to.fields.some((f) => f.name === ff.name));
-  if (dispatchWrites) {
-    if (
-      tIv.kind === "dyn"
-        ? !to.fields.every((f) => lowerer.dynConvertible(f.type))
-        : !to.fields.every((f) => typeEquals(f.type, tIv))
-    ) {
-      return null;
-    }
-  }
+  const { inits, fieldLifts, orderedFields, consumed } = plan;
   const key = `ovf:${fromId}:${toId}`;
   const existing = lowerer.valueHelpers.get(key);
   if (existing) return existing;
@@ -1092,7 +1159,7 @@ export function lowerRecordOvfCaptureHelper(
   // sources carry one).
   const ksT = arrayOf(STRING);
   if (fIv) {
-    const ovfLift = slotLift(fIv)!;
+    const ovfLift = plan.ovfLift!;
     body.push(
       {
         kind: "varDecl",

packages/compiler/src/frontend/lowering/expressions/asserted-receiver.ts

@@ -0,0 +1,207 @@
+/* A record assertion used as the object of a member PRESENCE PROBE:
+ * `typeof (stored as R).member`, `(stored as R).member !== undefined`,
+ * `null != (stored as R).member`.
+ *
+ * Node erases a type assertion, so the read answers off the value actually
+ * stored: arms of the stored union that declare the member answer their
+ * slot, and the others answer undefined (native records are closed). An
+ * eager checked extraction of R at the assertion would instead trap on
+ * every other arm, turning the probe — code that exists precisely BECAUSE
+ * the stored value may be another arm — into a TypeError where Node
+ * answers "undefined".
+ *
+ * The assertion's extraction is therefore deferred to the member read.
+ * lowerAsExpression erases it for this use; property access answers the
+ * read per arm when `assertedMemberRead` can (arms that declare the member
+ * against arms that do not), and applies the extraction the assertion
+ * would otherwise have taken eagerly when it cannot
+ * (applyDeferredAssertion).
+ *
+ * Only a probe defers. A real use of the asserted value — the member
+ * called, compared by identity, or read in any other context — keeps the
+ * checked extraction, so a lying assertion still throws at the use
+ * (divergence 38's lying-assertion stance). Types don't lie; a probe
+ * simply isn't the use.
+ */
+import * as ts from "../../ts7/adapter.js";
+import { type IrExpr, type IrType, type SrcLoc, isUnitType, typeEquals } from "../../../ir/ir.js";
+import type { Lowerer } from "../lowerer.js";
+import { tryLowerExpression } from "./try-lower-expression.js";
+import { hostedStorageView } from "./hosted-narrowing.js";
+
+/** The assertion node at the root of `node`, through parentheses; null
+ * when `node` is not an (`as` / `<T>`) assertion. */
+export function assertedReceiverNode(
+  node: ts.Expression,
+): ts.AsExpression | ts.TypeAssertion | null {
+  let e = node;
+  while (ts.isParenthesizedExpression(e)) e = e.expression;
+  return ts.isAsExpression(e) || ts.isTypeAssertion(e) ? e : null;
+}
+
+/** The property access this assertion is the receiver of, through
+ * parentheses: `(u as R).member`. Null for any other use, and for the
+ * optional chain `(u as R)?.member` (a unit arm answers undefined there
+ * instead of Node's read-of-undefined TypeError). */
+function assertedReceiverAccess(
+  assertion: ts.AsExpression | ts.TypeAssertion,
+): ts.PropertyAccessExpression | null {
+  let use: ts.Node = assertion;
+  for (;;) {
+    const parent = use.parent;
+    if (parent && ts.isParenthesizedExpression(parent) && parent.expression === use) {
+      use = parent;
+      continue;
+    }
+    break;
+  }
+  const access = use.parent;
+  return access &&
+    ts.isPropertyAccessExpression(access) &&
+    access.expression === use &&
+    access.questionDotToken === undefined
+    ? access
+    : null;
+}
+
+/** An operand that names the absent values (`undefined`, `null`), through
+ * parentheses and the `void 0` spelling. */
+function isAbsenceLiteral(node: ts.Expression): boolean {
+  let e = node;
+  while (ts.isParenthesizedExpression(e)) e = e.expression;
+  if (e.kind === ts.SyntaxKind.NullKeyword) return true;
+  if (ts.isIdentifier(e)) return e.text === "undefined";
+  return ts.isVoidExpression(e) && ts.isNumericLiteral(e.expression) && e.expression.text === "0";
+}
+
+/** The property access this assertion is the receiver of when the read is
+ * a presence probe: the operand of `typeof`, or an operand of `===` /
+ * `!==` / `==` / `!=` against undefined or null. Null for the optional
+ * chain and for every real use (a call's callee, an identity comparison,
+ * a read that flows into a typed slot), which keep the assertion's checked
+ * extraction. */
+export function assertedProbeAccessOf(
+  assertion: ts.AsExpression | ts.TypeAssertion,
+): ts.PropertyAccessExpression | null {
+  const access = assertedReceiverAccess(assertion);
+  if (access === null) return null;
+  let use: ts.Node = access;
+  while (ts.isParenthesizedExpression(use.parent) && use.parent.expression === use)
+    use = use.parent;
+  const parent = use.parent;
+  if (parent === undefined) return null;
+  if (ts.isTypeOfExpression(parent) && parent.expression === use) return access;
+  if (ts.isBinaryExpression(parent)) {
+    const comparable = parent.left === use || (parent.right === use && parent.left !== use);
+    if (!comparable) return null;
+    const op = parent.operatorToken.kind;
+    const equality =
+      op === ts.SyntaxKind.EqualsEqualsEqualsToken ||
+      op === ts.SyntaxKind.EqualsEqualsToken ||
+      op === ts.SyntaxKind.ExclamationEqualsEqualsToken ||
+      op === ts.SyntaxKind.ExclamationEqualsToken;
+    if (!equality) return null;
+    const other = parent.left === use ? parent.right : parent.left;
+    if (isAbsenceLiteral(other)) return access;
+  }
+  return null;
+}
+
+/** True when the union's record arms read `member` unevenly: at least one
+ * declares it as a data field and at least one does not (it declares a
+ * getter instead, or nothing). Those are exactly the arms
+ * assertedMemberRead answers per arm, calling a getter where an arm has
+ * one; anything else keeps the assertion's extraction (and with it the
+ * checked-narrow fence for a lying one), as does an arm with only a setter
+ * of that name. */
+export function partialRecordMember(lowerer: Lowerer, union: IrType, member: string): boolean {
+  if (union.kind !== "union") return false;
+  const arms = lowerer.unions.get(union.unionId)?.arms ?? [];
+  let records = 0;
+  let declaring = 0;
+  for (const arm of arms) {
+    if (isUnitType(arm)) continue;
+    if (arm.kind !== "record") return false;
+    const shape = lowerer.shapes.get(arm.shapeId);
+    if (!shape || shape.tuple || shape.indexValue) return false;
+    const getter = shape.fields.some((field) => field.name === `%get:${member}`);
+    if (!getter && shape.fields.some((field) => field.name === `%set:${member}`)) return false;
+    records++;
+    if (shape.fields.some((field) => field.name === member)) declaring++;
+  }
+  return records > 0 && declaring > 0 && declaring < records;
+}
+
+/** The asserted record type whose checked extraction is deferred for the
+ * probe `access`, or null when the read keeps the ordinary assertion
+ * lowering. `value` is the assertion's lowered operand. */
+export function deferredMemberReadTarget(
+  lowerer: Lowerer,
+  assertion: ts.AsExpression | ts.TypeAssertion,
+  access: ts.PropertyAccessExpression,
+  value: IrExpr,
+): IrType | null {
+  if (assertedProbeAccessOf(assertion) !== access) return null;
+  if (value.type.kind !== "union") return null;
+  const target = lowerer.mapTypeOf(lowerer.checker.getTypeFromTypeNode(assertion.type));
+  if (target?.kind !== "record") return null;
+  if (!partialRecordMember(lowerer, value.type, access.name.text)) return null;
+  return target;
+}
+
+/** Whether the property access `expr` is a probe read through an assertion
+ * that the per-arm answer claims (the same shape deferredMemberReadTarget
+ * recognizes). Checker-derived field targets must not claim it first: the
+ * field target would apply the assertion's checked extraction and trap on
+ * the arm the probe exists to detect. */
+export function deferredProbeReadAccess(
+  lowerer: Lowerer,
+  access: ts.PropertyAccessExpression,
+): boolean {
+  const assertion = assertedReceiverNode(access.expression);
+  if (assertion === null) return false;
+  if (assertedProbeAccessOf(assertion) !== access) return false;
+  const value = tryLowerExpression(lowerer, access.expression);
+  return value !== null && deferredMemberReadTarget(lowerer, assertion, access, value) !== null;
+}
+
+/** The stored value of an `in` probe whose receiver is an erased assertion
+ * (`"stream" in (stored as R)`): Node asks membership of the value actually
+ * stored, which for a union of fixed record shapes the union path answers
+ * per arm. Null when the receiver is not such an assertion, or the stored
+ * value is not a union of fixed record shapes (those keep the ordinary
+ * lowering, and its refusals). */
+export function assertedInOperand(lowerer: Lowerer, node: ts.Expression): IrExpr | null {
+  let rhs = node;
+  while (ts.isParenthesizedExpression(rhs)) rhs = rhs.expression;
+  if (!ts.isAsExpression(rhs) && !ts.isTypeAssertion(rhs)) return null;
+  const stored = tryLowerExpression(lowerer, rhs.expression);
+  if (stored === null || stored.type.kind !== "union") return null;
+  const arms = lowerer.unions.get(stored.type.unionId)?.arms ?? [];
+  const fixed = arms.every((arm) => {
+    if (arm.kind !== "record") return false;
+    const shape = lowerer.shapes.get(arm.shapeId);
+    return shape !== undefined && !shape.tuple && !shape.indexValue;
+  });
+  return arms.length > 0 && fixed ? stored : null;
+}
+
+/** Apply the extraction lowerAsExpression would have applied to the
+ * assertion's operand, for a member read with no per-arm answer. */
+export function applyDeferredAssertion(
+  lowerer: Lowerer,
+  value: IrExpr,
+  target: IrType,
+  loc: SrcLoc,
+): IrExpr {
+  if (value.type.kind === "union" && target.kind !== "union" && !typeEquals(target, value.type)) {
+    const helper = lowerer.narrowedArmHelper(value.type.unionId, target, loc);
+    if (helper !== null) return lowerer.checkedNarrowCall(helper, value, target, loc);
+  }
+  if (target.kind === "union" || target.kind === "record") {
+    const hosted = hostedStorageView(lowerer, value, target, loc);
+    if (hosted) return hosted;
+  }
+  lowerer.assertedValues.set(value, target);
+  return value;
+}

packages/compiler/src/frontend/lowering/expressions/binary.ts

@@ -2368,6 +2368,33 @@ export function lowerStrictEquality(
   negated: boolean,
   loc: SrcLoc,
 ): IrExpr {
+  // JavaScript: a checked-dynamic name the checker's flow analysis narrowed
+  // (`let c; ... (c = s[i]) === " "`) is compared unchecked. Strict
+  // equality never throws, and the flow type is the checker's inference
+  // (an indexed read past the end is undefined whatever it says).
+  const flowNarrowed = (n: ts.Node, e: IrExpr): IrExpr | null =>
+    ts.isIdentifier(n) &&
+    e.kind === "dynCheck" &&
+    e.value.type.kind === "dyn" &&
+    isJsSourceFile(ts.getSourceFileOfNode(n))
+      ? e.value
+      : null;
+  const scalar = (e: IrExpr): boolean =>
+    e.type.kind === "string" || e.type.kind === "f64" || e.type.kind === "bool";
+  {
+    const l = flowNarrowed(leftNode, left);
+    const r = flowNarrowed(rightNode, right);
+    if ((l !== null && (scalar(right) || r !== null)) || (r !== null && scalar(left))) {
+      const cmp: IrExpr = {
+        kind: "dynScalarEq",
+        left: l ?? lowerer.coerceInto(leftNode, left, DYN),
+        right: r ?? lowerer.coerceInto(rightNode, right, DYN),
+        type: BOOL,
+        loc,
+      };
+      return negated ? { kind: "unary", op: "!", operand: cmp, type: BOOL, loc } : cmp;
+    }
+  }
   const plainBothNum = left.type.kind === "f64" && right.type.kind === "f64";
   const bothStr = left.type.kind === "string" && right.type.kind === "string";
   const indexed = tryLowerIndexedComparison(lowerer, left, right, negated, loc);

packages/compiler/src/frontend/lowering/expressions/element-access.ts

@@ -66,6 +66,7 @@ import {
   lowerSafeIndexRead,
   primitiveElement,
   unionArrayValueRead,
+  unaffectedByIndex,
 } from "../array-values.js";
 import { lowerOptionalStringIndex, lowerUnionStringIndex } from "../string-index.js";
 import { InternalCompilerError } from "../../../errors.js";
@@ -2475,14 +2476,35 @@ export function lowerElementWrite(lowerer: Lowerer, expr: ts.BinaryExpression):
     (value.type.kind === "union" && lowerer.armTag(value.type.unionId, UNDEFINED_T) >= 0)
       ? arrayValueType(lowerer, arr.type.elem)
       : arr.type.elem;
-  return arrayValueStore(
-    lowerer,
-    arr,
-    index,
-    lowerer.coerceInto(expr.right, value, expected),
-    arr.type.elem,
-    locOf(expr),
-  );
+  const stored = lowerer.coerceInto(expr.right, value, expected);
+  // A binding's `T | undefined` value is stored by two writes that both
+  // read the index (arrayValueStore). A computed index, such as the key
+  // conversion of a union index that declares a temporary, is held once
+  // ahead of them. The stores then read the receiver after the index, so
+  // an index that can rebind the receiver (`a[(a = b, 0)] = u`) keeps the
+  // stores as they are, reading the receiver first.
+  if (
+    stored.kind === "varRef" &&
+    stored.type.kind === "union" &&
+    lowerer.armTag(stored.type.unionId, UNDEFINED_T) >= 0 &&
+    !typeEquals(stored.type, arr.type.elem) &&
+    arr.kind === "varRef" &&
+    index.kind !== "varRef" &&
+    index.kind !== "numLit" &&
+    unaffectedByIndex(lowerer, arr, index)
+  ) {
+    const loc = locOf(expr);
+    const held = lowerer.declareHiddenLocal("%arrayIndex", index.type);
+    return {
+      kind: "block",
+      body: [
+        { kind: "varDecl", localId: held.id, init: index, loc },
+        arrayValueStore(lowerer, arr, varRef(held.id, held.type, loc), stored, arr.type.elem, loc),
+      ],
+      loc,
+    };
+  }
+  return arrayValueStore(lowerer, arr, index, stored, arr.type.elem, locOf(expr));
 }
 
 /** Environment property names use ToPrimitive with the string hint, so

packages/compiler/src/frontend/lowering/expressions/narrowing.ts

@@ -56,6 +56,7 @@ import { assertsGlobalObject, extractionCopiesReference } from "../assertion-ide
 import { iterableSlotValue, iterableUnionOf } from "../iterable-values.js";
 import { isDeclBoundaryTypeNode } from "../../decl-projection.js";
 import { isMessageListenerCall } from "./message-listeners.js";
+import { assertedProbeAccessOf, deferredMemberReadTarget } from "./asserted-receiver.js";
 
 /** `c ? a : b`. An explicit destination constructs each branch at that
  * layout before joining it. Without one, retain the expression's inferred
@@ -528,6 +529,27 @@ export function maybeNarrow(lowerer: Lowerer, expr: IrExpr, node: ts.Node): IrEx
         )
       )
         return expr;
+      // A `let c;` binding without a type takes its flow types from what
+      // the code assigns, and the checker's types for those values can omit
+      // undefined (`c = s[i]` past the end): reads keep the actual value.
+      if (
+        symbol &&
+        symbol.declarations.some(
+          (decl) =>
+            ts.isVariableDeclaration(decl) &&
+            decl.initializer === undefined &&
+            decl.type === undefined &&
+            ts.isIdentifier(decl.name) &&
+            decl.parent?.parent !== undefined &&
+            ts.isVariableStatement(decl.parent.parent) &&
+            !/@type\b/.test(
+              ts
+                .getSourceFileOfNode(decl)
+                .text.slice(decl.parent?.parent?.pos ?? decl.pos, ts.tokenStartOfNode(decl)),
+            ),
+        )
+      )
+        return expr;
       if (
         symbol &&
         symbol.declarations.some(
@@ -1407,6 +1429,17 @@ export function lowerAsExpression(
   if (inner.type.kind !== "dyn") {
     const targetTs0 = lowerer.checker.getTypeFromTypeNode(expr.type);
     const target = lowerer.mapTypeOf(targetTs0);
+    // A PRESENCE PROBE of a member read through the assertion (`typeof
+    // (u as R).member`, `(u as R).member !== undefined`): the read answers
+    // off the value actually stored, since Node erases the assertion and an
+    // arm that does not declare the member answers undefined. The read
+    // answers that per arm (assertedMemberRead); an eager extraction of R
+    // here would trap first and turn the probe into a TypeError. Defer to
+    // the read (asserted-receiver.ts); every real use keeps the extraction.
+    {
+      const access = assertedProbeAccessOf(expr);
+      if (access && deferredMemberReadTarget(lowerer, expr, access, inner) !== null) return inner;
+    }
     if (target?.kind === "object") {
       const asserted = checkedClassAssertion(lowerer, inner, target, locOf(expr));
       if (asserted) return asserted;

packages/compiler/src/frontend/lowering/expressions/property-access.ts

@@ -125,6 +125,12 @@ import { dynUndefinedExpr, nodeThrowExpr, numLit, strLit, varRef } from "../../.
 import { typedArrayLacksMember } from "../builtins/typed-array-members.js";
 import { TEMPORAL_GLOBAL } from "../../builtin-modules.js";
 import { tryLowerExpression } from "./try-lower-expression.js";
+import {
+  applyDeferredAssertion,
+  assertedReceiverNode,
+  deferredMemberReadTarget,
+  deferredProbeReadAccess,
+} from "./asserted-receiver.js";
 import { lowerRecordInheritedMember } from "../record-inherited.js";
 import { DATE_METHOD_HINT } from "../builtins/dates.js";
 import {
@@ -1398,6 +1404,25 @@ export function lowerPropertyAccessExpr(
   // receiver that lowers cleanly means the MEMBER is the gap — name
   // the property and the receiver's type instead of "property access".
   let recvLowered = lowerer.lowerExpr(expr.expression);
+  // A PRESENCE PROBE of a member read through an erased assertion
+  // (`typeof (u as R).member`, `(u as R).member !== undefined`): Node
+  // reads the property off the value actually stored, so an arm that does
+  // not declare it answers undefined. lowerAsExpression deferred the
+  // assertion's checked extraction for this read; answer it per arm
+  // (assertedMemberRead), and only a member with no per-arm answer applies
+  // the extraction the assertion would otherwise have taken (a lying
+  // assertion still throws on a real use, never on the probe).
+  {
+    const assertion = assertedReceiverNode(expr.expression);
+    const target = assertion
+      ? deferredMemberReadTarget(lowerer, assertion, expr, recvLowered)
+      : null;
+    if (target !== null) {
+      const read = assertedMemberRead(lowerer, expr, recvLowered, loc);
+      if (read) return read;
+      recvLowered = applyDeferredAssertion(lowerer, recvLowered, target, loc);
+    }
+  }
   // A receiver whose value union holds record types the checker's subtype
   // reduction folded into its type (`{ status } | { status, value }` typed
   // `{ status }`, a call of a function with an inferred return type): read
@@ -1692,7 +1717,9 @@ function assertedMemberRead(
   const name = expr.name.text;
   let resultType: IrType | null = null;
   let records = 0;
-  const declaring: { tag: number; shapeId: string; type: IrType }[] = [];
+  // A getter arm answers by calling its getter, once per read, as Node's
+  // read does; a data arm reads its slot.
+  const declaring: { tag: number; shapeId: string; type: IrType; getter: boolean }[] = [];
   const units: number[] = [];
   for (let tag = 0; tag < arms.length; tag++) {
     const arm = arms[tag]!;
@@ -1704,16 +1731,21 @@ function assertedMemberRead(
     records++;
     const shape = lowerer.shapes.get(arm.shapeId);
     if (!shape || shape.tuple || shape.indexValue) return null;
-    const field = shape.fields.find((f) => f.name === name);
+    const getter = shape.fields.find((f) => f.name === `%get:${name}`);
+    if (!getter && shape.fields.some((f) => f.name === `%set:${name}`)) return null;
+    if (getter && getter.type.kind !== "func") return null;
+    const field = getter ?? shape.fields.find((f) => f.name === name);
     if (!field) continue;
+    const fieldType = field.type.kind === "func" && getter ? field.type.ret : field.type;
     // The declaring arms agree up to an undefined arm (`string` beside
     // `string | undefined`).
-    const armResult = lowerer.withUndefinedArmOf(field.type);
+    const armResult = lowerer.withUndefinedArmOf(fieldType);
     if (!armResult || (resultType && !typeEquals(resultType, armResult))) return null;
     resultType = armResult;
-    declaring.push({ tag, shapeId: arm.shapeId, type: field.type });
+    declaring.push({ tag, shapeId: arm.shapeId, type: field.type, getter: getter !== undefined });
   }
-  if (!resultType || declaring.length === records) return null;
+  if (!resultType || (declaring.length === records && declaring.every((d) => !d.getter)))
+    return null;
   const stable = lowerer.declareHiddenLocal("%assertedRecv", recv.type);
   const ref = (): IrExpr => ({ kind: "varRef", localId: stable.id, type: recv.type, loc });
   let result: IrExpr | null = lowerer.wrappedUndefined(resultType, loc);
@@ -1736,15 +1768,19 @@ function assertedMemberRead(
       loc,
     };
   }
-  for (const { tag, shapeId, type: fieldType } of declaring) {
-    const read: IrExpr = {
+  for (const { tag, shapeId, type: fieldType, getter } of declaring) {
+    const slot: IrExpr = {
       kind: "recordGet",
       obj: { kind: "unionNarrow", unionId, tag, value: ref(), type: arms[tag]!, loc },
       shapeId,
-      field: name,
+      field: getter ? `%get:${name}` : name,
       type: fieldType,
       loc,
     };
+    const read: IrExpr =
+      getter && fieldType.kind === "func"
+        ? { kind: "callValue", callee: slot, args: [], type: fieldType.ret, loc }
+        : slot;
     result = {
       kind: "ternary",
       cond: { kind: "unionIsTag", unionId, tag, negated: false, value: ref(), type: BOOL, loc },
@@ -3145,6 +3181,12 @@ export function lowerFieldRead(lowerer: Lowerer, expr: ts.PropertyAccessExpressi
         };
     }
   }
+  // A presence probe of a member read through a record assertion
+  // (`typeof (u as R).member`): the field target comes from the ASSERTED
+  // checker type and would apply the checked extraction, trapping on the
+  // very arm the probe exists to detect. Leave the read to the per-arm
+  // answer in lowerPropertyAccessExpr (asserted-receiver.ts).
+  if (deferredProbeReadAccess(lowerer, expr)) return null;
   const target = lowerer.fieldTarget(expr);
   if (target) return lowerer.fieldGetExpr(target, locOf(expr), expr);
   if (expr.questionDotToken) return null;
@@ -3550,11 +3592,12 @@ export function lowerUnionProperty(
  * message. The caller (the property/element dispatch) maybeNarrows. */
 export function lowerUnionKeyedRead(
   lowerer: Lowerer,
-  expr: ts.Expression,
+  expr: ts.Node,
   unionId: string,
   value: IrExpr,
   key: IrExpr,
   literalField: string | null,
+  missingFieldAsUndefined = false,
 ): IrExpr | null {
   const def = lowerer.unions.get(unionId);
   if (!def) return null;
@@ -3604,6 +3647,14 @@ export function lowerUnionKeyedRead(
     }
     if (!shape.indexValue) {
       if (literalField === null && shape.fields.length > 0) continue;
+      // A record arm that does not declare the key at all answers
+      // UNDEFINED in JS. Destructuring asks for that answer (its own
+      // desugar reads a missing property as undefined, Node-exact); a
+      // dotted read keeps refusing until narrowing proves the arm.
+      if (literalField !== null && missingFieldAsUndefined) {
+        push(UNDEFINED_T);
+        continue;
+      }
       return null;
     }
     if (!pushAnswer(shape.indexValue)) return null;
@@ -3648,6 +3699,12 @@ export function lowerUnionKeyedRead(
       if (!surfaces(declared)) return null;
       continue;
     }
+    if (literalField !== null && !shape.indexValue && missingFieldAsUndefined) {
+      // The missing-field answer above must surface as the join's
+      // undefined arm (the backend's keyed-read miss path stores it).
+      if (!(type.kind === "union" && lowerer.armTag(type.unionId, UNDEFINED_T) >= 0)) return null;
+      continue;
+    }
     const ovfShape: IrRecordShape =
       literalField !== null && shape.indexValue ? { ...shape, fields: [] } : shape;
     if (!recordKeyResultOk(lowerer, ovfShape, type)) return null;

packages/compiler/src/frontend/lowering/expressions/relational.ts

@@ -50,6 +50,7 @@ import { type ClassInfo, storedClassValueType } from "../lower-classes.js";
 import { classInstanceOf } from "../class-dynamic-dispatch.js";
 import { InternalCompilerError } from "../../../errors.js";
 import { tryLowerExpression } from "./try-lower-expression.js";
+import { assertedInOperand } from "./asserted-receiver.js";
 import { isJsSourceFile, locOf } from "../../program.js";
 import { checkedClassInstanceOf } from "../class-construction.js";
 import { unsupportedDiag } from "../../../diagnostics/diagnostic.js";
@@ -1672,7 +1673,11 @@ export function lowerInExpression(
       if (probed) return probed;
     }
   }
-  let recv = lowerer.lowerExpr(expr.right);
+  // `"stream" in (stored as R)` probes the value actually stored (Node
+  // erases the assertion): a union of fixed record shapes answers per arm
+  // below, so the assertion's checked extraction must not run first (it
+  // would trap on the very arm the probe exists to detect).
+  let recv = assertedInOperand(lowerer, expr.right) ?? lowerer.lowerExpr(expr.right);
   const classInfo =
     recv.type.kind === "classval" ? lowerer.classes.get(recv.type.className) : undefined;
   if (
@@ -1877,17 +1882,17 @@ export function lowerInExpression(
       return perValueUnionIn(lowerer, recv, unionId, key, answers, loc);
     }
     if (staticAnswers) {
-      const pureRecv =
-        recv.kind === "varRef" ||
-        recv.kind === "recordGet" ||
-        recv.kind === "fieldGet" ||
-        pureRecvNode;
+      // A field read is pure only over a pure object: the stored value of
+      // an asserted receiver (`"k" in (make().stored as R)`) still
+      // evaluates its call.
+      const pureRecv = isSafeToRepeat(recv) || pureRecvNode;
+      let value = recv;
       const isTag = (tag: number, negated: boolean): IrExpr => ({
         kind: "unionIsTag",
         unionId,
         tag,
         negated,
-        value: recv,
+        value,
         type: BOOL,
         loc,
       });
@@ -1913,27 +1918,29 @@ export function lowerInExpression(
       if (trues.length === 1) return isTag(trues[0]!.tag, false);
       const falses = answers.filter((a) => !a.has);
       if (falses.length === 1) return isTag(falses[0]!.tag, true);
-      // Several arms on each side: the OR chain re-reads the receiver,
-      // so only side-effect-free reads qualify.
-      if (pureRecv) {
-        let out: IrExpr = isTag(trues[0]!.tag, false);
-        for (const t of trues.slice(1)) {
-          out = {
-            kind: "logical",
-            op: "||",
-            left: out,
-            right: isTag(t.tag, false),
-            type: BOOL,
-            loc,
-          };
-        }
-        return out;
+      // Several arms on each side: the OR chain re-reads the receiver, so
+      // anything but a pure read is evaluated once into a hidden local.
+      const stable = pureRecv ? null : lowerer.declareHiddenLocal("%inRecv", recv.type);
+      if (stable) value = varRef(stable.id, recv.type, loc);
+      let out: IrExpr = isTag(trues[0]!.tag, false);
+      for (const t of trues.slice(1)) {
+        out = {
+          kind: "logical",
+          op: "||",
+          left: out,
+          right: isTag(t.tag, false),
+          type: BOOL,
+          loc,
+        };
       }
-      lowerer.unsupported(
-        "SC1090",
-        expr,
-        "statically-decided 'in' on computed receivers (bind the value to a variable first)",
-      );
+      if (!stable) return out;
+      return {
+        kind: "seqExpr",
+        stmts: [{ kind: "varDecl", localId: stable.id, init: recv, loc }],
+        result: out,
+        type: BOOL,
+        loc,
+      };
     }
   }
   // A literal member name over a typed array (`"BYTES_PER_ELEMENT" in a`):

packages/compiler/src/frontend/lowering/function-context.ts

@@ -12,6 +12,9 @@ export interface FnCtx {
   /** Lexical this has no checker symbol and one binding per function.
    * Direct parameters and threaded captures use this same slot. */
   thisLocal: IrLocal | null;
+  /** Set when a lookup of `this` returned thisLocal, from this function's
+   * body or from a nested function that captures it. */
+  thisRead?: true;
   /** Derived constructors initialize fields at the actual super() site,
    * including sites nested in a try/finally block. Nested functions do not
    * inherit this hook. */
@@ -135,7 +138,10 @@ export function bindingInContext(
   ctx: FnCtx,
   symbol: ts.Symbol | "arguments" | undefined,
 ): IrLocal | null {
-  if (symbol === undefined) return ctx.thisLocal;
+  if (symbol === undefined) {
+    if (ctx.thisLocal !== null) ctx.thisRead = true;
+    return ctx.thisLocal;
+  }
   if (symbol === "arguments") return ctx.argumentsLocal ?? null;
   for (let i = ctx.scopes.length - 1; i >= 0; i--) {
     const local = ctx.scopes[i]!.get(symbol);