Merge main and refresh the runtime profile on it

This commit is contained in:
Chris Tate
2026-10-29 18:39:53 -05:00
754 changed files with 32570 additions and 1999 deletions
+98 -25
View File
@@ -136,6 +136,8 @@ jobs:
run: pnpm generate:ts7-ast --check
- name: Verify the generated IR field readers
run: pnpm generate:ir-fields --check
- name: Verify the generated IR codec
run: pnpm generate:ir-codec --check
native_macos:
needs: [validation_scope, lanes]
@@ -464,6 +466,10 @@ jobs:
name: native-linux-x64-gnu-shape-overlay
path: ${{ runner.temp }}
- run: tar -xzf "$RUNNER_TEMP/native-linux-x64-gnu.tar.gz"
# The npm suites' Node island cases load Node from a libnode pack, and
# fail rather than skip in this lane when none is configured.
- uses: ./.github/actions/libnode
if: matrix.suite == 'npm' || matrix.suite == 'npm-static'
- name: Run the suite with the overlay on
run: |
case "${{ matrix.suite }}" in
@@ -686,11 +692,17 @@ jobs:
run: pnpm test:checker-identity --optimization dev
llvm_artifacts_linux_x64:
needs: [validation_scope, lanes]
needs: [validation_scope, lanes, native_linux]
if: ${{ needs.validation_scope.outputs.full == 'true' && needs.lanes.outputs.linux == 'true' }}
name: test (Linux x64 LLVM artifacts, no clang)
runs-on: ubuntu-24.04
timeout-minutes: 30
# The glibc helper and runtime pack come from native_linux, as the macOS
# artifact jobs take theirs from native_macos. The musl packages and the
# WASI smoke run in llvm_artifacts_linux_x64_musl and
# llvm_artifacts_linux_x64_wasi: together with a rebuild of the glibc
# packages, which a runtime change forces, the one job took 25 to 32
# minutes.
timeout-minutes: 20
steps:
- uses: actions/checkout@v4
- uses: pnpm/action-setup@v4
@@ -700,32 +712,13 @@ jobs:
with:
node-version-file: .node-version
cache: pnpm
- uses: ./.github/actions/llvm-linux
- run: pnpm install --frozen-lockfile
- uses: ./.github/actions/zig
- uses: ./.github/actions/native-packages
- uses: actions/download-artifact@v4
with:
target: linux-x64-gnu
- name: Build and verify Linux x64 musl release packages
run: |
pnpm --filter @scriptc/llvm-linux-x64-musl build:native
CC=zig AR=zig pnpm --filter @scriptc/runtime-linux-x64-musl build:native
helper=packages/llvm-linux-x64-musl/bin/scriptc-llvm-codegen
readelf -l "$helper" > "$RUNNER_TEMP/musl-helper-program-headers"
if grep -q INTERP "$RUNNER_TEMP/musl-helper-program-headers"; then
echo "musl helper must be a static executable" >&2
exit 1
fi
docker run --rm --volume "$GITHUB_WORKSPACE:/work:ro" alpine:3.22 \
/work/$helper version --format=json | node -e '
const value = JSON.parse(require("node:fs").readFileSync(0, "utf8"));
if (value.default_target !== "x86_64-unknown-linux-musl" || !value.supported_targets?.includes("wasm32-unknown-wasi")) process.exit(1);
'
node packages/runtime-linux-x64-musl/scripts/verify.mjs
- uses: ./.github/actions/native-packages
with:
target: wasm32-wasi
helper: "false"
name: native-linux-x64-gnu
path: ${{ runner.temp }}
- run: tar -xzf "$RUNNER_TEMP/native-linux-x64-gnu.tar.gz"
- run: pnpm build
- name: Linux helper object format and no-clang output contract
run: |
@@ -751,6 +744,82 @@ jobs:
SCRIPTC_NO_CACHE=1 SCRIPTC_LEGACY_C_PIPELINE=0 node packages/cli/dist/main.js build tests/corpus/001-hello.ts -o /tmp/hello
test "$(/tmp/hello)" = "hello world"
'
llvm_artifacts_linux_x64_musl:
needs: [validation_scope, lanes, native_linux]
if: ${{ needs.validation_scope.outputs.full == 'true' && needs.lanes.outputs.linux == 'true' }}
name: test (Linux x64 LLVM artifacts, no clang, musl)
runs-on: ubuntu-24.04
# The static musl helper and runtime pack, restored from the
# native-packages cache when their inputs are unchanged: about 9 minutes
# when both are built. The runtime build emits its units through the
# host's glibc helper (runtime-unit.mjs), which comes from native_linux.
timeout-minutes: 25
steps:
- uses: actions/checkout@v4
- uses: pnpm/action-setup@v4
with:
version: 11.1.3
- uses: actions/setup-node@v4
with:
node-version-file: .node-version
cache: pnpm
- uses: ./.github/actions/llvm-linux
- run: pnpm install --frozen-lockfile
- uses: ./.github/actions/zig
- uses: actions/download-artifact@v4
with:
name: native-linux-x64-gnu
path: ${{ runner.temp }}
- run: tar -xzf "$RUNNER_TEMP/native-linux-x64-gnu.tar.gz"
- uses: ./.github/actions/native-packages
with:
target: linux-x64-musl
- name: Verify the Linux x64 musl release helper
run: |
helper=packages/llvm-linux-x64-musl/bin/scriptc-llvm-codegen
readelf -l "$helper" > "$RUNNER_TEMP/musl-helper-program-headers"
if grep -q INTERP "$RUNNER_TEMP/musl-helper-program-headers"; then
echo "musl helper must be a static executable" >&2
exit 1
fi
docker run --rm --volume "$GITHUB_WORKSPACE:/work:ro" alpine:3.22 \
/work/$helper version --format=json | node -e '
const value = JSON.parse(require("node:fs").readFileSync(0, "utf8"));
if (value.default_target !== "x86_64-unknown-linux-musl" || !value.supported_targets?.includes("wasm32-unknown-wasi")) process.exit(1);
'
llvm_artifacts_linux_x64_wasi:
needs: [validation_scope, lanes, native_linux]
if: ${{ needs.validation_scope.outputs.full == 'true' && needs.lanes.outputs.linux == 'true' }}
name: test (Linux x64 LLVM artifacts, no clang, wasm32)
runs-on: ubuntu-24.04
# The wasm32-wasi runtime pack (restored from the native-packages cache
# when unchanged; its units go through native_linux's glibc helper) and
# the WASI smoke tests, which take 6 to 9 minutes.
timeout-minutes: 25
steps:
- uses: actions/checkout@v4
- uses: pnpm/action-setup@v4
with:
version: 11.1.3
- uses: actions/setup-node@v4
with:
node-version-file: .node-version
cache: pnpm
- uses: ./.github/actions/llvm-linux
- run: pnpm install --frozen-lockfile
- uses: ./.github/actions/zig
- uses: actions/download-artifact@v4
with:
name: native-linux-x64-gnu
path: ${{ runner.temp }}
- run: tar -xzf "$RUNNER_TEMP/native-linux-x64-gnu.tar.gz"
- uses: ./.github/actions/native-packages
with:
target: wasm32-wasi
helper: "false"
- run: pnpm build
- name: WASI helper object/runtime-pack smoke
run: pnpm test packages/cli/test/wasi-runtime-pack.test.ts tests/harness/wasm-library.test.ts
@@ -1120,6 +1189,8 @@ jobs:
ts7_parity,
checker_identity,
llvm_artifacts_linux_x64,
llvm_artifacts_linux_x64_musl,
llvm_artifacts_linux_x64_wasi,
llvm_artifacts_macos_arm64,
llvm_artifacts_macos_arm64_contracts,
llvm_artifacts_macos_x64,
@@ -1154,6 +1225,8 @@ jobs:
lane ts7_parity "${{ needs.ts7_parity.result }}" "${{ needs.lanes.outputs.linux }}"
lane checker_identity "${{ needs.checker_identity.result }}" "${{ needs.lanes.outputs.linux }}"
lane llvm_artifacts_linux_x64 "${{ needs.llvm_artifacts_linux_x64.result }}" "${{ needs.lanes.outputs.linux }}"
lane llvm_artifacts_linux_x64_musl "${{ needs.llvm_artifacts_linux_x64_musl.result }}" "${{ needs.lanes.outputs.linux }}"
lane llvm_artifacts_linux_x64_wasi "${{ needs.llvm_artifacts_linux_x64_wasi.result }}" "${{ needs.lanes.outputs.linux }}"
lane llvm_artifacts_macos_arm64 "${{ needs.llvm_artifacts_macos_arm64.result }}" "${{ needs.lanes.outputs.macos }}"
lane llvm_artifacts_macos_arm64_contracts "${{ needs.llvm_artifacts_macos_arm64_contracts.result }}" "${{ needs.lanes.outputs.macos }}"
lane llvm_artifacts_macos_x64 "${{ needs.llvm_artifacts_macos_x64.result }}" "${{ needs.lanes.outputs.macos }}"
+2 -2
View File
@@ -231,8 +231,8 @@ jobs:
llvm_dist: x64
use_zig: true
runs-on: ${{ matrix.runner }}
# The Intel macOS build took 24 minutes for 0.3.2 and passed 30 for 0.3.3.
timeout-minutes: 45
# The Intel macOS build took 24 minutes for 0.3.2, 40 for 0.4.5, and passed 45 for 0.4.6.
timeout-minutes: 90
steps:
- uses: actions/checkout@v4
- uses: pnpm/action-setup@v4
+1
View File
@@ -35,6 +35,7 @@ output/
/packages/runtime-*/artifacts/
/packages/runtime-*/runtime-pack.json
/packages/runtime-*/.runtime-pack-*
/packages/runtime-*/runtime-pack.verified.json*
# libnode packs: built by build:native, never committed
/packages/libnode-*/libnode.so.*
/packages/libnode-*/libnode.*.dylib
+40 -2
View File
@@ -6,6 +6,46 @@ All notable changes to scriptc will be documented in this file.
<!-- release:start -->
## 0.4.6
### Highlights
- **Node islands:** programs can load Node.js in-process on first use and run npm packages that scriptc doesn't compile in Node islands.
- Native values, closures and promises pass into islands by reference, and island handles, classes and collections can be used from compiled code.
- Modules are shared between a program and its islands, and island modules are evaluated only when the program uses them.
- Once Node loads, the runtime's event loop runs on Node's libuv, and programs exit cleanly with Node loaded.
- **New options:**
- `--exceptions=rare` unwinds through recursive functions instead of checking a status after each call.
- `--static` builds static Linux executables (opt-in).
- `--large-pages` moves executable code onto huge pages; `--cpu-dispatch` adds x86-64-v3 copies of hot functions.
- Type patches replace a package's declaration files or supply missing declarations.
- The shape overlay types JavaScript from facts the compiler proves (opt-in).
- **Faster programs:**
- Exceptions raise through cached unwind plans on x86-64 Linux, and every native function has an unwind table entry.
- Ordinary objects are stored as a layout plus value slots; unions of classes are the instance pointer itself; class instances that stay in their frame use frame storage.
- Object groups that die together are allocated in arenas, and the cycle collector does cheaper trial deletion and adapts its nursery trigger.
- The speed runtime is optimized with a committed runtime profile, and Math functions are computed as V8 does, with libm linked only when needed.
- Loops: short constant-count loops run together, for and while loops are versioned on the ranges their locals enter with, counted loops unroll by the period of a guard, and small leaf functions called from loops are inlined.
- Typed arrays: header fields are read once at a speculative loop's entry, products under bitwise operators stay in 32 bits, and shared typed-array elements are read and written with plain accesses at a counted loop's counter.
- Strings: UTF-16 indexes of long non-ASCII strings map in constant time, ASCII case converts eight bytes at a time, template literals allocate at their exact length, and strings iterate over their bytes.
- Untyped code: class instance fields are read inline at dynamic property sites with two inline-cache entries, native arrays iterate without type-name scans, and numbers pass to directly called untyped functions as JavaScript values.
- I/O: fs thread-pool completions and pool-thread writes make fewer syscalls; sockets, clients and agent entries register in constant time; fetch() reuses keep-alive connections; web streams, WritableStream and TransformStream run natively.
- Large executables map their text with 2 MiB pages, and freed heap pages are returned to the system after the frontend is disposed.
- **Faster builds:**
- Small release and speed programs compile in balanced partitions, the entry module's top-level code gets its own partition, and unchanged partitions stay cached across edits.
- Optimized executables that stay one partition generate code concurrently, and emission's workers stay busy.
- The native compiler is laid out by context-sensitive profiles collected in two phases.
- Type checking uses a shared tsc-ts server by default on Linux and macOS, and lib diagnostics are reused across options and module sets that cannot change them.
- Rebuilds skip the helper version probe and the digests of unchanged files, and the CLI links without huge text pages, so cold builds use less memory.
- **Node.js parity:**
- `Error.captureStackTrace`, Node's navigator, builtin globals read through the global object, and unhandled rejections reported in programs without async work.
- node:zlib is built on the zlib Node bundles; `Response.clone` and `Request.clone` work on typed values; fetch bodies have the members of a web ReadableStream; redirects whose chunked body has not ended are followed.
- Classes that extend Array compile; strings spread into their code points; frozen checked records stay frozen; sloppy-mode `this` is the global object.
- `util.format` runtime format strings, `inspect.custom` and colors, and untyped `promisify`.
- **TypeScript:** scriptc reads TypeScript 7.1's binary AST format and API server beside 7.0's, and tsc-ts is updated to 0.0.10.
<!-- release:end -->
## 0.4.5
### Highlights
@@ -40,8 +80,6 @@ All notable changes to scriptc will be documented in this file.
- `util.inspect` of dynamic classes, functions and `%o` arguments, and `Symbol.toStringTag` fields and getters, match Node.
- **npm packages:** more npm and CommonJS dependencies compile statically, including namespace-style packages with guarded declarations and `var` exports, and untyped Maps, Sets and arrays are typed from their uses.
<!-- release:end -->
## 0.4.4
### Highlights
@@ -0,0 +1,4 @@
{
"entry": "workloads/stream-buffered-drain.ts",
"args": []
}
@@ -0,0 +1,36 @@
// Readable streams that buffer many chunks before anyone reads them, then
// drain: 200,000 objects pushed into an object-mode Readable and taken
// with read() in a 'readable' handler, and 100,000 small Buffers pushed
// into a byte-mode Readable and piped into a Writable.
import { Readable, Writable } from "node:stream";
const objects = new Readable({ objectMode: true, highWaterMark: 16, read() {} });
for (let i = 0; i < 200000; i++) objects.push({ id: i, qty: i % 9 });
objects.push(null);
let taken = 0;
let qty = 0;
objects.on("readable", () => {
let row: { id: number; qty: number } | null;
while ((row = objects.read() as { id: number; qty: number } | null) !== null) {
taken++;
qty += row.qty;
}
});
objects.on("end", () => {
console.log(`objects taken=${taken} qty=${qty}`);
const bytes = new Readable({ read() {} });
for (let i = 0; i < 100000; i++) bytes.push(Buffer.from([i & 0xff, (i >> 8) & 0xff, 7]));
bytes.push(null);
let length = 0;
let sum = 0;
const sink = new Writable({
write(chunk: Buffer, _enc, cb) {
length += chunk.length;
for (const b of chunk) sum = (sum + b) % 65521;
cb();
},
});
sink.on("finish", () => console.log(`bytes length=${length} sum=${sum}`));
bytes.pipe(sink);
});
+2 -2
View File
@@ -309,14 +309,14 @@ A profile that already holds context-sensitive counts cannot start a second phas
### Block counts
A profile matches a function only while the function's control flow hashes as it did when the profile was collected, so a change to the program or to scriptc leaves the changed functions without counts. A block-count file beside the profile, with its name and a `.blocks` extension in place of `.profdata` (`app.blocks` beside `app.profdata`), describes each function by the labels of its blocks instead. `--profile-use` reads it when it exists and annotates the functions whose control flow changed (the mismatched ones; a function the profile has no record of keeps no counts): blocks that keep their place in the function take their recorded counts, and the counts of the other blocks follow from the counts of the blocks and edges around them. The report then adds a line, and the warning counts only the functions that remain without counts:
A profile matches a function only while the function's control flow hashes as it did when the profile was collected, so a change to the program or to scriptc leaves the changed functions without counts. A block-count file beside the profile, with its name and a `.blocks` extension in place of `.profdata` (`app.blocks` beside `app.profdata`), describes each function by the labels of its blocks instead. `--profile-use` reads it when it exists and annotates the functions whose control flow changed (the mismatched ones): blocks that keep their place in the function take their recorded counts, and the counts of the other blocks follow from the counts of the blocks and edges around them. The file also names the functions that never ran while the profile was collected, which a profile merged with `--sparse` has no record of. Those functions are compiled as cold code (an entry count of 0, optimized for size and placed with the other unlikely code), which shortens the build. A function the file does not name, for example one added since the profile was collected, keeps no counts and is optimized as without a profile. The report then adds a line, and the warning counts only the functions that remain without counts:
```
scriptc: profile app-cs.profdata: 0 functions annotated, 1 mismatched, 0 without a record
scriptc: profile app-cs.profdata: 1 function without profile counts annotated from app-cs.blocks
```
A `--profile-use` build writes the block counts of the functions its profile annotates when the `SCRIPTC_RECORD_PROFILE_BLOCKS` environment variable names a path; such a build reuses no cached artifacts. For a context-sensitive profile, record them in the second phase, which the first-phase profile annotates:
A `--profile-use` build writes the block counts of the functions its profile annotates, and the names of the functions it has no record of, when the `SCRIPTC_RECORD_PROFILE_BLOCKS` environment variable names a path; such a build reuses no cached artifacts. For a context-sensitive profile, record them in the second phase, which the first-phase profile annotates:
```console
$ SCRIPTC_RECORD_PROFILE_BLOCKS=app-cs.blocks scriptc build app.ts --optimization=speed --profile-use app.profdata --profile-generate profiles-cs/%p.profraw -o app-instrumented-cs
+4
View File
@@ -100,6 +100,8 @@ title: "Limitations"
<include>../limitations/builtin-apis/file-streams.mdx</include>
<include>../limitations/builtin-apis/file-watching.mdx</include>
<include>../limitations/builtin-apis/crypto.mdx</include>
<include>../limitations/builtin-apis/assertions.mdx</include>
@@ -172,6 +174,8 @@ title: "Limitations"
<include>../limitations/runtime-differences-from-node-js.mdx</include>
<include>../limitations/runtime-differences-from-node-js/event-order-across-native-code-and-islands.mdx</include>
### Array bounds and runtime errors
<include>../limitations/array-bounds-and-runtime-errors.mdx</include>
@@ -1,7 +1,7 @@
- **Date:** Spread constructor arguments and locale formatters are unsupported.
- **Date:** Spread constructor arguments, `toLocaleDateString()`, and locale or options arguments to the locale formatters are unsupported.
Dates are objects with identity: `===` and `==` compare identity, `new Date(date)` makes a new Date, setters change the one Date every reference holds, and Dates compile in arrays (`includes`, `indexOf`, `lastIndexOf`, `join`, and default `sort` by `toString()` text), in unions with other types, as Map and Set keys and values, and in records, tuples, class fields, closures, promises and generators.
Thrown and rejected Dates keep their identity; `structuredClone` and `postMessage` copy them.
Zero-argument, single number/string, and local-time year/month field constructors, `getTime`, `valueOf`, `toISOString`, `toUTCString`, `toJSON` (a `string | null`: null for an invalid date, as in Node), `toString`, `toDateString`, `toTimeString`, the local and UTC calendar getters and setters, `setTime`, `getTimezoneOffset`, and single-string `Date.parse` compile.
Zero-argument, single number/string, and local-time year/month field constructors, `getTime`, `valueOf`, `toISOString`, `toUTCString`, `toJSON` (a `string | null`: null for an invalid date, as in Node), `toString`, `toDateString`, `toTimeString`, the argument-free `toLocaleString()` and `toLocaleTimeString()` (the default locale's ICU patterns over the host time zone, as Node prints them), the local and UTC calendar getters and setters, `setTime`, `getTimezoneOffset`, and single-string `Date.parse` compile.
Unary `+`, `-` and `Number(date)` read the time value; `String(date)`, template literals and `+` with a string read the `toString()` text; `date[Symbol.toPrimitive](hint)` compiles with a literal hint.
A Date converted to `unknown` is the same Date object.
See [Dates and timezones](#dates-and-timezones) for timezone data.
@@ -0,0 +1,5 @@
- **File watching:** `fs.watch` returns an `FSWatcher` whose `on`, `once`, `off`, and `close` take `change`, `rename`, `error`, and `close` listeners.
A listener may take the event name and the entry name, but the name arrives as a string (Node's default encoding), so a listener annotated with the `Buffer` arm of `@types/node`'s `change` overload has no lowering.
Watching a directory on macOS with a listener that reads the entry name is refused with `SC2020` because FSEvents is not implemented yet; file watches and listeners that ignore the name work on every target.
- **Stat polling:** `fs.watchFile` polls the resolved path every `interval` milliseconds (default 5007) and calls its listener with `(current, previous)` `Stats`; `fs.unwatchFile` removes the last listener registered with that callback, or every listener when called without one.
The call must be a statement of its own, because the `StatWatcher` it returns in Node.js is not lowered, and `bigint: true` is refused.
@@ -1,3 +1,4 @@
- **Intl.Segmenter:** Unicode grapheme segmentation, iteration, and `containing()` compile, with the default locale or one language tag of the form language[-script][-region] written as a string literal (grapheme clusters do not depend on the locale).
In JavaScript dependencies, word and sentence segmenters construct, and their `segment()` throws an SC2020 error.
Locale negotiation over several locales, word and sentence segmentation, `resolvedOptions()`, and detached methods are unsupported.
Word granularity compiles too, over ICU's UAX #29 word iterator at that locale; its segments carry `isWordLike` exactly as Node.js's do.
In JavaScript dependencies, sentence segmenters construct, and their `segment()` throws an SC2020 error; in TypeScript sources the construction is refused.
Locale negotiation over several locales, sentence segmentation, `resolvedOptions()`, and detached methods are unsupported.
@@ -1,2 +1,5 @@
- **Process memory:** `process.memoryUsage()` reads `rss` from the operating system and `heapTotal` and `heapUsed` from the runtime's allocator, so the heap figures start near zero and grow as the program allocates.
`external` and `arrayBuffers` are `0`, because Buffer and typed-array storage is part of the same heap.
`arrayBuffers` is the storage of the ArrayBuffers, typed arrays, Buffers, DataViews and SharedArrayBuffers the program holds, and `external` is the same storage without SharedArrayBuffers, as in Node.
Node also counts its own startup allocations in both, so a program that holds no such storage reports `0` where Node reports a few megabytes.
In a program with worker threads, `external` and the non-shared part of `arrayBuffers` count the calling thread's storage.
The storage comes from the same allocator, so it also counts in `heapTotal` and `heapUsed`.
@@ -35,7 +35,9 @@
`stream.compose` accepts streams and async generator functions in any position, an async function as the last stage, and an array, a string, or a generator as the first stage; one stage is `Duplex.from` of it.
A web `ReadableStream` is read as an async iterable by `Readable.from` and as the first stage of `compose` or `pipeline`, and through its reader by `Readable.fromWeb` (with `objectMode`, `highWaterMark`, `encoding`, and `signal` in an options object literal) and `Duplex.from`.
In TypeScript, `new TransformStream()` without arguments builds the identity transform: its readable side is a web `ReadableStream`, its writable side supports `getWriter`, `abort`, `close`, and `locked`, and the writer supports `write`, `close`, `abort`, `releaseLock`, `ready`, `closed`, and `desiredSize`, with Node's backpressure (a write completes when the readable side reads its chunk).
A web stream in a later compose stage and `Readable.toWeb` are unsupported, and in TypeScript, so are a `TransformStream` with a transformer or queuing strategies, the `WritableStream` constructor, `pipeTo`, and `pipeThrough`.
In TypeScript, `tee`, `pipeTo`, and `pipeThrough`, the `WritableStream` constructor with an underlying sink, and a `TransformStream` with a transformer or queuing strategies compile: a program that uses one of them builds its `ReadableStream`, `WritableStream`, and `TransformStream` values with the same classes as JavaScript, and a `Response` or `Request` body has those members too.
A TypeScript program that uses only `getReader`, `cancel`, `locked`, and the identity `TransformStream` keeps the smaller native streams.
A web stream in a later compose stage and `Readable.toWeb` are unsupported.
The `ERR_INVALID_ARG_VALUE` message for a stream that is no longer readable or writable names the stream's class instead of printing the stream.
`readable.compose(stage, { signal })` is `stream.compose(readable, stage)` with the signal added; the options are an object literal whose `signal` is a variable or a property read.
`pipeline` from `node:stream/promises` accepts a trailing `{ signal, end }` object, with or without function stages, and `finished` accepts `readable`, `writable`, `error`, `cleanup`, and `signal` options, with boolean literals for the flags.
@@ -60,7 +62,7 @@
The `readline/promises` `Readline` class writes to the same outputs.
`readline.emitKeypressEvents` emits `'keypress'` events on `process.stdin` or a readable stream, shared with terminal interfaces on that stream as in Node; a listener's first parameter must admit `undefined`, which it receives for escape sequences.
Chained listener registration is unsupported.
`node:stream/web` exports the same classes as the globals; in TypeScript, only `ReadableStream` and the identity `TransformStream` have lowerings.
`node:stream/web` exports the same classes as the globals; in TypeScript, `ReadableStream`, `WritableStream`, and `TransformStream` have lowerings.
In JavaScript, `ReadableStream`, `WritableStream`, and `TransformStream` with their default readers, writers, and controllers, `CountQueuingStrategy`, `ByteLengthQueuingStrategy`, `TextEncoderStream`, and `TextDecoderStream`, as globals and as `node:stream/web` exports, are constructor values with Node's prototypes, brand checks, and errors.
Their underlying sources, sinks, and transformers, queuing strategies with a `size` function, readers and writers, `tee`, `pipeTo` and `pipeThrough` with their options and an abort signal, `ReadableStream.from`, and async iteration settle promises in Node's order, and the streams take own properties.
`TextDecoderStream` decodes UTF-8; its other encodings are unsupported.
+2 -2
View File
@@ -1,5 +1,5 @@
- **V8 heap statistics:** `node:v8` can be imported.
`v8.getHeapStatistics()` returns Node's fields in Node's order, computed from the runtime allocator like `process.memoryUsage()`: `total_heap_size` and `total_physical_size` are the bytes the heap holds, `used_heap_size` the bytes allocated, `heap_size_limit` the machine's memory (the heap has no fixed ceiling), and `total_available_size` the limit minus the used bytes.
Fields that describe V8 itself (`total_heap_size_executable`, `malloced_memory`, `peak_malloced_memory`, the global handle sizes, `external_memory`, and `does_zap_garbage`) are `0`, and `number_of_native_contexts` is `1`.
`v8.getHeapStatistics()` returns Node's fields in Node's order, computed from the runtime allocator like `process.memoryUsage()`: `total_heap_size` and `total_physical_size` are the bytes the heap holds, `used_heap_size` the bytes allocated, `external_memory` the same figure as `process.memoryUsage().external`, `heap_size_limit` the machine's memory (the heap has no fixed ceiling), and `total_available_size` the limit minus the used bytes.
Fields that describe V8 itself (`total_heap_size_executable`, `malloced_memory`, `peak_malloced_memory`, the global handle sizes, and `does_zap_garbage`) are `0`, and `number_of_native_contexts` is `1`.
The values are approximate and are not V8 heap figures.
The other `node:v8` members observe the V8 engine, such as heap snapshots, profiles, flags, and serialization, and throw `SC2020` where they are used.
@@ -1,6 +1,8 @@
Some builtins can be stored and passed as functions:
- Lowered `node:path` functions across the bare, POSIX, and win32 modules, including optional `basename` and variadic `join`/`resolve`.
Called through an untyped value, they validate their arguments with Node's `ERR_INVALID_ARG_TYPE`.
A path module held by untyped code is a plain object with Node's members in Node's order.
- Supported zero-argument `node:os` functions and `querystring.escape`/`unescape`.
- `util.isDeepStrictEqual`, `styleText`, `stripVTControlCharacters`, `toUSVString`, `inherits`, `deprecate`, `callbackify`, `debuglog`, and `debug`.
- `fs.existsSync`, `unlinkSync`, `chmodSync`, `chownSync`, `renameSync`, and `closeSync`, and `fs/promises.unlink`, `chmod`, and `rename`.
@@ -12,9 +12,11 @@ Stored global console values and detached output methods retain identity and acc
Stored methods do not support format specifiers with substitution arguments.
Other stored console APIs, custom `Console` instances, and console mutation are unsupported.
JavaScript can store `process.stdin`, `stdout`, and `stderr`, replace output `write` methods, and call saved methods with their stream receiver.
JavaScript and TypeScript can store `process.stdin`, `stdout`, and `stderr`, replace output `write` methods, and call saved methods with their stream receiver.
A replaced `write` method does not receive console's error-callback argument.
Input supports data/end/error listeners, pause/resume, buffered reads, and TTY raw mode.
Output writes are synchronous, with successful callbacks on the next-tick queue; queued backpressure is not modeled.
Because no write is ever queued, `writableLength` is 0, `writableNeedDrain` is false, `writableHighWaterMark` is 65536, and `'drain'` never fires, so drain listeners register silently and `events.once(process.stdout, "drain")` stays pending, as it does in Node when no write returned false.
Stored streams do not support piping, encoding changes, arbitrary properties, or other events.
A TypeScript `NodeJS.WritableStream` parameter that only receives `process.stdout` or `process.stderr` supports `write(string)`.
When other streams flow into the same parameter, the process streams join them as `Writable` instances with the full writable surface.
@@ -0,0 +1,3 @@
- **Event order across native code and islands:** Timers, worker output, and other macrotasks from the program and from JavaScript it runs in a Node island share one event loop under Node.js, so their order is fixed.
A compiled program runs both on the event loop it hosts for Node.js, but it does not merge them into one order, so callbacks from the two sides can run in a different order than under Node.js, and the order can vary between runs.
Only programs that depend on the ordering between native and island callbacks are affected.
@@ -30842,9 +30842,9 @@
"inherited": true
},
"static": {
"status": "not-implemented",
"detail": "No static compiler lowering is registered for this API yet.",
"verification": "registry-gap"
"status": "partial",
"detail": "Implemented by a dedicated static compiler/runtime path.",
"verification": "test-backed"
}
},
{
@@ -30911,9 +30911,9 @@
"inherited": true
},
"static": {
"status": "not-implemented",
"detail": "No static compiler lowering is registered for this API yet.",
"verification": "registry-gap"
"status": "partial",
"detail": "Implemented by a dedicated static compiler/runtime path.",
"verification": "test-backed"
}
},
{
@@ -32659,9 +32659,9 @@
"inherited": true
},
"static": {
"status": "not-implemented",
"detail": "No static compiler lowering is registered for this API yet.",
"verification": "registry-gap"
"status": "partial",
"detail": "Implemented by a dedicated static compiler/runtime path.",
"verification": "test-backed"
}
},
{
@@ -44292,6 +44292,7 @@
"evidence": "derived:descendants",
"tests": [
"tests/corpus/1403-buffer-fs.ts",
"tests/corpus/1564-fs-watch.ts",
"tests/corpus/2595-fs-arg-ladders.cjs",
"tests/corpus/2686-fs-file-handle.ts",
"tests/corpus/2844-node-disposable-resources.ts",
@@ -44328,6 +44329,8 @@
"tests/corpus/6207-fs-constants-object.ts",
"tests/corpus/6208-fs-constants-require.cjs",
"tests/corpus/fs-native-realpath.ts",
"tests/corpus/fs-watch-file-names.ts",
"tests/corpus/fs-watch-file-polling.ts",
"tests/corpus/node24-fs-links-callbacks.cjs",
"tests/corpus/node24-fs-links-errors.cjs",
"tests/corpus/node24-fs-links-options.cjs",
@@ -47429,8 +47432,13 @@
"inherited": true
},
"static": {
"status": "not-implemented",
"evidence": "compiler-unmatched:fs"
"status": "partial",
"evidence": "compiler-dedicated:fs.unwatchFile",
"tests": [
"tests/corpus/1564-fs-watch.ts",
"tests/corpus/fs-watch-file-names.ts",
"tests/corpus/fs-watch-file-polling.ts"
]
},
"anchorSource": "exact"
},
@@ -47601,8 +47609,13 @@
"inherited": true
},
"static": {
"status": "not-implemented",
"evidence": "compiler-unmatched:fs"
"status": "partial",
"evidence": "compiler-dedicated:fs.watchFile",
"tests": [
"tests/corpus/1564-fs-watch.ts",
"tests/corpus/fs-watch-file-names.ts",
"tests/corpus/fs-watch-file-polling.ts"
]
},
"anchorSource": "exact"
},
@@ -49669,8 +49682,13 @@
"inherited": true
},
"static": {
"status": "not-implemented",
"evidence": "compiler-unmatched:fs"
"status": "partial",
"evidence": "compiler-dedicated:fs.FSWatcher",
"tests": [
"tests/corpus/1564-fs-watch.ts",
"tests/corpus/fs-watch-file-names.ts",
"tests/corpus/fs-watch-file-polling.ts"
]
},
"anchorSource": "exact"
},
@@ -79679,6 +79697,7 @@
"tests/corpus/process-builtin-values.js",
"tests/corpus/process-hrtime.js",
"tests/corpus/process-named-signals.cjs",
"tests/corpus/process-stream-drain.ts",
"tests/corpus/process-uncaught-handler-throws.cjs",
"tests/corpus/process-uncaught-loop.cjs",
"tests/corpus/process-uncaught-module.mjs",
@@ -79686,9 +79705,11 @@
"tests/corpus/process-uncaught-rejection.cjs",
"tests/corpus/process-uncaught-sync.cjs",
"tests/corpus/process-versions-storage.ts",
"tests/corpus/process-write-dynamic-chunks.ts",
"tests/corpus/stdio-values.mjs",
"tests/corpus/stdio-write-errors.mjs",
"tests/corpus/stdio-write-overrides.mjs",
"tests/corpus/stdout-write-patch.ts",
"tests/fixtures/node-types/env-files.ts",
"tests/fixtures/node-types/util-callables.ts",
"tests/harness/event-loop.test.ts",
@@ -82047,6 +82068,9 @@
"tests/corpus/stdio-values.mjs",
"tests/corpus/stdio-write-overrides.mjs",
"tests/corpus/stdio-write-errors.mjs",
"tests/corpus/stdout-write-patch.ts",
"tests/corpus/process-stream-drain.ts",
"tests/corpus/process-write-dynamic-chunks.ts",
"tests/harness/event-loop.test.ts"
]
},
@@ -82089,6 +82113,9 @@
"tests/corpus/stdio-values.mjs",
"tests/corpus/stdio-write-overrides.mjs",
"tests/corpus/stdio-write-errors.mjs",
"tests/corpus/stdout-write-patch.ts",
"tests/corpus/process-stream-drain.ts",
"tests/corpus/process-write-dynamic-chunks.ts",
"tests/harness/event-loop.test.ts"
]
},
@@ -82131,6 +82158,9 @@
"tests/corpus/stdio-values.mjs",
"tests/corpus/stdio-write-overrides.mjs",
"tests/corpus/stdio-write-errors.mjs",
"tests/corpus/stdout-write-patch.ts",
"tests/corpus/process-stream-drain.ts",
"tests/corpus/process-write-dynamic-chunks.ts",
"tests/harness/event-loop.test.ts"
]
},
@@ -86933,7 +86963,8 @@
"tests/corpus/5611-readable-from-class-iterables.ts",
"tests/corpus/5612-readable-from-iterables.cjs",
"tests/corpus/5617-pipeline-function-ends.ts",
"tests/corpus/5619-pipeline-function-ends.cjs"
"tests/corpus/5619-pipeline-function-ends.cjs",
"tests/corpus/writable-error-callbacks.ts"
]
},
"anchorSource": "chapter"
@@ -87059,7 +87090,8 @@
"tests/corpus/5425-pipeline-generator-stage.ts",
"tests/corpus/5428-pipeline-generator-stages.cjs",
"tests/corpus/5617-pipeline-function-ends.ts",
"tests/corpus/5619-pipeline-function-ends.cjs"
"tests/corpus/5619-pipeline-function-ends.cjs",
"tests/corpus/writable-error-callbacks.ts"
]
},
"anchorSource": "exact"
@@ -87093,7 +87125,8 @@
"tests/corpus/5425-pipeline-generator-stage.ts",
"tests/corpus/5428-pipeline-generator-stages.cjs",
"tests/corpus/5617-pipeline-function-ends.ts",
"tests/corpus/5619-pipeline-function-ends.cjs"
"tests/corpus/5619-pipeline-function-ends.cjs",
"tests/corpus/writable-error-callbacks.ts"
]
},
"anchorSource": "exact"
@@ -87127,7 +87160,8 @@
"tests/corpus/5425-pipeline-generator-stage.ts",
"tests/corpus/5428-pipeline-generator-stages.cjs",
"tests/corpus/5617-pipeline-function-ends.ts",
"tests/corpus/5619-pipeline-function-ends.cjs"
"tests/corpus/5619-pipeline-function-ends.cjs",
"tests/corpus/writable-error-callbacks.ts"
]
},
"anchorSource": "exact"
@@ -87184,7 +87218,8 @@
"tests/corpus/5425-pipeline-generator-stage.ts",
"tests/corpus/5428-pipeline-generator-stages.cjs",
"tests/corpus/5617-pipeline-function-ends.ts",
"tests/corpus/5619-pipeline-function-ends.cjs"
"tests/corpus/5619-pipeline-function-ends.cjs",
"tests/corpus/writable-error-callbacks.ts"
]
},
"anchorSource": "exact"
@@ -87218,7 +87253,8 @@
"tests/corpus/5425-pipeline-generator-stage.ts",
"tests/corpus/5428-pipeline-generator-stages.cjs",
"tests/corpus/5617-pipeline-function-ends.ts",
"tests/corpus/5619-pipeline-function-ends.cjs"
"tests/corpus/5619-pipeline-function-ends.cjs",
"tests/corpus/writable-error-callbacks.ts"
]
},
"anchorSource": "exact"
@@ -87252,7 +87288,8 @@
"tests/corpus/5425-pipeline-generator-stage.ts",
"tests/corpus/5428-pipeline-generator-stages.cjs",
"tests/corpus/5617-pipeline-function-ends.ts",
"tests/corpus/5619-pipeline-function-ends.cjs"
"tests/corpus/5619-pipeline-function-ends.cjs",
"tests/corpus/writable-error-callbacks.ts"
]
},
"anchorSource": "exact"
@@ -87648,7 +87685,8 @@
"tests/corpus/5425-pipeline-generator-stage.ts",
"tests/corpus/5428-pipeline-generator-stages.cjs",
"tests/corpus/5617-pipeline-function-ends.ts",
"tests/corpus/5619-pipeline-function-ends.cjs"
"tests/corpus/5619-pipeline-function-ends.cjs",
"tests/corpus/writable-error-callbacks.ts"
]
},
"anchorSource": "exact"
@@ -87705,7 +87743,8 @@
"tests/corpus/5425-pipeline-generator-stage.ts",
"tests/corpus/5428-pipeline-generator-stages.cjs",
"tests/corpus/5617-pipeline-function-ends.ts",
"tests/corpus/5619-pipeline-function-ends.cjs"
"tests/corpus/5619-pipeline-function-ends.cjs",
"tests/corpus/writable-error-callbacks.ts"
]
},
"anchorSource": "exact"
@@ -87739,7 +87778,8 @@
"tests/corpus/5425-pipeline-generator-stage.ts",
"tests/corpus/5428-pipeline-generator-stages.cjs",
"tests/corpus/5617-pipeline-function-ends.ts",
"tests/corpus/5619-pipeline-function-ends.cjs"
"tests/corpus/5619-pipeline-function-ends.cjs",
"tests/corpus/writable-error-callbacks.ts"
]
},
"anchorSource": "exact"
@@ -87796,7 +87836,8 @@
"tests/corpus/5425-pipeline-generator-stage.ts",
"tests/corpus/5428-pipeline-generator-stages.cjs",
"tests/corpus/5617-pipeline-function-ends.ts",
"tests/corpus/5619-pipeline-function-ends.cjs"
"tests/corpus/5619-pipeline-function-ends.cjs",
"tests/corpus/writable-error-callbacks.ts"
]
},
"anchorSource": "ancestor"
@@ -87830,7 +87871,8 @@
"tests/corpus/5425-pipeline-generator-stage.ts",
"tests/corpus/5428-pipeline-generator-stages.cjs",
"tests/corpus/5617-pipeline-function-ends.ts",
"tests/corpus/5619-pipeline-function-ends.cjs"
"tests/corpus/5619-pipeline-function-ends.cjs",
"tests/corpus/writable-error-callbacks.ts"
]
},
"anchorSource": "ancestor"
@@ -87864,7 +87906,8 @@
"tests/corpus/5425-pipeline-generator-stage.ts",
"tests/corpus/5428-pipeline-generator-stages.cjs",
"tests/corpus/5617-pipeline-function-ends.ts",
"tests/corpus/5619-pipeline-function-ends.cjs"
"tests/corpus/5619-pipeline-function-ends.cjs",
"tests/corpus/writable-error-callbacks.ts"
]
},
"anchorSource": "ancestor"
@@ -88260,7 +88303,8 @@
"tests/corpus/5425-pipeline-generator-stage.ts",
"tests/corpus/5428-pipeline-generator-stages.cjs",
"tests/corpus/5617-pipeline-function-ends.ts",
"tests/corpus/5619-pipeline-function-ends.cjs"
"tests/corpus/5619-pipeline-function-ends.cjs",
"tests/corpus/writable-error-callbacks.ts"
]
},
"anchorSource": "ancestor"
@@ -88363,7 +88407,8 @@
"tests/corpus/5425-pipeline-generator-stage.ts",
"tests/corpus/5428-pipeline-generator-stages.cjs",
"tests/corpus/5617-pipeline-function-ends.ts",
"tests/corpus/5619-pipeline-function-ends.cjs"
"tests/corpus/5619-pipeline-function-ends.cjs",
"tests/corpus/writable-error-callbacks.ts"
]
},
"anchorSource": "exact"
@@ -88423,7 +88468,8 @@
"tests/corpus/5425-pipeline-generator-stage.ts",
"tests/corpus/5428-pipeline-generator-stages.cjs",
"tests/corpus/5617-pipeline-function-ends.ts",
"tests/corpus/5619-pipeline-function-ends.cjs"
"tests/corpus/5619-pipeline-function-ends.cjs",
"tests/corpus/writable-error-callbacks.ts"
]
},
"anchorSource": "exact"
@@ -88457,7 +88503,8 @@
"tests/corpus/5425-pipeline-generator-stage.ts",
"tests/corpus/5428-pipeline-generator-stages.cjs",
"tests/corpus/5617-pipeline-function-ends.ts",
"tests/corpus/5619-pipeline-function-ends.cjs"
"tests/corpus/5619-pipeline-function-ends.cjs",
"tests/corpus/writable-error-callbacks.ts"
]
},
"anchorSource": "exact"
@@ -88491,7 +88538,8 @@
"tests/corpus/5425-pipeline-generator-stage.ts",
"tests/corpus/5428-pipeline-generator-stages.cjs",
"tests/corpus/5617-pipeline-function-ends.ts",
"tests/corpus/5619-pipeline-function-ends.cjs"
"tests/corpus/5619-pipeline-function-ends.cjs",
"tests/corpus/writable-error-callbacks.ts"
]
},
"anchorSource": "exact"
@@ -88548,7 +88596,8 @@
"tests/corpus/5425-pipeline-generator-stage.ts",
"tests/corpus/5428-pipeline-generator-stages.cjs",
"tests/corpus/5617-pipeline-function-ends.ts",
"tests/corpus/5619-pipeline-function-ends.cjs"
"tests/corpus/5619-pipeline-function-ends.cjs",
"tests/corpus/writable-error-callbacks.ts"
]
},
"anchorSource": "exact"
@@ -89141,7 +89190,8 @@
"tests/corpus/5425-pipeline-generator-stage.ts",
"tests/corpus/5428-pipeline-generator-stages.cjs",
"tests/corpus/5617-pipeline-function-ends.ts",
"tests/corpus/5619-pipeline-function-ends.cjs"
"tests/corpus/5619-pipeline-function-ends.cjs",
"tests/corpus/writable-error-callbacks.ts"
]
},
"anchorSource": "exact"
@@ -89175,7 +89225,8 @@
"tests/corpus/5425-pipeline-generator-stage.ts",
"tests/corpus/5428-pipeline-generator-stages.cjs",
"tests/corpus/5617-pipeline-function-ends.ts",
"tests/corpus/5619-pipeline-function-ends.cjs"
"tests/corpus/5619-pipeline-function-ends.cjs",
"tests/corpus/writable-error-callbacks.ts"
]
},
"anchorSource": "exact"
@@ -89209,7 +89260,8 @@
"tests/corpus/5425-pipeline-generator-stage.ts",
"tests/corpus/5428-pipeline-generator-stages.cjs",
"tests/corpus/5617-pipeline-function-ends.ts",
"tests/corpus/5619-pipeline-function-ends.cjs"
"tests/corpus/5619-pipeline-function-ends.cjs",
"tests/corpus/writable-error-callbacks.ts"
]
},
"anchorSource": "exact"
@@ -89243,7 +89295,8 @@
"tests/corpus/5425-pipeline-generator-stage.ts",
"tests/corpus/5428-pipeline-generator-stages.cjs",
"tests/corpus/5617-pipeline-function-ends.ts",
"tests/corpus/5619-pipeline-function-ends.cjs"
"tests/corpus/5619-pipeline-function-ends.cjs",
"tests/corpus/writable-error-callbacks.ts"
]
},
"anchorSource": "exact"
@@ -89277,7 +89330,8 @@
"tests/corpus/5425-pipeline-generator-stage.ts",
"tests/corpus/5428-pipeline-generator-stages.cjs",
"tests/corpus/5617-pipeline-function-ends.ts",
"tests/corpus/5619-pipeline-function-ends.cjs"
"tests/corpus/5619-pipeline-function-ends.cjs",
"tests/corpus/writable-error-callbacks.ts"
]
},
"anchorSource": "exact"
@@ -89311,7 +89365,8 @@
"tests/corpus/5425-pipeline-generator-stage.ts",
"tests/corpus/5428-pipeline-generator-stages.cjs",
"tests/corpus/5617-pipeline-function-ends.ts",
"tests/corpus/5619-pipeline-function-ends.cjs"
"tests/corpus/5619-pipeline-function-ends.cjs",
"tests/corpus/writable-error-callbacks.ts"
]
},
"anchorSource": "exact"
@@ -89345,7 +89400,8 @@
"tests/corpus/5425-pipeline-generator-stage.ts",
"tests/corpus/5428-pipeline-generator-stages.cjs",
"tests/corpus/5617-pipeline-function-ends.ts",
"tests/corpus/5619-pipeline-function-ends.cjs"
"tests/corpus/5619-pipeline-function-ends.cjs",
"tests/corpus/writable-error-callbacks.ts"
]
},
"anchorSource": "exact"
@@ -89379,7 +89435,8 @@
"tests/corpus/5425-pipeline-generator-stage.ts",
"tests/corpus/5428-pipeline-generator-stages.cjs",
"tests/corpus/5617-pipeline-function-ends.ts",
"tests/corpus/5619-pipeline-function-ends.cjs"
"tests/corpus/5619-pipeline-function-ends.cjs",
"tests/corpus/writable-error-callbacks.ts"
]
},
"anchorSource": "exact"
@@ -89413,7 +89470,8 @@
"tests/corpus/5425-pipeline-generator-stage.ts",
"tests/corpus/5428-pipeline-generator-stages.cjs",
"tests/corpus/5617-pipeline-function-ends.ts",
"tests/corpus/5619-pipeline-function-ends.cjs"
"tests/corpus/5619-pipeline-function-ends.cjs",
"tests/corpus/writable-error-callbacks.ts"
]
},
"anchorSource": "exact"
@@ -90832,7 +90890,8 @@
"tests/corpus/5425-pipeline-generator-stage.ts",
"tests/corpus/5428-pipeline-generator-stages.cjs",
"tests/corpus/5617-pipeline-function-ends.ts",
"tests/corpus/5619-pipeline-function-ends.cjs"
"tests/corpus/5619-pipeline-function-ends.cjs",
"tests/corpus/writable-error-callbacks.ts"
]
},
"anchorSource": "exact"
@@ -91096,7 +91155,8 @@
"tests/corpus/5425-pipeline-generator-stage.ts",
"tests/corpus/5428-pipeline-generator-stages.cjs",
"tests/corpus/5617-pipeline-function-ends.ts",
"tests/corpus/5619-pipeline-function-ends.cjs"
"tests/corpus/5619-pipeline-function-ends.cjs",
"tests/corpus/writable-error-callbacks.ts"
]
},
"anchorSource": "ancestor"
@@ -91130,7 +91190,8 @@
"tests/corpus/5425-pipeline-generator-stage.ts",
"tests/corpus/5428-pipeline-generator-stages.cjs",
"tests/corpus/5617-pipeline-function-ends.ts",
"tests/corpus/5619-pipeline-function-ends.cjs"
"tests/corpus/5619-pipeline-function-ends.cjs",
"tests/corpus/writable-error-callbacks.ts"
]
},
"anchorSource": "ancestor"
+4 -3
View File
@@ -314,7 +314,7 @@
{ "chapter": "process", "signatures": ["`process.hrtime([time])`", "`process.hrtime.bigint()`"], "status": "partial", "evidence": ["tests/corpus/process-hrtime.js"] },
{ "chapter": "errors", "signatures": ["`stackTraceLimit` Type: {number}"], "status": "partial", "evidence": ["tests/corpus/error-stack-limit.ts"] },
{ "chapter": "errors", "signatures": ["`stack` Type: {string}"], "status": "partial", "evidence": ["tests/corpus/error-stack-frames.ts", "tests/corpus/error-stack-limit.ts"] },
{ "chapter": "process", "signatures": ["`stdin` Type: {Stream}", "`stdout` Type: {Stream}", "`stderr` Type: {Stream}"], "status": "partial", "evidence": ["tests/corpus/stdio-values.mjs", "tests/corpus/stdio-write-overrides.mjs", "tests/corpus/stdio-write-errors.mjs", "tests/harness/event-loop.test.ts"] },
{ "chapter": "process", "signatures": ["`stdin` Type: {Stream}", "`stdout` Type: {Stream}", "`stderr` Type: {Stream}"], "status": "partial", "evidence": ["tests/corpus/stdio-values.mjs", "tests/corpus/stdio-write-overrides.mjs", "tests/corpus/stdio-write-errors.mjs", "tests/corpus/stdout-write-patch.ts", "tests/corpus/process-stream-drain.ts", "tests/corpus/process-write-dynamic-chunks.ts", "tests/harness/event-loop.test.ts"] },
{ "chapter": "process", "signatures": ["Signal events"], "status": "partial", "evidence": ["tests/corpus/1443-signal-handlers.ts", "tests/corpus/process-named-signals.cjs"] },
{ "chapter": "process", "signatures": ["Event: `'uncaughtException'`", "Event: `'uncaughtExceptionMonitor'`"], "status": "partial", "evidence": ["tests/corpus/process-uncaught-sync.cjs", "tests/corpus/process-uncaught-loop.cjs", "tests/corpus/process-uncaught-module.mjs", "tests/corpus/process-uncaught-rejection.cjs", "tests/corpus/process-uncaught-monitor.cjs", "tests/corpus/process-uncaught-handler-throws.cjs"] },
{ "chapter": "process", "signatures": ["`process.getBuiltinModule(id)`"], "status": "partial", "evidence": ["tests/corpus/process-builtin-values.js", "packages/compiler/test/process-builtin-values.test.ts"] },
@@ -478,13 +478,14 @@
{ "symbols": ["StringDecoder", "stringDecoder.write", "stringDecoder.end"], "status": "partial", "evidence": ["tests/corpus/1662-string-decoder-encodings.ts", "tests/corpus/node24-string-decoder.ts", "tests/corpus/5882-string-decoder-runtime-encoding.ts"] },
{ "symbols": ["setImmediate", "setInterval", "setTimeout", "clearImmediate", "clearInterval", "clearTimeout", "timersPromises.setTimeout", "timersPromises.setImmediate", "timersPromises.setInterval", "immediate.hasRef", "immediate.ref", "immediate.unref", "timeout.hasRef", "timeout.ref", "timeout.unref", "timeout.refresh"], "status": "partial", "evidence": ["tests/corpus/1804-timers-module.ts", "tests/corpus/1805-timer-callback-args.ts", "tests/corpus/2093-timers-promises.ts", "tests/corpus/2852-timers-promises-interval.ts", "tests/corpus/5830-timers-promises-signal.ts"] },
{ "symbols": ["timersPromises.scheduler.wait", "timersPromises.scheduler.yield"], "status": "partial", "evidence": ["tests/corpus/5839-timers-promises-scheduler.ts", "tests/corpus/5889-timers-scheduler-signal.ts"] },
{ "symbols": ["stream.finished", "stream.pipeline", "stream.getDefaultHighWaterMark", "readable.push", "readable.unshift", "readable.read", "readable.pause", "readable.resume", "readable.isPaused", "readable.setEncoding", "readable.pipe", "readable.unpipe", "readable.destroy", "writable.write", "writable.end", "writable.cork", "writable.uncork", "writable.setDefaultEncoding"], "status": "partial", "evidence": ["tests/corpus/1685-stream-readable-basics.ts", "tests/corpus/1688-stream-writable-basics.ts", "tests/corpus/1814-stream-pipeline.cjs", "tests/corpus/5302-stream-promises-options.ts", "tests/corpus/5303-stream-finished-callback-opts.cjs", "tests/corpus/5425-pipeline-generator-stage.ts", "tests/corpus/5428-pipeline-generator-stages.cjs", "tests/corpus/5617-pipeline-function-ends.ts", "tests/corpus/5619-pipeline-function-ends.cjs"] },
{ "symbols": ["stream.finished", "stream.pipeline", "stream.getDefaultHighWaterMark", "readable.push", "readable.unshift", "readable.read", "readable.pause", "readable.resume", "readable.isPaused", "readable.setEncoding", "readable.pipe", "readable.unpipe", "readable.destroy", "writable.write", "writable.end", "writable.cork", "writable.uncork", "writable.setDefaultEncoding"], "status": "partial", "evidence": ["tests/corpus/1685-stream-readable-basics.ts", "tests/corpus/1688-stream-writable-basics.ts", "tests/corpus/1814-stream-pipeline.cjs", "tests/corpus/5302-stream-promises-options.ts", "tests/corpus/5303-stream-finished-callback-opts.cjs", "tests/corpus/5425-pipeline-generator-stage.ts", "tests/corpus/5428-pipeline-generator-stages.cjs", "tests/corpus/5617-pipeline-function-ends.ts", "tests/corpus/5619-pipeline-function-ends.cjs", "tests/corpus/writable-error-callbacks.ts"] },
{ "symbols": ["net.createServer", "net.connect", "net.createConnection", "net.getDefaultAutoSelectFamilyAttemptTimeout", "net.setDefaultAutoSelectFamilyAttemptTimeout", "server.address", "server.close", "server.listen", "socket.destroy", "socket.end", "socket.pause", "socket.resume", "socket.setEncoding", "socket.setTimeout", "socket.write"], "status": "partial", "evidence": ["tests/corpus/2500-net-autosel-timeout.ts", "tests/corpus/2596-net-arg-ladders.cjs"] },
{ "symbols": ["dgram.createSocket", "socket.address", "socket.bind", "socket.close", "socket.connect", "socket.ref", "socket.remoteAddress", "socket.send", "socket.unref"], "status": "partial", "evidence": ["tests/corpus/2597-dgram-send-ladders.cjs"] },
{ "symbols": ["net.SocketAddress", "net.BlockList"], "status": "partial", "evidence": ["tests/corpus/5659-net-blocklist-socketaddress.ts", "tests/fixtures/node-types/net-blocklist.ts"] },
{ "symbols": ["SocketAddress.parse", "BlockList.isBlockList", "blockList.addAddress", "blockList.addAddresses", "blockList.addCIDR", "blockList.addCIDRs", "blockList.addRange", "blockList.addSubnet", "blockList.check", "blockList.clear", "blockList.fromJSON", "blockList.removeAddress", "blockList.removeCIDR", "blockList.removeRange", "blockList.removeSubnet", "blockList.toJSON"], "status": "supported", "evidence": ["tests/corpus/5659-net-blocklist-socketaddress.ts", "tests/fixtures/node-types/net-blocklist.ts"] },
{ "symbols": ["readline.createInterface", "rl.close"], "status": "partial", "evidence": ["tests/corpus/1475-readline-closed-stdin.ts", "tests/corpus/5878-readline-dynamic-streams.mjs"] },
{ "symbols": ["test", "it", "test.skip", "test.todo", "test.only", "describe", "suite", "before", "after", "beforeEach", "afterEach", "context.test", "context.skip", "context.todo", "context.assert"], "status": "partial", "evidence": ["tests/harness/node-test.test.ts"] },
{ "symbols": ["http.request", "http.get", "http.createServer", "https.request", "https.get", "https.createServer", "request.end", "request.destroy", "request.setTimeout", "request.write", "response.end", "response.getHeader", "response.hasHeader", "response.removeHeader", "response.setHeader", "response.write", "response.writeHead", "headersTimeout", "requestTimeout", "timeout", "keepAliveTimeout", "keepAliveTimeoutBuffer"], "status": "partial", "evidence": ["tests/corpus/1780-http-res-surface.ts", "tests/corpus/2672-http-request-response-callback.ts", "tests/corpus/2689-http-server-timeout-properties.ts"] }
{ "symbols": ["http.request", "http.get", "http.createServer", "https.request", "https.get", "https.createServer", "request.end", "request.destroy", "request.setTimeout", "request.write", "response.end", "response.getHeader", "response.hasHeader", "response.removeHeader", "response.setHeader", "response.write", "response.writeHead", "headersTimeout", "requestTimeout", "timeout", "keepAliveTimeout", "keepAliveTimeoutBuffer"], "status": "partial", "evidence": ["tests/corpus/1780-http-res-surface.ts", "tests/corpus/2672-http-request-response-callback.ts", "tests/corpus/2689-http-server-timeout-properties.ts"] },
{ "symbols": ["fs.FSWatcher", "fs.watchFile", "fs.unwatchFile"], "status": "partial", "evidence": ["tests/corpus/1564-fs-watch.ts", "tests/corpus/fs-watch-file-names.ts", "tests/corpus/fs-watch-file-polling.ts"] }
]
}
+143 -24
View File
@@ -252,15 +252,20 @@ struct UnnameStructTypes : PassInfoMixin<UnnameStructTypes> {
// over partitions: functions that received its counts, functions whose
// control flow changed since it was collected (hash mismatch), and functions
// it has no record of (new since it was collected, or never run in training
// when the profile was merged with --sparse). A partition reused from the cache without its counts is counted
// in Unknown. Recovered counts the mismatched functions that --profile-blocks
// annotated, so it never exceeds Mismatched.
// when the profile was merged with --sparse). A partition reused from the cache
// without its counts is counted in Unknown. Recovered counts the mismatched
// functions that --profile-blocks annotated, so it never exceeds Mismatched.
// The two weights sum the profile's counters for the annotated and for the
// mismatched functions, so a changed hot function weighs more than a changed
// cold one.
struct ProfileMatch {
std::atomic<uint64_t> Annotated{0};
std::atomic<uint64_t> Mismatched{0};
std::atomic<uint64_t> Missing{0};
std::atomic<uint64_t> Recovered{0};
std::atomic<uint64_t> Unknown{0};
std::atomic<uint64_t> AnnotatedCounts{0};
std::atomic<uint64_t> MismatchedCounts{0};
// Set once the annotation before inlining has run. The context-sensitive
// annotation after inlining, which a single-module pipeline also runs,
// reports its mismatched and missing functions again; they are not
@@ -272,27 +277,32 @@ struct ProfileMatch {
Mismatched += Other.Mismatched;
Missing += Other.Missing;
Recovered += Other.Recovered;
AnnotatedCounts += Other.AnnotatedCounts;
MismatchedCounts += Other.MismatchedCounts;
}
std::string serialize() const {
return formatv("{0} {1} {2} {3}", Annotated.load(), Mismatched.load(),
Missing.load(), Recovered.load())
return formatv("{0} {1} {2} {3} {4} {5}", Annotated.load(),
Mismatched.load(), Missing.load(), Recovered.load(),
AnnotatedCounts.load(), MismatchedCounts.load())
.str();
}
bool deserialize(StringRef Text) {
SmallVector<StringRef, 4> Parts;
SmallVector<StringRef, 6> Parts;
Text.split(Parts, ' ');
uint64_t Values[4];
if (Parts.size() != 4)
uint64_t Values[6];
if (Parts.size() != 6)
return false;
for (size_t I = 0; I < 4; ++I)
for (size_t I = 0; I < 6; ++I)
if (Parts[I].getAsInteger(10, Values[I]))
return false;
Annotated += Values[0];
Mismatched += Values[1];
Missing += Values[2];
Recovered += Values[3];
AnnotatedCounts += Values[4];
MismatchedCounts += Values[5];
return true;
}
};
@@ -374,23 +384,34 @@ bool hasProfileCounts(const Function &F) {
// Runs right after the profile annotation, before inlining and dead code
// elimination change the functions it annotated: counts them and records
// their block counts, then annotates the functions whose control flow
// changed from FallbackCounts. A function the profile has no record of keeps
// no counts, as without block counts. Returns the analyses that remain valid.
// their block counts, then annotates from FallbackCounts the functions whose
// control flow changed, and the functions the profile has no record of that
// FallbackCounts records as never run. A sparse profile leaves out the
// functions its training never ran; the block counts name them, so they get
// an entry count of 0 and LLVM optimizes them as cold code, as a profile with
// zero counts would make it. A function neither names keeps no counts, as
// without block counts: it was added after the profile was collected.
// Returns the analyses that remain valid.
PreservedAnalyses afterProfileAnnotation(Module &M, ProfileMatch &Match) {
std::vector<Function *> Mismatched;
std::vector<Function *> Unrecorded;
for (Function &F : M) {
if (F.isDeclaration())
continue;
if (!hasProfileCounts(F)) {
if (hasHashMismatchTag(F))
Mismatched.push_back(&F);
else
Unrecorded.push_back(&F);
continue;
}
++Match.Annotated;
if (Recorder)
Recorder->add(F);
}
if (Recorder)
for (Function *F : Unrecorded)
Recorder->addNeverRan(*F);
if (!FallbackCounts)
return PreservedAnalyses::all();
ProfileSummaryInfo Summary(M);
@@ -403,6 +424,13 @@ PreservedAnalyses afterProfileAnnotation(Module &M, ProfileMatch &Match) {
++Match.Recovered;
Preserved = PreservedAnalyses::none();
}
for (Function *F : Unrecorded) {
auto Record = FallbackCounts->find(F->getName());
if (Record == FallbackCounts->end() || Record->second.EntryCount != 0)
continue;
annotateFromBlocks(*F, Record->second, Summary);
Preserved = PreservedAnalyses::none();
}
return Preserved;
}
@@ -412,11 +440,77 @@ void reportProfileMatch(const ProfileMatch &Match) {
{"mismatched", Match.Mismatched.load()},
{"missing", Match.Missing.load()},
{"recovered", Match.Recovered.load()},
{"unknown_partitions", Match.Unknown.load()}};
{"unknown_partitions", Match.Unknown.load()},
{"annotated_counts", Match.AnnotatedCounts.load()},
{"mismatched_counts", Match.MismatchedCounts.load()}};
json::Object Response{{"ok", true}, {"profile", std::move(Profile)}};
outs() << formatv("{0}\n", json::Value(std::move(Response)));
}
// The sum of each function's counters in the indexed profile at `Path`, by
// profile name, read once per process. An unreadable profile has no counts
// (the annotation reports it).
const StringMap<uint64_t> &profileCounts(const std::string &Path) {
static std::once_flag Once;
static StringMap<uint64_t> Counts;
std::call_once(Once, [&Path] {
Expected<std::unique_ptr<IndexedInstrProfReader>> Reader =
IndexedInstrProfReader::create(Path, *vfs::getRealFileSystem());
if (!Reader) {
consumeError(Reader.takeError());
return;
}
for (const NamedInstrProfRecord &Record : **Reader) {
uint64_t Sum = 0;
for (uint64_t Count : Record.Counts)
Sum += Count;
Counts[Record.Name] += Sum;
}
consumeError((*Reader)->getError());
});
return Counts;
}
// Weighs one module's profile match by the profile's counters: collects the
// annotated and the mismatched functions while the module is optimized,
// then adds their counts to the match.
class MatchWeigher {
public:
explicit MatchWeigher(const Module &M) {
for (const Function &F : M)
if (!F.isDeclaration())
ProfileNames[F.getName()] = getIRPGOFuncName(F);
}
void weigh(const std::string &ProfileUse, ProfileMatch &Match) const {
const StringMap<uint64_t> &Counts = profileCounts(ProfileUse);
auto CountsOf = [&Counts](StringRef Name) -> uint64_t {
auto It = Counts.find(Name);
return It == Counts.end() ? 0 : It->second;
};
// A context-sensitive profile annotates the module twice; each function
// counts once.
StringSet<> Seen;
for (const std::string &Name : Annotated)
if (Seen.insert(Name).second)
Match.AnnotatedCounts += CountsOf(Name);
// The mismatch warnings name functions by symbol.
for (const std::string &Name : Mismatched) {
auto It = ProfileNames.find(Name);
StringRef ProfileName =
It == ProfileNames.end() ? StringRef(Name) : StringRef(It->second);
if (Seen.insert(ProfileName).second)
Match.MismatchedCounts += CountsOf(ProfileName);
}
}
std::vector<std::string> Annotated;
std::vector<std::string> Mismatched;
private:
StringMap<std::string> ProfileNames;
};
// Reads the profile's header once, before any partition needs it: it must
// hold IR-level counts. Context-sensitive counts lay out the code by them;
// a profile that has them cannot take a second set.
@@ -515,13 +609,15 @@ struct Pipeline {
});
}
// Adds to Names the profile names of the functions the annotation gave
// counts to, as soon as that pass has run.
// Adds to Names the profile names of the functions the annotation before
// inlining gave counts to, as soon as that pass has run.
void collectAnnotated(std::vector<std::string> &Names) {
Callbacks.registerAfterPassCallback(
[&Names](StringRef PassID, Any IR, const PreservedAnalyses &) {
if (PassID != "PGOInstrumentationUse")
[&Names, Done = false](StringRef PassID, Any IR,
const PreservedAnalyses &) mutable {
if (Done || PassID != "PGOInstrumentationUse")
return;
Done = true;
if (const auto *M = llvm::any_cast<const Module *>(&IR))
for (const Function &F : **M) {
std::optional<Function::ProfileCount> Entry = F.getEntryCount();
@@ -714,13 +810,21 @@ emitModule(Module &M, TargetMachine &Machine, OptimizationLevel Level,
StringRef OutputPath, std::vector<SmallString<256>> &Temporaries,
ProfileMatch &Match) {
bool Annotating = Profile && Profile->Action == PGOOptions::IRUse;
std::string ProfileUse = Annotating ? Profile->ProfileFile : "";
MatchWeigher Weigher(M);
if (Annotating)
M.getContext().setDiagnosticHandler(
std::make_unique<ProfileMatchHandler>(Match));
std::make_unique<ProfileMatchHandler>(Match, &Weigher.Mismatched));
Pipeline P(Machine, Level, std::move(Profile));
if (Annotating)
if (Annotating) {
// Registered first, so it lists the functions the profile itself
// annotated, before block counts recover any.
P.collectAnnotated(Weigher.Annotated);
P.afterProfileAnnotation(Match);
}
P.PB.buildPerModuleDefaultPipeline(Level).run(M, P.MAM);
if (Annotating)
Weigher.weigh(ProfileUse, Match);
if (std::optional<EmitFailure> Failure =
verify(M, "post_optimization_verification_failed"))
return Failure;
@@ -911,14 +1015,16 @@ compileOptimizedPartitions(const EmitOptions &Options,
return Failures.set({"target_machine_failed", LookupError});
LLVMContext Context;
ProfileMatch PartitionMatch;
std::vector<std::string> MismatchedNames;
if (Annotating)
Context.setDiagnosticHandler(
std::make_unique<ProfileMatchHandler>(PartitionMatch));
Context.setDiagnosticHandler(std::make_unique<ProfileMatchHandler>(
PartitionMatch, &MismatchedNames));
Expected<std::unique_ptr<Module>> Loaded =
loadPartition(Options, Plan.Modules[I], I, Context, *Machine);
if (!Loaded)
return Failures.set({"partition_failed", toString(Loaded.takeError())});
Module &Mod = **Loaded;
MatchWeigher Weigher(Mod);
if (!Options.ImportBitcode.empty()) {
// See emit(): the attribute is inert without a sanitizer pass and
// would block inlining of the imported runtime bodies.
@@ -937,8 +1043,12 @@ compileOptimizedPartitions(const EmitOptions &Options,
// Profile instrumentation or annotation happens here, before the thin
// link; the post-link optimization reads the annotated weights.
Pipeline P(*Machine, Level, profileOptions(Options));
if (Annotating)
if (Annotating) {
// Registered first, so it lists the functions the profile itself
// annotated, before block counts recover any.
P.collectAnnotated(Weigher.Annotated);
P.afterProfileAnnotation(PartitionMatch);
}
ModulePassManager MPM = P.PB.buildThinLTOPreLinkDefaultPipeline(Level);
MPM.addPass(UnnameStructTypes());
// The module hash is set at the thin link instead (see Names).
@@ -948,6 +1058,8 @@ compileOptimizedPartitions(const EmitOptions &Options,
MPM.run(Mod, P.MAM);
}
if (Annotating) {
Weigher.Mismatched = std::move(MismatchedNames);
Weigher.weigh(Options.ProfileUse, PartitionMatch);
Match.add(PartitionMatch);
// Stored before the summary, so a reused summary finds its counts.
Cache.store(Key, "match", PartitionMatch.serialize());
@@ -1445,15 +1557,22 @@ std::optional<EmitFailure>
emitSplitCodegen(const EmitOptions &Options, Module &M, TargetMachine &Machine,
OptimizationLevel Level, ProfileMatch &Match) {
bool Annotating = !Options.ProfileUse.empty();
MatchWeigher Weigher(M);
if (Annotating)
M.getContext().setDiagnosticHandler(
std::make_unique<ProfileMatchHandler>(Match));
std::make_unique<ProfileMatchHandler>(Match, &Weigher.Mismatched));
{
Pipeline P(Machine, Level, profileOptions(Options));
if (Annotating)
if (Annotating) {
// Registered first, so it lists the functions the profile itself
// annotated, before block counts recover any.
P.collectAnnotated(Weigher.Annotated);
P.afterProfileAnnotation(Match);
}
P.PB.buildPerModuleDefaultPipeline(Level).run(M, P.MAM);
}
if (Annotating)
Weigher.weigh(Options.ProfileUse, Match);
if (std::optional<EmitFailure> Failure =
verify(M, "post_optimization_verification_failed"))
return Failure;
+67 -30
View File
@@ -176,71 +176,108 @@ void balanceRoots(std::vector<Symbol> &Symbols, ArrayRef<unsigned> Roots,
ArrayRef<char> Live, unsigned Count) {
if (Count >= MinGroupedPartitions)
return;
// A root's size leaves out the digits of metadata references, which an
// edit elsewhere in the module can renumber.
auto TextBytes = [](StringRef Text) {
uint64_t Bytes = 0;
for (size_t I = 0; I < Text.size(); ++I) {
++Bytes;
if (Text[I] == '!')
while (I + 1 < Text.size() && isDigit(Text[I + 1]))
++I;
}
return Bytes;
};
DenseMap<unsigned, uint64_t> RootBytes;
uint64_t Total = 0;
for (unsigned I = 0; I < Symbols.size(); ++I)
if (Symbols[I].Kind == SymbolKind::Function && Live[I])
RootBytes[Roots[I]] += Symbols[I].Header.size() + Symbols[I].Body.size();
// Each root counts in whole 4 KiB units, so that an edit that grows a root
// by less than that leaves every comparison, and every other root's
// partition, unchanged.
constexpr uint64_t Unit = 4096;
for (auto &Entry : RootBytes) {
Entry.second = (Entry.second + Unit - 1) / Unit * Unit;
Total += Entry.second;
}
RootBytes[Roots[I]] +=
TextBytes(Symbols[I].Header) + TextBytes(Symbols[I].Body);
std::vector<unsigned> Order;
for (auto &Entry : RootBytes)
Order.push_back(Entry.first);
llvm::sort(Order, [&](unsigned A, unsigned B) {
return Symbols[A].Name < Symbols[B].Name;
});
DenseSet<unsigned> EntryRoots;
for (unsigned I = 0; I < Symbols.size(); ++I)
if (Symbols[I].Entry && Live[I])
EntryRoots.insert(Roots[I]);
// A root of at least one unit counts in whole units, so that an edit that
// grows it by less than that leaves every comparison, and every other
// root's partition, unchanged.
constexpr uint64_t Unit = 4096;
uint64_t Total = 0;
for (unsigned Root : Order)
if (RootBytes[Root] >= Unit) {
RootBytes[Root] = (RootBytes[Root] + Unit - 1) / Unit * Unit;
Total += RootBytes[Root];
}
uint64_t Cap = Total * 5 / 4 / Count;
std::vector<uint64_t> Load(Count, 0);
// Partitions that take no further root.
std::vector<char> Closed(Count, 0);
DenseMap<unsigned, unsigned> Placed;
auto FirstOpen = [&](unsigned Home) {
for (unsigned K = 0; K < Count; ++K)
if (!Closed[(Home + K) % Count])
return (Home + K) % Count;
return Home;
};
// A root without room anywhere stays in the first open partition from its
// own.
auto Place = [&](unsigned Root) {
uint64_t Bytes = RootBytes[Root];
unsigned Home = stableHash(Symbols[Root].Name) % Count, Chosen = Home;
bool Found = false;
for (unsigned K = 0; K < Count && !Found; ++K) {
unsigned Home = stableHash(Symbols[Root].Name) % Count;
unsigned Chosen = FirstOpen(Home);
for (unsigned K = 0; K < Count; ++K) {
unsigned P = (Home + K) % Count;
if (!Closed[P] && Load[P] + Bytes <= Cap) {
Chosen = P;
Found = true;
}
}
for (unsigned K = 0; K < Count && !Found; ++K) {
unsigned P = (Home + K) % Count;
if (!Closed[P]) {
Chosen = P;
Found = true;
break;
}
}
Load[Chosen] += Bytes;
Placed[Root] = Chosen;
};
DenseSet<unsigned> EntryRoots;
for (unsigned I = 0; I < Symbols.size(); ++I)
if (Symbols[I].Entry && Live[I])
EntryRoots.insert(Roots[I]);
// Entry roots first, each closing its partition, then roots of more than
// half the cap, in name order, so that small roots fill around them; the
// others follow in name order.
// Entry roots first, each closing its partition; then roots of more than
// half the cap, in name order, so that the others fill around them; then
// the other roots of at least one unit, in name order.
for (unsigned Root : Order)
if (EntryRoots.contains(Root)) {
Place(Root);
Closed[Placed[Root]] = 1;
}
for (unsigned Root : Order)
if (!EntryRoots.contains(Root) && RootBytes[Root] * 2 > Cap)
if (!EntryRoots.contains(Root) && RootBytes[Root] >= Unit &&
RootBytes[Root] * 2 > Cap)
Place(Root);
for (unsigned Root : Order)
if (!EntryRoots.contains(Root) && RootBytes[Root] * 2 <= Cap)
if (!EntryRoots.contains(Root) && RootBytes[Root] >= Unit &&
RootBytes[Root] * 2 <= Cap)
Place(Root);
// Roots under one unit (runtime helpers, mostly) count in no load of the
// larger roots, so a helper that an edit adds or removes moves no larger
// root. Each goes to the first open partition from its name's whose load,
// the helpers placed there included, is below the cap, or else to its
// name's: the helpers fill the partitions with room, and once those are
// full they spread by name instead of gathering in one partition (which
// could then hold most of the module and have it optimized whole).
std::vector<uint64_t> SmallLoad(Count, 0);
for (unsigned Root : Order)
if (!EntryRoots.contains(Root) && RootBytes[Root] < Unit) {
unsigned Home = stableHash(Symbols[Root].Name) % Count;
unsigned Chosen = FirstOpen(Home);
for (unsigned K = 0; K < Count; ++K) {
unsigned P = (Home + K) % Count;
if (!Closed[P] && Load[P] + SmallLoad[P] < Cap) {
Chosen = P;
break;
}
}
SmallLoad[Chosen] += RootBytes[Root];
Placed[Root] = Chosen;
}
for (unsigned I = 0; I < Symbols.size(); ++I)
if (Symbols[I].Kind == SymbolKind::Function && Live[I])
Symbols[I].Partition = Placed[Roots[I]];
@@ -424,6 +424,16 @@ void annotateFromBlocks(Function &F, const FunctionBlocks &Record,
markFrequency(F, EntryCount, MaxCount, Summary);
}
void BlockRecorder::addNeverRan(const Function &F) {
std::string Text;
raw_string_ostream OS(Text);
OS << "f ";
writeEscaped(OS, F.getName());
OS << " 0 0\n";
std::lock_guard<std::mutex> Guard(Lock);
Records.emplace(F.getName().str(), std::move(Text));
}
void BlockRecorder::add(const Function &F) {
std::optional<Function::ProfileCount> Entry = F.getEntryCount();
if (!Entry || Entry->isSynthetic())
+2
View File
@@ -71,6 +71,8 @@ class BlockRecorder {
public:
// Records F's block counts, which the profile annotation just gave it.
void add(const llvm::Function &F);
// Records F, which the profile has no record of, as never run.
void addNeverRan(const llvm::Function &F);
// Writes the file through a private sibling; returns an error message on
// failure.
std::optional<std::string> write(llvm::StringRef Path);
+1
View File
@@ -16,6 +16,7 @@
"test:ts7": "node scripts/test-ts7.mjs",
"test:checker-identity": "node scripts/checker-identity.mjs",
"generate:ts7-ast": "node scripts/generate-ts7-ast-schema.mjs",
"generate:ir-codec": "node scripts/generate-ir-codec.mjs",
"generate:ir-fields": "node scripts/generate-ir-fields.mjs",
"test:fetch-conformance": "vitest run tests/harness/fetch-conformance.test.ts",
"test:test262": "tsx tests/test262/run.ts",
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@scriptc/cli-darwin-arm64",
"version": "0.4.5",
"version": "0.4.6",
"description": "Native scriptc compiler for darwin-arm64",
"license": "Apache-2.0",
"homepage": "https://scriptc.dev",
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@scriptc/cli-darwin-x64",
"version": "0.4.5",
"version": "0.4.6",
"description": "Native scriptc compiler for darwin-x64",
"license": "Apache-2.0",
"homepage": "https://scriptc.dev",
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@scriptc/cli-linux-arm64-gnu",
"version": "0.4.5",
"version": "0.4.6",
"description": "Native scriptc compiler for linux-arm64-gnu",
"license": "Apache-2.0",
"homepage": "https://scriptc.dev",
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@scriptc/cli-linux-arm64-musl",
"version": "0.4.5",
"version": "0.4.6",
"description": "Native scriptc compiler for linux-arm64-musl",
"license": "Apache-2.0",
"homepage": "https://scriptc.dev",
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@scriptc/cli-linux-x64-gnu",
"version": "0.4.5",
"version": "0.4.6",
"description": "Native scriptc compiler for linux-x64-gnu",
"license": "Apache-2.0",
"homepage": "https://scriptc.dev",
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@scriptc/cli-linux-x64-musl",
"version": "0.4.5",
"version": "0.4.6",
"description": "Native scriptc compiler for linux-x64-musl",
"license": "Apache-2.0",
"homepage": "https://scriptc.dev",
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@scriptc/cli-win32-x64-msvc",
"version": "0.4.5",
"version": "0.4.6",
"description": "Native scriptc compiler for win32-x64-msvc",
"license": "Apache-2.0",
"homepage": "https://scriptc.dev",
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "scriptc",
"version": "0.4.5",
"version": "0.4.6",
"description": "Compile ordinary TypeScript and JavaScript to small, fast native executables and WebAssembly modules that run without Node.js",
"license": "Apache-2.0",
"homepage": "https://scriptc.dev",
+5 -2
View File
@@ -4,6 +4,7 @@ import {
mkdtempSync,
mkdirSync,
readFileSync,
realpathSync,
renameSync,
rmSync,
statSync,
@@ -22,8 +23,10 @@ afterEach(() => {
});
function fixture() {
const root = mkdtempSync(
join(process.platform === "win32" ? tmpdir() : "/tmp", "scriptc-install-"),
// installNativeCli resolves the installation directory, and on macOS /tmp
// is a link to /private/tmp, so the fixture's paths start resolved.
const root = realpathSync(
mkdtempSync(join(process.platform === "win32" ? tmpdir() : "/tmp", "scriptc-install-")),
);
roots.push(root);
const directory = join(root, "installation");
+45 -5
View File
@@ -604,11 +604,29 @@ declare module "node:fs" {
): void;
/* fs.watch — the file-watching slice (scr_watch.c, kqueue EVFILT_VNODE):
* "rename"/"change" events on ONE path, watcher.close() to stop. The
* listener takes zero parameters or the eventType string (the filename
* parameter has no lowering); an open watcher keeps the loop alive. A
* path that cannot be opened THROWS Node's fs error synchronously. */
* listener takes (), (eventType) or (eventType, filename); an open
* watcher keeps the loop alive. A path that cannot be opened THROWS
* Node's fs error synchronously. */
export type WatchEventType = "rename" | "change";
/* (eventType, filename): filename is the changed entry's name (null when
* the platform does not provide one). */
export type WatchListener<T> = (event: WatchEventType, filename: T | null) => void;
export interface FSWatcher {
close(): void;
on(event: "change", listener: (eventType: string, filename: string | Buffer) => void): this;
on(event: "error", listener: (error: Error) => void): this;
on(event: "close", listener: () => void): this;
once(event: "change", listener: (eventType: string, filename: string | Buffer) => void): this;
once(event: "error", listener: (error: Error) => void): this;
once(event: "close", listener: () => void): this;
addListener(
event: "change",
listener: (eventType: string, filename: string | Buffer) => void,
): this;
addListener(event: "error", listener: (error: Error) => void): this;
addListener(event: "close", listener: () => void): this;
off(event: "change" | "error" | "close", listener: (...args: any[]) => void): this;
removeListener(event: "change" | "error" | "close", listener: (...args: any[]) => void): this;
}
/* The options-record stance (see http.RequestOptions): persistent: true,
* recursive: false, and encoding: "utf8" state the lowered behavior and
@@ -620,12 +638,34 @@ declare module "node:fs" {
encoding?: string;
[option: string]: unknown;
}
export function watch(path: string, listener?: (eventType: string) => void): FSWatcher;
export function watch(path: string, listener?: WatchListener<string>): FSWatcher;
export function watch(
path: string,
options: WatchOptions,
listener?: (eventType: string) => void,
listener?: WatchListener<string>,
): FSWatcher;
/* fs.watchFile — stat polling (scr_watch_file.c, libuv's uv_fs_poll
* shape): the listener runs with (current, previous) Stats when a poll
* sees a difference, the first poll only records the baseline, and a
* path that cannot be stat'd polls as all-zero Stats. One poller per
* resolved path is shared by every listener on it. interval defaults to
* 5007 ms, persistent defaults to true; bigint: true (BigIntStats) has
* no lowering. The call is a statement: Node's StatWatcher has no
* lowering. */
export interface StatWatcher {}
export function watchFile(
filename: string,
options: { interval?: number; persistent?: boolean; bigint?: false },
listener: (current: Stats, previous: Stats) => void,
): StatWatcher;
export function watchFile(
filename: string,
listener: (current: Stats, previous: Stats) => void,
): StatWatcher;
export function unwatchFile(
filename: string,
listener?: (current: Stats, previous: Stats) => void,
): void;
/* fs.promises IS the fs/promises module (Node's rule) — the namespace-
* import form `fs.promises.readFile(...)` lowers through the same
* fs/promises table as `import { readFile } from "node:fs/promises"`. */
+43 -13
View File
@@ -246,12 +246,13 @@ declare var process: {
/* "v" plus versions.node: the compatibility target's release. */
readonly version: string;
/* The build-configuration snapshot Node exposes from its gyp config.
* A scriptc binary has no V8 and no gyp, so `variables` answers the
* honest capability record (v8_enable_i18n_support: 0 — no ICU; asan
* mirrors the sanitizer build flag) and `target_defaults` is absent —
* probe code takes its documented fallbacks. Reads of undeclared
* variables go through the index signature (string | number | boolean
* | undefined) and answer undefined. */
* A scriptc binary has no V8 and no gyp, so `variables` answers what is
* true of a compiled binary: v8_enable_i18n_support is 1 like Node's
* (the read links ICU, which the binary then carries), while the gyp
* variables it cannot answer (icu_path, asan, …) read undefined through
* the index signature (string | number | boolean | undefined), and
* `target_defaults` is absent — probe code takes its documented
* fallbacks. */
readonly config: {
readonly variables: { readonly [name: string]: string | number | boolean | undefined };
readonly target_defaults?: { readonly default_configuration: string };
@@ -1115,9 +1116,9 @@ declare var ReadableStream: {
from(iterable: string): ReadableStream<string>;
readonly prototype: ReadableStream<unknown>;
};
/* The writable side of a TransformStream and its writer. The
* WritableStream constructor has no native lowering; writable streams come
* from `new TransformStream()`. */
/* The writable side of a TransformStream and its writer. A program that
* constructs a WritableStream, or a TransformStream with a transformer,
* builds its streams with the web stream classes (scr_webstreams.c). */
interface WritableStreamDefaultWriter<W = unknown> {
readonly closed: Promise<void>;
readonly desiredSize: number | null;
@@ -1133,14 +1134,25 @@ interface WritableStream<W = unknown> {
close(): Promise<void>;
getWriter(): WritableStreamDefaultWriter<W>;
}
/* The identity transform: chunks written to `writable` are read from
* `readable`, and a write completes when its chunk is read. */
/* Without a transformer, the identity transform: chunks written to
* `writable` are read from `readable`, and a write completes when its chunk
* is read. */
interface TransformStream<I = unknown, O = unknown> {
readonly readable: ReadableStream<O>;
readonly writable: WritableStream<I>;
}
interface Transformer<I = unknown, O = unknown> {
start?(controller: TransformStreamDefaultController<O>): unknown;
transform?(chunk: I, controller: TransformStreamDefaultController<O>): unknown;
flush?(controller: TransformStreamDefaultController<O>): unknown;
cancel?(reason?: unknown): unknown;
}
declare var TransformStream: {
new <I = unknown, O = I>(): TransformStream<I, O>;
new <I = unknown, O = I>(
transformer?: Transformer<I, O>,
writableStrategy?: QueuingStrategy<I>,
readableStrategy?: QueuingStrategy<O>,
): TransformStream<I, O>;
readonly prototype: TransformStream<unknown, unknown>;
};
/* The other web stream interfaces. JavaScript sources construct and read
@@ -1158,8 +1170,14 @@ interface WritableStreamDefaultController {
readonly signal: AbortSignal;
error(reason?: unknown): void;
}
interface UnderlyingSink<W = unknown> {
start?(controller: WritableStreamDefaultController): unknown;
write?(chunk: W, controller: WritableStreamDefaultController): unknown;
close?(): unknown;
abort?(reason?: unknown): unknown;
}
declare var WritableStream: {
new <W = unknown>(sink?: object, strategy?: QueuingStrategy<W>): WritableStream<W>;
new <W = unknown>(sink?: UnderlyingSink<W>, strategy?: QueuingStrategy<W>): WritableStream<W>;
readonly prototype: WritableStream<unknown>;
};
declare var WritableStreamDefaultWriter: {
@@ -1747,6 +1765,18 @@ declare module "tty" {
/* The bytes queued behind the last write: the compiled binary's fd
* 1/2 writes are synchronous, so it is always 0. */
readonly writableLength: number;
/* Node's default for the process streams (65536 for a pipe, a file and a
* TTY): a constant, because nothing is ever queued. */
readonly writableHighWaterMark: number;
/* Always false: a synchronous write never returns false, so no 'drain'
* is ever pending. */
readonly writableNeedDrain: boolean;
/* 'drain' and the other events by name. Node returns the stream. */
on(event: string, listener: (...args: any[]) => void): this;
once(event: string, listener: (...args: any[]) => void): this;
addListener(event: string, listener: (...args: any[]) => void): this;
off(event: string, listener: (...args: any[]) => void): this;
removeListener(event: string, listener: (...args: any[]) => void): this;
getWindowSize(): [number, number];
}
export function isatty(fd: number): boolean;
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@scriptc/compiler",
"version": "0.4.5",
"version": "0.4.6",
"description": "The scriptc compiler — TypeScript/JavaScript frontend, typed IR, LLVM backend",
"license": "Apache-2.0",
"homepage": "https://scriptc.dev",
@@ -20,9 +20,12 @@ export function executableLinkFeatures(mod: IrModule, largePages = false): Nativ
...(features.wasm ? { wasm: true } : {}),
...(features.warningReplace ? { warningReplace: true } : {}),
...(features.stdioExtended ? { stdioExtended: true } : {}),
...(features.streamWriteErrors ? { streamWriteErrors: true } : {}),
...(features.stdioWriteSlot ? { stdioWriteSlot: true } : {}),
...(features.nodeIsland ? { nodeIsland: true } : {}),
...(features.moduleNamespace ? { moduleNamespace: true } : {}),
...(features.moduleNamespaceDyn ? { moduleNamespaceDyn: true } : {}),
...(features.primitiveWrapper ? { primitiveWrapper: true } : {}),
zlib: features.zlib,
...(features.brotli ? { brotli: true } : {}),
assert: features.assert,
@@ -67,6 +70,7 @@ export function executableLinkFeatures(mod: IrModule, largePages = false): Nativ
dgramDyn: features.dgramDyn,
textCodecDyn: features.textCodecDyn,
watch: features.fsWatch,
watchFile: features.fsWatchFile,
foreignFfi: hasForeignFfiCallback(mod.ffiImports ?? []),
nodeTest: features.nodeTest,
tls: features.tls,
@@ -79,5 +83,6 @@ export function executableLinkFeatures(mod: IrModule, largePages = false): Nativ
x509: features.x509,
webcrypto: features.webcrypto,
perfLoop: features.perfLoop,
...(features.icu ? { intl: true } : {}),
};
}
@@ -56,6 +56,7 @@ export async function compilePackedLibrary(
dgramDyn: false,
textCodecDyn: false,
watch: false,
watchFile: false,
foreignFfi: false,
nodeTest: false,
tls: false,
+39 -2
View File
@@ -7,7 +7,7 @@
import { execFile } from "node:child_process";
import { createHash, randomUUID } from "node:crypto";
import { existsSync, realpathSync, statSync } from "node:fs";
import { mkdtemp, readFile, rename, rm, stat } from "node:fs/promises";
import { copyFile, mkdtemp, readFile, rename, rm, stat } from "node:fs/promises";
import { delimiter } from "node:path";
import { basename, dirname, isAbsolute, join, resolve } from "node:path";
import { promisify } from "node:util";
@@ -21,7 +21,11 @@ import {
toolchainEnvironmentFingerprint,
} from "./toolchain-environment.js";
import { type NativeArtifactDependency } from "./native/contracts.js";
import { RuntimePackError, stageRuntimePackArtifacts } from "./runtime-pack.js";
import {
RuntimePackError,
stageRuntimePackArtifacts,
type RuntimePackSelection,
} from "./runtime-pack.js";
import type { NativeLinkPlan } from "./link-plan.js";
import { inputsWithoutLld, lldLinkerArgs } from "./lld.js";
import { hugeTextLinkerArgs, rewriteHugeTextExecutable } from "./huge-text.js";
@@ -225,6 +229,38 @@ async function objectLinkerDependencyPaths(
.sort();
}
/** Install the selected runtime pack's RUN-time data files (ICU's
* icudt78l.dat) beside the executable that links them. The compiler's link
* step is the only moment the executable's directory is known, and a data
* file next to the binary is what the runtime looks for first.
*
* A file already present at the pinned size is left alone: the pack ships
* one data set per file name, and re-copying 31.6 MiB on every build would
* be pure cost. The copy is atomic (private temporary, then rename) so two
* concurrent builds of the same program never observe a partial file. */
export async function installRuntimeDataFiles(
selection: RuntimePackSelection,
outputPath: string,
): Promise<void> {
if (selection.dataFiles.length === 0) return;
const outputDir = dirname(resolve(outputPath));
for (const source of selection.dataFiles) {
const destination = join(outputDir, basename(source));
const sourceSize = (await stat(source)).size;
const existing = await stat(destination).catch(() => null);
if (existing?.isFile() && existing.size === sourceSize) continue;
const temporary = join(
outputDir,
`.${basename(destination)}.scriptc-${process.pid}-${randomUUID()}`,
);
await copyFile(source, temporary);
await rename(temporary, destination).catch(async () => {
await rm(destination, { force: true });
await rename(temporary, destination);
});
}
}
export async function linkNativeExecutable(
plan: NativeLinkPlan,
options: {
@@ -314,6 +350,7 @@ export async function linkNativeExecutable(
await rm(plan.outputPath, { force: true });
await rename(privateOut, plan.outputPath);
});
await installRuntimeDataFiles(plan.runtimePack, plan.outputPath);
if (plan.target.platform === "darwin" && !plan.darwinDebugSymbols) {
await rm(`${plan.outputPath}.dSYM`, { recursive: true, force: true });
}
@@ -1,7 +1,10 @@
import { expect, test } from "vitest";
import {
BOOL,
BYTES_U8,
DYN,
F64,
STRING,
VOID,
funcOf,
type IrExpr,
@@ -260,6 +263,59 @@ test("receiver elision admits only synchronous receiver-free work", () => {
expect(callbackIgnoresReceiver(fn)).toBe(false);
});
test("receiver elision admits library work that runs no user code", () => {
const str: IrExpr = { kind: "strLit", value: "a", type: STRING, loc };
const num = (value: number): IrExpr => ({ kind: "numLit", value, type: F64, loc });
const dyn = ref("d", DYN);
const ignores = (expr: IrExpr): boolean => {
const fn = fixture().functions[0]!;
fn.body.unshift({ kind: "exprStmt", expr, loc });
return callbackIgnoresReceiver(fn);
};
const lib = (fn: string, args: IrExpr[]): IrExpr =>
({ kind: "libCall", fn, args, type: F64, loc }) as IrExpr;
const intrinsic = (name: string, args: IrExpr[]): IrExpr =>
({ kind: "intrinsic", name, args, type: VOID, loc }) as IrExpr;
// String and number members over primitives; a callback operand may run.
expect(ignores(lib("string.lastIndexOfFrom", [str, str, num(0)]))).toBe(true);
expect(ignores(lib("num.sameValue", [num(0), num(1)]))).toBe(true);
expect(ignores(lib("string.lastIndexOf", [str, closure("predicate")]))).toBe(false);
expect(ignores(lib("perf.now", []))).toBe(true);
// `new Error(message, options)` with a primitive message and no options.
const message: IrExpr = { kind: "dynFrom", value: str, type: DYN, loc };
expect(ignores(lib("error.newOptions", [message, message]))).toBe(true);
expect(ignores(lib("error.newOptions", [message, dyn]))).toBe(false);
// The module binding check only throws; console output may inspect.
const initialized: IrExpr = { kind: "boolLit", value: true, type: BOOL, loc };
expect(ignores(intrinsic("module.tdzCheck", [initialized, str]))).toBe(true);
expect(ignores(intrinsic("console.log", [dyn]))).toBe(false);
// A typed array built from nothing or a typed source; a dynamic source
// may be an iterable.
const bytes = (source: IrExpr | null): IrExpr => ({
kind: "bytesNew",
source,
type: BYTES_U8,
loc,
});
expect(ignores(bytes(null))).toBe(true);
expect(ignores(bytes(dyn))).toBe(false);
});
test("receiver elision admits array for-of loops", () => {
const fn = fixture().functions[0]!;
const numbers = { kind: "array", elem: F64 } as const;
const one: IrExpr = { kind: "numLit", value: 1, type: F64, loc };
fn.locals.push({ id: "x", name: "x", type: F64, mutable: false });
fn.body.unshift({
kind: "forOf",
localId: "x",
iterable: { kind: "arrayLit", elems: [one], type: numbers, loc },
body: [],
loc,
});
expect(callbackIgnoresReceiver(fn)).toBe(true);
});
function callsOf(name: string, callee: string, extra: IrStmt[] = []): IrFunction {
return {
name,
+163 -121
View File
@@ -478,6 +478,10 @@ export function emitModuleParts(emitter: LlEmitter): string[] {
const usesEvents = runtimeFeatures.processEvents;
const usesChildProcess = runtimeFeatures.childProcess;
const usesFsWatch = runtimeFeatures.fsWatch;
// fs.watchFile programs register the poller unit's exit cleanup — the
// same one-line gating (native/executable.ts links scr_watch_file.c on
// the same predicate).
const usesFsWatchFile = runtimeFeatures.fsWatchFile;
// Programs that call into Node record the libnode pack and its Node
// version before %main (scr_node_island.c).
const usesNodeIsland = runtimeFeatures.nodeIsland;
@@ -541,6 +545,7 @@ export function emitModuleParts(emitter: LlEmitter): string[] {
if (workerStdioBox || workerStdinBox)
emitter.declare(`declare void @scr_worker_stdio_install(ptr, ptr, ptr)`);
if (usesFsWatch) emitter.declare(`declare void @scr_watch_install()`);
if (usesFsWatchFile) emitter.declare(`declare void @scr_watch_file_install()`);
if (usesNodeIsland) emitter.declare(`declare void @scr_node_island_install(ptr, ptr)`);
// Interned now: the C-string constants flush before main assembles.
const nodeIslandConfig = usesNodeIsland
@@ -1226,6 +1231,9 @@ export function emitModuleParts(emitter: LlEmitter): string[] {
// fs.watch programs fill the loop's watch hooks the same way —
// scr_watch.c links only when this line is emitted.
...(usesFsWatch ? [` call void @scr_watch_install()`] : []),
// fs.watchFile programs register the poller unit's atexit cleanup the
// same way — scr_watch_file.c links only when this line is emitted.
...(usesFsWatchFile ? [` call void @scr_watch_file_install()`] : []),
...(nodeIslandConfig
? [
` call void @scr_node_island_install(ptr ${nodeIslandConfig[0]}, ptr ${nodeIslandConfig[1]})`,
@@ -1377,25 +1385,147 @@ export function emitModuleParts(emitter: LlEmitter): string[] {
return out;
}
/** The out-of-line hashed probe of a number-key map (scr_map_find_f64):
* `keyLines` define the normalized key word %kb from the parameter, and
* `runtimeKey` is the double passed to the runtime lookup. A map with at
* most SCR_MAP_LINEAR_LIMIT entries keeps no bucket table: the absent-key
* filter is tested before its entries are scanned. A bucket table is
* probed without the filter: the filter keeps two of its 64 bits per key,
* so past a few dozen keys it rejects few absent keys, and the probe's
* first bucket read answers those. Compact tables tag each 24-bit entry
* index (+1) with the hash's top byte; wider tables take the runtime
* lookup. */
/** The `.loop` lookup helpers only read memory and always return, as
* declared: LLVM then merges two lookups of one key (a has() followed by
* get()) before it inlines them. */
const INLINED_LOOKUP_ATTRS = "alwaysinline memory(read) nounwind willreturn";
/** The number-key entry lookups (sc_map_entry_f64 and sc_map_entry_f64_int)
* and their hashed probes. With `loop` they are the `.loop` flavor, which
* lookups inside a loop call (emitNumberMapEntry): each of its functions is
* inlined, so a lookup that finds its key in a bucket table makes no call,
* and the loop keeps no argument moves or spills around one. */
function numberMapEntryHelpers(sz: string, loop: boolean): string[] {
const flavor = loop ? ".loop" : "";
const attrs = loop ? `${INLINED_LOOKUP_ATTRS} ${FN_ATTRS}` : FN_ATTRS;
let mapTmp = 0;
const mapIndex = exactIndexLines("%k", "%nd", sz, () => `%ix${mapTmp++}`);
return [
`define internal ptr @sc_map_entry_f64${flavor}(ptr %m, double %k) ${attrs} {`,
`entry:`,
` %ndp = getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 16`,
...(sz === "i64"
? [` %nd32 = load i32, ptr %ndp`, ` %nd = zext i32 %nd32 to i64`]
: [` %nd = load i32, ptr %ndp`]),
// A non-integral key inside the table is as absent as one outside it.
...mapIndex.lines.map((line) => ` ${line}`),
` br i1 ${mapIndex.ok}, label %slot, label %slow`,
`slot:`,
sz === "i64"
? ` %i = add i64 ${mapIndex.wide}, 0`
: ` %i = trunc i64 ${mapIndex.wide} to ${sz}`,
` %dp = getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 15`,
` %dense = load ptr, ptr %dp`,
` %sp = getelementptr inbounds i32, ptr %dense, ${sz} %i`,
` %s = load i32, ptr %sp`,
` %empty = icmp eq i32 %s, 0`,
` br i1 %empty, label %miss, label %hit`,
`hit:`,
` %e32 = sub i32 %s, 1`,
sz === "i32" ? ` %e = add i32 %e32, 0` : ` %e = zext i32 %e32 to ${sz}`,
` %ep = getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 11`,
` %entries = load ptr, ptr %ep`,
` %found = getelementptr inbounds %ScrMapNumEntry, ptr %entries, ${sz} %e`,
` ret ptr %found`,
`miss:`,
` ret ptr null`,
`slow:`,
` %hashed = call ptr @sc_map_entry_f64_hashed${flavor}(ptr %m, double %k)`,
` ret ptr %hashed`,
`}`,
``,
// The same lookup for a key the emitter knows is an integer (`%ki`,
// the double `%k`'s exact value): the direct index tests the integer
// itself, so the double is converted only by the hashed probe. A
// negative key is not below the index's size as an unsigned value.
`define internal ptr @sc_map_entry_f64_int${flavor}(ptr %m, i64 %ki, double %k) ${attrs} {`,
`entry:`,
` %ndp = getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 16`,
` %nd32 = load i32, ptr %ndp`,
` %nd = zext i32 %nd32 to i64`,
` %inside = icmp ult i64 %ki, %nd`,
` br i1 %inside, label %slot, label %slow`,
`slot:`,
sz === "i64" ? ` %i = add i64 %ki, 0` : ` %i = trunc i64 %ki to ${sz}`,
` %dp = getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 15`,
` %dense = load ptr, ptr %dp`,
` %sp = getelementptr inbounds i32, ptr %dense, ${sz} %i`,
` %s = load i32, ptr %sp`,
` %empty = icmp eq i32 %s, 0`,
` br i1 %empty, label %miss, label %hit`,
`hit:`,
` %e32 = sub i32 %s, 1`,
sz === "i32" ? ` %e = add i32 %e32, 0` : ` %e = zext i32 %e32 to ${sz}`,
` %ep = getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 11`,
` %entries = load ptr, ptr %ep`,
` %found = getelementptr inbounds %ScrMapNumEntry, ptr %entries, ${sz} %e`,
` ret ptr %found`,
`miss:`,
` ret ptr null`,
`slow:`,
` %hashed = call ptr @sc_map_entry_f64_hashed_int${flavor}(ptr %m, i64 %ki, double %k)`,
` ret ptr %hashed`,
`}`,
``,
// Hashing (scr_map_find_f64): the SameValueZero key word and its
// scr_key_mix hash. sc_map_entry_f64_int's probe takes the key's double
// with its integer view: its bits are the key word, never NaN, and +0
// for the integer 0. The caller converts the integer: cvtsi2sd writes
// only the low half of its register, so in a function entry, where
// LLVM cannot clear the register first, it waits for the register's
// last value, in a loop of lookups often a value the previous
// iteration loaded.
...hashedNumberProbe(
`sc_map_entry_f64_hashed${flavor}`,
loop,
"double %k",
sz,
[
` %nan = fcmp uno double %k, %k`,
` %zero = fcmp oeq double %k, 0.0`,
` %kz = select i1 %zero, double 0.0, double %k`,
` %kzb = bitcast double %kz to i64`,
` %kb = select i1 %nan, i64 9221120237041090560, i64 %kzb`,
],
"%k",
),
...hashedNumberProbe(
`sc_map_entry_f64_hashed_int${flavor}`,
loop,
"i64 %ki, double %k",
sz,
[
` %kzero = icmp eq i64 %ki, 0`,
` %kbits = bitcast double %k to i64`,
` %kb = select i1 %kzero, i64 0, i64 %kbits`,
],
"%k",
),
];
}
/** The hashed probe of a number-key map (scr_map_find_f64), out of line,
* or inlined with `loop`: `keyLines` define the normalized key word %kb
* from the parameter, and `runtimeKey` is the double passed to the runtime
* lookup. A map with at most SCR_MAP_LINEAR_LIMIT entries keeps no bucket
* table, and neither does a map with the direct index, whose lookups reach
* the probe only for a key outside the index, which is absent. In a small
* map the absent-key filter is tested before its entries are scanned.
* A bucket table is probed without the filter: the filter keeps two of its
* 64 bits per key, so past a few dozen keys it rejects few absent keys, and
* the probe's first bucket read answers those. Compact tables tag each
* 24-bit entry index (+1) with the hash's top byte; wider tables take the
* runtime lookup. */
function hashedNumberProbe(
name: string,
loop: boolean,
param: string,
sz: string,
keyLines: string[],
runtimeKey: string,
): string[] {
return [
`define internal ptr @${name}(ptr %m, ${param}) noinline ${FN_ATTRS} {`,
`define internal ptr @${name}(ptr %m, ${param}) ${loop ? INLINED_LOOKUP_ATTRS : "noinline"} ${FN_ATTRS} {`,
`entry:`,
...keyLines,
` %x1s = lshr i64 %kb, 30`,
@@ -1415,7 +1545,14 @@ function hashedNumberProbe(
` %eps = getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 11`,
` %ents = load ptr, ptr %eps`,
` %linear = icmp eq ${sz} %nb, 0`,
` br i1 %linear, label %filter, label %buckets`,
` br i1 %linear, label %unbucketed, label %buckets`,
// A map without a bucket table either indexes its keys directly, and
// then holds none outside the index, or scans its entries.
`unbucketed:`,
` %ndp = getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 16`,
` %nd = load i32, ptr %ndp`,
` %indexed = icmp ne i32 %nd, 0`,
` br i1 %indexed, label %miss, label %filter`,
`filter:`,
` %f1s = lshr i64 %h, 44`,
` %f1a = and i64 %f1s, 63`,
@@ -1582,117 +1719,22 @@ export function helperDefs(emitter: LlEmitter): string[] {
}
// Number-key Map/Set lookups probe the runtime's direct index inline:
// while ndense is nonzero every live key is an integer below it, so an
// in-range integral key reads its slot (entry index + 1, 0 = absent) and
// any other key is absent. Maps without the index (ndense 0) hash the
// key in sc_map_entry_f64_hashed, which scans the entries of a small map
// or probes a compact bucket table as scr_map_find_f64 does, and calls
// the runtime lookup for a wide table. It stays out of line so that the
// direct-index path stays small at every call site. Both only read
// memory, so LLVM merges lookups of one key when nothing is stored in
// between (a has() followed by get(), for example).
if (emitter.decls.has(NUMBER_MAP_ENTRY_DECL)) {
const sz = emitter.sizeType;
let mapTmp = 0;
const mapIndex = exactIndexLines("%k", "%nd", sz, () => `%ix${mapTmp++}`);
// in-range integral key reads its slot (entry index + 1, 0 = absent).
// Any other key goes to sc_map_entry_f64_hashed, which probes a compact
// bucket table as scr_map_find_f64 does, calls the runtime lookup for a
// wide table, and in a map without a bucket table answers absent when the
// map has the index (a map with buckets keeps no index, scr_map.h) and
// scans the entries otherwise. A lookup in a hashed map so tests the
// index once. The probe stays out of line so that the direct-index path
// stays small at every call site, except in the `.loop` flavor
// (numberMapEntryHelpers). Both only read memory, so LLVM merges lookups
// of one key when nothing is stored in between (a has() followed by
// get(), for example).
if (emitter.decls.has(NUMBER_MAP_ENTRY_DECL))
defs.push(
`define internal ptr @sc_map_entry_f64(ptr %m, double %k) ${FN_ATTRS} {`,
`entry:`,
` %ndp = getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 16`,
...(sz === "i64"
? [` %nd32 = load i32, ptr %ndp`, ` %nd = zext i32 %nd32 to i64`]
: [` %nd = load i32, ptr %ndp`]),
// A non-integral key inside the table is as absent as one outside it.
...mapIndex.lines.map((line) => ` ${line}`),
` br i1 ${mapIndex.ok}, label %slot, label %outside`,
`slot:`,
sz === "i64"
? ` %i = add i64 ${mapIndex.wide}, 0`
: ` %i = trunc i64 ${mapIndex.wide} to ${sz}`,
` %dp = getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 15`,
` %dense = load ptr, ptr %dp`,
` %sp = getelementptr inbounds i32, ptr %dense, ${sz} %i`,
` %s = load i32, ptr %sp`,
` %empty = icmp eq i32 %s, 0`,
` br i1 %empty, label %miss, label %hit`,
`hit:`,
` %e32 = sub i32 %s, 1`,
sz === "i32" ? ` %e = add i32 %e32, 0` : ` %e = zext i32 %e32 to ${sz}`,
` %ep = getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 11`,
` %entries = load ptr, ptr %ep`,
` %found = getelementptr inbounds %ScrMapNumEntry, ptr %entries, ${sz} %e`,
` ret ptr %found`,
`miss:`,
` ret ptr null`,
`outside:`,
` %dense.any = icmp ne ${sz} %nd, 0`,
` br i1 %dense.any, label %miss, label %slow`,
`slow:`,
` %hashed = call ptr @sc_map_entry_f64_hashed(ptr %m, double %k)`,
` ret ptr %hashed`,
`}`,
``,
// The same lookup for a key the emitter knows is an integer (`%ki`,
// the double `%k`'s exact value): the direct index tests the integer
// itself, so the double is converted only by the hashed probe. A
// negative key is not below the index's size as an unsigned value.
`define internal ptr @sc_map_entry_f64_int(ptr %m, i64 %ki, double %k) ${FN_ATTRS} {`,
`entry:`,
` %ndp = getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 16`,
` %nd32 = load i32, ptr %ndp`,
` %nd = zext i32 %nd32 to i64`,
` %inside = icmp ult i64 %ki, %nd`,
` br i1 %inside, label %slot, label %outside`,
`slot:`,
sz === "i64" ? ` %i = add i64 %ki, 0` : ` %i = trunc i64 %ki to ${sz}`,
` %dp = getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 15`,
` %dense = load ptr, ptr %dp`,
` %sp = getelementptr inbounds i32, ptr %dense, ${sz} %i`,
` %s = load i32, ptr %sp`,
` %empty = icmp eq i32 %s, 0`,
` br i1 %empty, label %miss, label %hit`,
`hit:`,
` %e32 = sub i32 %s, 1`,
sz === "i32" ? ` %e = add i32 %e32, 0` : ` %e = zext i32 %e32 to ${sz}`,
` %ep = getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 11`,
` %entries = load ptr, ptr %ep`,
` %found = getelementptr inbounds %ScrMapNumEntry, ptr %entries, ${sz} %e`,
` ret ptr %found`,
`miss:`,
` ret ptr null`,
`outside:`,
` %dense.any = icmp ne i64 %nd, 0`,
` br i1 %dense.any, label %miss, label %slow`,
`slow:`,
` %hashed = call ptr @sc_map_entry_f64_hashed_int(ptr %m, i64 %ki)`,
` ret ptr %hashed`,
`}`,
``,
// Hashing (scr_map_find_f64): the SameValueZero key word and its
// scr_key_mix hash. sc_map_entry_f64_int's probe converts the integer
// view to the key word, which is never NaN or -0, and so skips the
// normalization.
...hashedNumberProbe(
"sc_map_entry_f64_hashed",
"double %k",
sz,
[
` %nan = fcmp uno double %k, %k`,
` %zero = fcmp oeq double %k, 0.0`,
` %kz = select i1 %zero, double 0.0, double %k`,
` %kzb = bitcast double %kz to i64`,
` %kb = select i1 %nan, i64 9221120237041090560, i64 %kzb`,
],
"%k",
),
...hashedNumberProbe(
"sc_map_entry_f64_hashed_int",
"i64 %ki",
sz,
[` %kf = sitofp i64 %ki to double`, ` %kb = bitcast double %kf to i64`],
"%kf",
),
...numberMapEntryHelpers(emitter.sizeType, false),
...numberMapEntryHelpers(emitter.sizeType, true),
);
}
// Map/Set iteration steps read the entries array inline: an index is
// live when it is below the entry count and its entry's cached hash is
// nonzero (a tombstone's is zero), as scr_map_iter_live decides, or for
@@ -286,6 +286,8 @@ import {
cryptoIntThunkFor,
type BytesThunkValue,
fsRenameThunkFor,
watchNameThunkFor,
watchErrorThunkFor,
streamCbThunkFor,
zlibBytesThunkFor,
} from "./expr-stream-callbacks.js";
@@ -8161,6 +8163,14 @@ export class LlEmitter {
return fsRenameThunkFor(this, cbT);
}
watchNameThunkFor(cbT: IrType & { kind: "func" }): string {
return watchNameThunkFor(this, cbT);
}
watchErrorThunkFor(cbT: IrType & { kind: "func" }): string {
return watchErrorThunkFor(this, cbT);
}
cryptoBytesThunkFor(
cbT: IrType & { kind: "func" },
arrayBuffer = false,
@@ -1003,6 +1003,16 @@ export function emitCallExpr(
} else if (needsReceiver) B.line("call void @scr_dyn_this_pop()");
return result;
};
// The direct entry of a known target brackets the call when the target
// may observe the receiver. A signature adapter that only forwards the
// receiver to its original carries the original's RECEIVER_FREE bit
// (set where the adapter closure is built), so its entry tests the bit,
// like the entry through the closure's own pointer.
const invokeKnown = (target: string): string => {
const readers = host.receiverReaders;
if (readers.ignores(known!)) return invoke(target, false);
return invoke(target, true, args, readers.forwardsOnly(known!));
};
const lending =
lent && !dynamic
? known!.captures !== undefined
@@ -1025,7 +1035,7 @@ export function emitCallExpr(
(e.callee.kind === "closure" ||
(e.callee.kind === "varRef" && host.knownCallbacks.has(e.callee.localId)))
) {
t = invoke(lending ?? direct, !host.receiverReaders.ignores(known!));
t = invokeKnown(lending ?? direct);
if (quiet) pending = false;
} else {
const fnp = B.tmp(),
@@ -1072,7 +1082,7 @@ export function emitCallExpr(
B.line(`${matches} = icmp eq ptr ${fn}, ${direct}`);
B.condBr(matches, fast, fallback);
B.startBlock(fast);
const specialized = invoke(lending ?? direct, !host.receiverReaders.ignores(known!));
const specialized = invokeKnown(lending ?? direct);
// `fast` may have split for exception checks; join from the end.
const fastEnd = B.currentLabel();
B.br(done);
@@ -1,7 +1,7 @@
/* Focused LLVM expression emission extracted from emitter.ts. */
import { InternalCompilerError } from "../../errors.js";
import { type IrType, isRefCounted, typeKey } from "../../ir/ir.js";
import { FN_ATTRS, releaseSym, retainSym, traceArg } from "./shapes.js";
import { type IrType, isRefCounted, STRING, typeKey } from "../../ir/ir.js";
import { FN_ATTRS, releaseSym, retainSym, traceArg, vAdapters } from "./shapes.js";
import type { LlvmEmitterContext } from "./expr-context.js";
import { unionPeekLines, unionTagLines } from "./union-repr.js";
import { registerReplay } from "./emit-journal.js";
@@ -279,8 +279,6 @@ export function streamDoneFnFor(
host.declare(
`declare void @${entry}(ptr, ptr${twoSlot ? ", ptr, ptr" : valueData ? ", ptr" : ""})`,
);
host.declare(`declare ptr @scr_box_get_ref(ptr)`);
host.declare(`declare void @scr_stream_release_v(ptr)`);
host.declare(`declare ptr @scr_error_retain_v(ptr)`);
host.declare(`declare void @scr_union_release(ptr)`);
const params = [
@@ -295,7 +293,10 @@ export function streamDoneFnFor(
...(twoSlot ? [` %bslot = alloca ptr`, ` %sslot = alloca ptr`] : []),
` %capsp = getelementptr inbounds %ScrClosure, ptr %self, i64 1`,
` %bx = load ptr, ptr %capsp`,
` %s = call ptr @scr_box_get_ref(ptr %bx) ; the stream, +1`,
// the stream, borrowed: the box holds it, and the closure (alive for
// the call) holds the box
` %sp = getelementptr inbounds %ScrBox, ptr %bx, i64 0, i32 5`,
` %s = load ptr, ptr %sp`,
` store ptr null, ptr %eslot`,
...(twoSlot ? [` store ptr null, ptr %bslot`, ` store ptr null, ptr %sslot`] : []),
];
@@ -347,7 +348,6 @@ export function streamDoneFnFor(
} else {
d.push(` call void @${entry}(ptr %s, ptr %ev) ; moves err; borrows s`);
}
d.push(` call void @scr_stream_release_v(ptr %s)`);
if (errT !== undefined) d.push(` call void ${releaseSym(host.shapeHost, errT)}(ptr %e)`);
if (dataT !== undefined && isRefCounted(dataT))
d.push(` call void ${releaseSym(host.shapeHost, dataT)}(ptr %d)`);
@@ -868,18 +868,46 @@ export function streamCbThunkFor(
host.declare(`declare void @scr_dyn_release(ptr)`);
host.declare(`declare zeroext i1 @scr_exc_pending()`);
if (hasThis) host.declare(`declare void @scr_stream_release_v(ptr)`);
// An object-mode write without an encoding argument passes
// undefined: a pointer union whose unit arm is undefined holds it as
// the null pointer, so that case skips the conversion.
const pointer = p.kind === "union" ? host.nullableUnions.get(p.unionId) : null;
const undefinedIsNull =
pointer !== null &&
p.kind === "union" &&
host.unionsById.get(p.unionId)?.arms[pointer.unitTag]?.kind === "undefinedT";
if (undefinedIsNull) {
host.declare(`declare zeroext i1 @scr_stream_chunk_encoding_undefined(ptr)`);
d.push(
` br label %etchk${i}`,
`etchk${i}:`,
` %eu${i} = call zeroext i1 @scr_stream_chunk_encoding_undefined(ptr %s)`,
` br i1 %eu${i}, label %etok${i}, label %etconv${i}`,
`etconv${i}:`,
);
}
d.push(
` %ev${i} = call ptr @scr_stream_chunk_encoding_value(ptr %s)`,
` %et${i} = call ${ty === "i1" ? "zeroext i1" : ty} @${helper}(ptr %ev${i}, ptr null)`,
` call void @scr_dyn_release(ptr %ev${i})`,
` %etbad${i} = call zeroext i1 @scr_exc_pending()`,
` br i1 %etbad${i}, label %etfail${i}, label %etok${i}`,
` br i1 %etbad${i}, label %etfail${i}, label %${undefinedIsNull ? "etconvok" : "etok"}${i}`,
`etfail${i}:`,
...(hasThis ? [` call void @scr_stream_release_v(ptr %sr)`] : []),
` ret void`,
`etok${i}:`,
);
passed.push(`${ty} %et${i}`);
if (undefinedIsNull) {
d.push(
`etconvok${i}:`,
` br label %etok${i}`,
`etok${i}:`,
` %etv${i} = phi ptr [ null, %etchk${i} ], [ %et${i}, %etconvok${i} ]`,
);
passed.push(`${ty} %etv${i}`);
} else {
d.push(`etok${i}:`);
passed.push(`${ty} %et${i}`);
}
} else if (isErrPos) {
// destroy's error argument: `Error | null` — wrap type-directedly.
// The finished/pipeline callback ("e") may declare `Error | null |
@@ -966,6 +994,161 @@ registerReplay("streamDoneFnFor", (host, args) =>
registerReplay("fsRenameThunkFor", (host, args) =>
fsRenameThunkFor(host, args[0] as IrType & { kind: "func" }),
);
/** The fs.watch listener adapter for an (eventType, filename) listener, or
* for an (eventType) listener whose parameter is dynamic: the runtime hands
* the borrowed C strings, the thunk builds what the listener declared. The
* listener owns its +1 arguments (the universal convention
* scr_watch_thunk_event follows), so a string the thunk boxes into a dynamic
* value is released once the box holds it. */
export function watchNameThunkFor(
host: LlvmEmitterContext,
cbT: IrType & { kind: "func" },
): string {
const key = `watchname:${typeKey(cbT)}`;
let sym = host.resolveThunks.get(key);
if (sym) return sym;
sym = host.contentSymbol("sc_watchname_", key);
host.resolveThunks.set(key, sym);
host.journal?.allocated(sym, "watchNameThunkFor", [cbT]);
const S = host.sizeType;
host.declare(`declare ${S} @strlen(ptr)`);
host.declare(`declare ptr @scr_str_new(ptr, ${S})`);
host.declare(`declare void @scr_str_release(ptr)`);
const d: string[] = [
`define internal void @${sym}(ptr %cb, ptr %event, ptr %name) ${FN_ATTRS} { ; fs.watch listener ${typeKey(cbT)}`,
`entry:`,
` %evlen = call ${S} @strlen(ptr %event)`,
` %evstr = call ptr @scr_str_new(ptr %event, ${S} %evlen)`,
];
const named = cbT.params.length === 2;
if (named) {
d.push(
` %slot = alloca ptr`,
` %has = icmp ne ptr %name, null`,
` br i1 %has, label %yes, label %no`,
`yes:`,
);
const nameParam = cbT.params[1]!;
if (nameParam.kind === "dyn") {
host.declare(`declare ptr @scr_dyn_new_null()`);
host.declare(`declare ptr @scr_dyn_new_str(ptr)`);
d.push(
` %nlen = call ${S} @strlen(ptr %name)`,
` %nstr = call ptr @scr_str_new(ptr %name, ${S} %nlen)`,
` %ndyn = call ptr @scr_dyn_new_str(ptr %nstr)`,
` call void @scr_str_release(ptr %nstr)`,
` store ptr %ndyn, ptr %slot`,
` br label %go`,
`no:`,
` %dn = call ptr @scr_dyn_new_null()`,
` store ptr %dn, ptr %slot`,
);
} else {
if (nameParam.kind !== "union")
throw new InternalCompilerError("llvm emitter bug: fs.watch filename param is not a union");
const def = host.unionsById.get(nameParam.unionId);
const strTag = def ? def.arms.findIndex((a) => a.kind === "string") : -1;
const nullTag = def ? def.arms.findIndex((a) => a.kind === "nullT") : -1;
if (strTag < 0 || nullTag < 0)
throw new InternalCompilerError("llvm emitter bug: fs.watch filename union lacks its arms");
if (host.nullableUnions.has(nameParam.unionId)) {
// `string | null` erases: the owned payload IS the union's value.
d.push(
` %nlen = call ${S} @strlen(ptr %name)`,
` %nstr = call ptr @scr_str_new(ptr %name, ${S} %nlen)`,
` store ptr %nstr, ptr %slot`,
` br label %go`,
`no:`,
` store ptr null, ptr %slot`,
);
} else {
const rc = vAdapters(host.shapeHost, STRING);
host.declare(`declare ptr @scr_union_new_ref(i32, ptr, ptr, ptr, ptr)`);
d.push(
` %nlen = call ${S} @strlen(ptr %name)`,
` %nstr = call ptr @scr_str_new(ptr %name, ${S} %nlen)`,
` %nu = call ptr @scr_union_new_ref(i32 ${strTag}, ptr %nstr, ptr ${rc.retain}, ptr ${rc.release}, ptr ${traceArg(host.shapeHost, STRING)})`,
` store ptr %nu, ptr %slot`,
` br label %go`,
`no:`,
` store ptr ${host.unitInstanceRef(nameParam.unionId, nullTag)}, ptr %slot`,
);
}
}
d.push(` br label %go`, `go:`);
}
let ev: string;
if (cbT.params[0]?.kind === "dyn") {
// The box retains the string; the thunk's own reference goes.
host.declare(`declare ptr @scr_dyn_new_str(ptr)`);
d.push(
` %evd = call ptr @scr_dyn_new_str(ptr %evstr)`,
` call void @scr_str_release(ptr %evstr)`,
);
ev = "%evd";
} else {
ev = "%evstr";
}
d.push(
` %fnp = getelementptr inbounds %ScrClosure, ptr %cb, i64 0, i32 1`,
` %fn = load ptr, ptr %fnp`,
);
if (named)
d.push(` %arg = load ptr, ptr %slot`, ` call void %fn(ptr %cb, ptr ${ev}, ptr %arg)`);
else d.push(` call void %fn(ptr %cb, ptr ${ev})`);
d.push(` ret void`, `}`, ``);
host.resolveThunkDefs.push(...d);
host.journal?.built();
return sym;
}
/** The FSWatcher 'error' listener adapter: the runtime hands a borrowed
* ScrError, the listener receives the declared dyn value or an Error
* object it owns. A zero-parameter listener uses the runtime's
* scr_watch_err_thunk0 and never reaches here. */
export function watchErrorThunkFor(
host: LlvmEmitterContext,
cbT: IrType & { kind: "func" },
): string {
const key = `watcherr:${typeKey(cbT)}`;
let sym = host.resolveThunks.get(key);
if (sym) return sym;
sym = host.contentSymbol("sc_watcherr_", key);
host.resolveThunks.set(key, sym);
host.journal?.allocated(sym, "watchErrorThunkFor", [cbT]);
const param = cbT.params[0]!;
const d: string[] = [
`define internal void @${sym}(ptr %cb, ptr %err) ${FN_ATTRS} { ; FSWatcher error listener ${typeKey(cbT)}`,
`entry:`,
];
if (param.kind === "dyn") {
host.declare(`declare ptr @scr_dyn_from_error(ptr)`);
d.push(` %de = call ptr @scr_dyn_from_error(ptr %err)`);
} else if (param.kind === "object" && param.className === "%Error") {
host.declare(`declare ptr @scr_error_retain(ptr)`);
d.push(` %de = call ptr @scr_error_retain(ptr %err)`);
} else {
throw new InternalCompilerError("llvm emitter bug: FSWatcher error param is not an Error");
}
d.push(
` %fnp = getelementptr inbounds %ScrClosure, ptr %cb, i64 0, i32 1`,
` %fn = load ptr, ptr %fnp`,
` call void %fn(ptr %cb, ptr %de)`,
` ret void`,
`}`,
``,
);
host.resolveThunkDefs.push(...d);
host.journal?.built();
return sym;
}
registerReplay("watchNameThunkFor", (host, args) =>
watchNameThunkFor(host, args[0] as IrType & { kind: "func" }),
);
registerReplay("watchErrorThunkFor", (host, args) =>
watchErrorThunkFor(host, args[0] as IrType & { kind: "func" }),
);
registerReplay("cryptoIntThunkFor", (host, args) =>
cryptoIntThunkFor(host, args[0] as IrType & { kind: "func" }),
);
@@ -214,6 +214,9 @@ export function emitIoLibCall(host: LlvmEmitterContext, e: LibCallExpr): LlValue
export function emitGenericLibCall(host: LlvmEmitterContext, e: LibCallExpr): LlValue {
const B = host.B;
// The generic emitter registrations queue the maxListeners leak warning
// for the next tick like emitter.on (lib-process.ts).
if (e.fn === "emitter.onFlex" || e.fn === "emitter.onAny") host.usesTimers = true;
if (host.coroAwait && e.fn === "async.hop") {
host.emitCoroAwait(null);
return { name: "", type: e.type };
@@ -1185,6 +1188,10 @@ export function emitLibCall(host: LlvmEmitterContext, e: LibCallExpr): LlValue {
case "str":
case "regexp":
case "intl":
// ICU-backed Intl (scr_intl.c): ordinary libCalls whose C symbols the
// runtime provides — the same generic emission the engine-free intl
// forms use.
case "icu":
case "sym":
case "perf":
case "number":
@@ -1272,6 +1279,9 @@ export function emitLibCall(host: LlvmEmitterContext, e: LibCallExpr): LlValue {
return host.emitGenericLibCall(e);
case "nodeIsland":
return host.emitGenericLibCall(e);
// Primitive wrapper objects (scr_js_wrapper.c).
case "wrapper":
return host.emitGenericLibCall(e);
case "webstream":
// A stream's start, pull and cancel steps run as promise jobs, so
// a program with the classes runs its event loop.
@@ -1,7 +1,7 @@
/* Focused LLVM library-call emission extracted from emitter.ts. */
import { InternalCompilerError } from "../../errors.js";
import { undefinedArmTag } from "../../ir/analysis.js";
import { DYN, F64, STRING, type IrExpr } from "../../ir/ir.js";
import { DYN, F64, STRING, type IrExpr, type IrType } from "../../ir/ir.js";
import { mangleRecordNew } from "../mangle.js";
import { arrNewCall, traceArg, vAdapters } from "./shapes.js";
import type { LlvmEmitterContext, LibCallExpr, LlValue } from "./expr-context.js";
@@ -173,6 +173,22 @@ function emitObjEntryPairs(host: LlvmEmitterContext, e: LibCallExpr): LlValue {
return out;
}
/** The adapter of an fs.watch 'change' listener, by its declared shape: the
* runtime's for no parameters and for a string eventType, and one this
* module emits for a dynamic eventType or an (eventType, filename) listener.
* Only the runtime's adapters need a declaration. */
function watchChangeAdapter(host: LlvmEmitterContext, cbT: IrType & { kind: "func" }): string {
const runtime =
cbT.params.length === 0
? "scr_watch_thunk0"
: cbT.params.length === 1 && cbT.params[0]!.kind !== "dyn"
? "scr_watch_thunk_event"
: null;
if (runtime === null) return host.watchNameThunkFor(cbT);
host.declare(`declare void @${runtime}(ptr, ptr, ptr)`);
return runtime;
}
export function emitFilesystemLibCall(host: LlvmEmitterContext, e: LibCallExpr): LlValue {
const B = host.B;
const alwaysThrowSym = FS_ALWAYS_THROW_SYMS[e.fn];
@@ -448,17 +464,19 @@ export function emitFilesystemLibCall(host: LlvmEmitterContext, e: LibCallExpr):
// loop (usesTimers).
host.usesTimers = true;
const path = host.emitExpr(e.args[0]!);
host.declare(`declare ptr @scr_fs_watch(ptr, ptr, ptr)`);
host.declare(`declare ptr @scr_fs_watch(ptr, ptr, ptr, i1 zeroext)`);
const t = B.tmp();
B.line(`${t} = call ptr @scr_fs_watch(ptr ${path.name}, ptr null, ptr null)`);
B.line(`${t} = call ptr @scr_fs_watch(ptr ${path.name}, ptr null, ptr null, i1 false)`);
const out = host.own({ name: t, type: e.type });
host.emitPendingCheck();
return out;
}
if (e.fn === "fs.watchCb") {
// The callback MOVES into the watcher's registry; the adapter is
// runtime-provided per listener shape (zero-param, or the eventType
// string). May throw (ENOENT) — the standard pending check.
// runtime-provided per listener shape (zero-param, the eventType
// string, or the compiler's (eventType, filename) adapter). May
// throw (ENOENT, or the macOS directory-names refusal) — the
// standard pending check.
host.usesTimers = true;
const cbT = e.args[1]!.type;
if (cbT.kind !== "func")
@@ -466,15 +484,64 @@ export function emitFilesystemLibCall(host: LlvmEmitterContext, e: LibCallExpr):
const path = host.emitExpr(e.args[0]!);
const cb = host.emitExpr(e.args[1]!);
host.toHeap(cb);
const adapter = cbT.params.length === 0 ? "scr_watch_thunk0" : "scr_watch_thunk_event";
host.declare(`declare void @${adapter}(ptr, ptr)`);
host.declare(`declare ptr @scr_fs_watch(ptr, ptr, ptr)`);
const adapter = watchChangeAdapter(host, cbT);
host.declare(`declare ptr @scr_fs_watch(ptr, ptr, ptr, i1 zeroext)`);
const t = B.tmp();
B.line(`${t} = call ptr @scr_fs_watch(ptr ${path.name}, ptr ${cb.name}, ptr @${adapter})`);
B.line(
`${t} = call ptr @scr_fs_watch(ptr ${path.name}, ptr ${cb.name}, ptr @${adapter}, i1 ${cbT.params.length === 2 ? "true" : "false"})`,
);
const out = host.own({ name: t, type: e.type });
host.emitPendingCheck();
return out;
}
if (e.fn === "watcher.on") {
// (recv, event, flags, listener) — flags: 1 once, 2 prepend. The
// listener MOVES in; the adapter follows the event (0 change, 1
// error, 2 close — CLOSE takes no adapter, a plain zero-arg call).
// Registration can throw (the macOS directory-names refusal).
const flagsArg = e.args[2]!;
if (flagsArg.kind !== "numLit")
throw new InternalCompilerError("llvm emitter bug: watcher.on flags not a literal");
const flags = flagsArg.value;
const codeArg = e.args[1]!;
if (codeArg.kind !== "numLit")
throw new InternalCompilerError("llvm emitter bug: watcher.on event not a literal");
const code = codeArg.value;
const cbT = e.args[3]!.type;
if (cbT.kind !== "func")
throw new InternalCompilerError("llvm emitter bug: watcher.on listener not a func");
const w = host.emitExpr(e.args[0]!);
const event = host.emitExpr(e.args[1]!);
const cb = host.emitExpr(e.args[3]!);
host.toHeap(cb);
let adapter = "null";
let wantsName = false;
if (code === 0) {
adapter = watchChangeAdapter(host, cbT);
wantsName = cbT.params.length === 2;
} else if (code === 1) {
adapter = cbT.params.length === 0 ? "scr_watch_err_thunk0" : host.watchErrorThunkFor(cbT);
if (cbT.params.length === 0) host.declare(`declare void @scr_watch_err_thunk0(ptr, ptr)`);
}
host.declare(
`declare void @scr_watcher_on(ptr, double, ptr, ptr, i1 zeroext, i1 zeroext, i1 zeroext)`,
);
B.line(
`call void @scr_watcher_on(ptr ${w.name}, double ${event.name}, ptr ${cb.name}, ptr ${adapter === "null" ? "null" : `@${adapter}`}, i1 ${flags & 1 ? "true" : "false"}, i1 ${flags & 2 ? "true" : "false"}, i1 ${wantsName ? "true" : "false"})`,
);
host.emitPendingCheck();
return { name: "", type: e.type };
}
if (e.fn === "watcher.off") {
// Removal is by pointer identity: the listener stays borrowed
// (never heap-moved), exactly like process.offExit's.
const w = host.emitExpr(e.args[0]!);
const event = host.emitExpr(e.args[1]!);
const cb = host.emitExpr(e.args[2]!);
host.declare(`declare void @scr_watcher_off(ptr, double, ptr)`);
B.line(`call void @scr_watcher_off(ptr ${w.name}, double ${event.name}, ptr ${cb.name})`);
return { name: "", type: e.type };
}
if (e.fn === "watcher.close") {
// Idempotent; never throws.
const w = host.emitExpr(e.args[0]!);
@@ -482,6 +549,45 @@ export function emitFilesystemLibCall(host: LlvmEmitterContext, e: LibCallExpr):
B.line(`call void @scr_watcher_close(ptr ${w.name})`);
return { name: "", type: e.type };
}
if (e.fn === "fs.watchFile") {
// (path, interval, persistent, listener) — the listener MOVES into
// the poller's registry and the adapter is runtime-provided per
// listener shape (zero, one or two Stats parameters). The poller
// re-arms a timer every interval, so this is a timer program. May
// throw (a null byte in the path, or an interval that is not a
// uint32) — the standard pending check.
host.usesTimers = true;
const cbT = e.args[3]!.type;
if (cbT.kind !== "func")
throw new InternalCompilerError("llvm emitter bug: fs.watchFile listener not a func");
const path = host.emitExpr(e.args[0]!);
const interval = host.emitExpr(e.args[1]!);
const persistent = host.emitExpr(e.args[2]!);
const cb = host.emitExpr(e.args[3]!);
host.toHeap(cb);
const adapter = `scr_watch_file_thunk${cbT.params.length}`;
host.declare(`declare void @${adapter}(ptr, ptr, ptr)`);
host.declare(`declare void @scr_fs_watch_file(ptr, double, i1 zeroext, ptr, ptr)`);
B.line(
`call void @scr_fs_watch_file(ptr ${path.name}, double ${interval.name}, i1 ${persistent.name}, ptr ${cb.name}, ptr @${adapter})`,
);
host.emitPendingCheck();
return { name: "", type: e.type };
}
if (e.fn === "fs.unwatchFile") {
// Removal is by closure identity: the listener stays borrowed
// (never heap-moved), exactly like watcher.off's; an absent one is
// the NULL that removes every listener on the path. A null byte in
// the path throws.
const path = host.emitExpr(e.args[0]!);
const cb = e.args[1] === undefined ? null : host.emitExpr(e.args[1]!);
host.declare(`declare void @scr_fs_unwatch_file(ptr, ptr)`);
B.line(
`call void @scr_fs_unwatch_file(ptr ${path.name}, ptr ${cb === null ? "null" : cb.name})`,
);
host.emitPendingCheck();
return { name: "", type: e.type };
}
if (e.fn === "fs.globTypesSync" || e.fn === "fsp.globTypes") {
// Dirent rows from one glob snapshot (scr_glob.c): mode 1 is the
// globSync form, 3 the fs/promises form with resolved parent paths.
@@ -867,6 +867,10 @@ export function emitErrorsEventsLibCall(host: LlvmEmitterContext, e: LibCallExpr
const cbT = e.args[2]!.type;
if (cbT.kind !== "func")
throw new InternalCompilerError("llvm emitter bug: emitter.on listener not a func");
// A registration past the emitter's maxListeners queues the leak
// warning for the next tick (scr_events_emitter.c), which the loop
// delivers before the process exits.
host.usesTimers = true;
const args = e.args.map((a) => host.emitExpr(a));
const { fn: adapterFn, shim } = host.emitterFixedAdapter(cbT);
// The wrapper's capture box owns its OWN +1 of the listener; the
@@ -897,6 +901,7 @@ export function emitErrorsEventsLibCall(host: LlvmEmitterContext, e: LibCallExpr
const adT = e.args[3]!.type;
if (adT.kind !== "func")
throw new InternalCompilerError("llvm emitter bug: emitter.onDyn adapter not a func");
host.usesTimers = true; // the leak warning's tick (emitter.on)
const args = e.args.map((a) => host.emitExpr(a));
const { fn: adapterFn, shim } = host.emitterFixedAdapter(adT);
host.toHeap(args[3]!); // the frame's +1 moves into the wrapper's box
@@ -1046,6 +1051,7 @@ export function emitErrorsEventsLibCall(host: LlvmEmitterContext, e: LibCallExpr
// emitter.on/onDyn, but the DATA adapter (the three-slot payload ABI
// — scr_stream_emit_data) behind the arity-3 fixed shim.
const isDyn = e.fn === "emitter.onDataDyn";
host.usesTimers = true; // the leak warning's tick (emitter.on)
const cbT = e.args[isDyn ? 3 : 2]!.type;
if (cbT.kind !== "func")
throw new InternalCompilerError(`llvm emitter bug: ${e.fn} listener not a func`);
@@ -67,6 +67,11 @@ export const LIB_FN_SYMS: Record<string, string> = {
"http.reqH2StreamOrThrow": "scr_http_req_h2_stream_or_throw",
"intl.segmenterNew": "scr_intl_segmenter_new",
"intl.segmenterRefusing": "scr_intl_segmenter_refusing",
"icu.segmenterWord": "scr_intl_segmenter_word_new",
"icu.numFormatDefault": "scr_intl_num_format_default",
"icu.i18nSupport": "scr_intl_i18n_support",
"icu.dateLocaleString": "scr_intl_date_locale_string",
"icu.dateLocaleTimeString": "scr_intl_date_locale_time_string",
"weakMap.is": "scr_weak_map_is",
"weakSet.is": "scr_weak_set_is",
"weakMap.new": "scr_weak_map_new",
@@ -261,6 +266,8 @@ export const LIB_FN_SYMS: Record<string, string> = {
"os.userName": "scr_os_user_name",
"os.userShell": "scr_os_user_shell",
"os.userHomedir": "scr_os_user_homedir",
"os.userUid": "scr_os_user_uid",
"os.userGid": "scr_os_user_gid",
"os.tmpdir": "scr_os_tmpdir",
"process.argv": "scr_process_argv",
"process.execArgv": "scr_process_exec_argv",
@@ -304,9 +311,19 @@ export const LIB_FN_SYMS: Record<string, string> = {
"navigator.languages": "scr_navigator_languages",
"navigator.platform": "scr_navigator_platform",
"process.builtinModule": "scr_process_builtin_module",
"process.builtinObject": "scr_process_builtin_object",
"process.builtinUnsupported": "scr_process_builtin_unsupported",
"module.namespaceValue": "scr_module_namespace_value",
"module.namespaceDyn": "scr_module_namespace_dyn_value",
"wrapper.new": "scr_js_wrapper_new",
"wrapper.toObject": "scr_js_wrapper_to_object",
"wrapper.instanceOf": "scr_js_wrapper_instance_of",
"wrapper.objectTag": "scr_js_wrapper_object_tag",
"wrapper.typeIs": "scr_js_wrapper_type_is",
"wrapper.guard": "scr_js_wrapper_guard",
"wrapper.guardList": "scr_js_wrapper_guard_list",
"wrapper.isStringMethod": "scr_js_wrapper_is_string_method",
"wrapper.thisString": "scr_js_wrapper_this_string",
"fs.callbackValue": "scr_fs_callback_value",
"fs.callbackCall": "scr_fs_callback_call",
"process.hrtimeValue": "scr_process_hrtime_value",
@@ -379,9 +396,14 @@ export const LIB_FN_SYMS: Record<string, string> = {
"process.envPairs": "scr_env_pairs",
"process.stdoutWrite": "scr_process_stdout_write",
"process.stdio": "scr_process_stdio",
"process.stdioSlot": "scr_process_stdio",
"process.stdioWriteDyn": "scr_process_stdio_write_dyn",
"process.stderrWrite": "scr_process_stderr_write",
"process.isTTY": "scr_process_is_tty",
"process.isTTYValue": "scr_process_is_tty_value",
// A promise that never settles (events.once of an event a synchronous
// stream never emits); a fresh pending promise is its owned result.
"events.pendingForever": "scr_promise_new",
"date.now": "scr_date_now",
"date.newNow": "scr_date_now",
"date.newMs": "scr_date_new_ms",
@@ -441,6 +463,12 @@ export const LIB_FN_SYMS: Record<string, string> = {
"fsp.globError": "scr_fsp_glob_error",
"fs.realpathSync": "scr_fs_realpath",
"fs.realpathValue": "scr_fs_realpath_value",
"dyn.validateString": "scr_value_string",
"dyn.validateStrings": "scr_value_strings",
"path.resolveValue": "scr_path_resolve_value",
"path.parseValue": "scr_path_parse_value",
"path.formatValue": "scr_path_format_value",
"path.toNamespacedValue": "scr_path_to_namespaced_value",
"fs.realpathNativeSync": "scr_fs_realpath_promise",
"fs.unlinkSync": "scr_fs_unlink",
"fs.chmodSync": "scr_fs_chmod",
@@ -1896,6 +1924,11 @@ export const USES_TIMERS_LIB_FNS = new Set<string>([
"worker.receiveMessageOnPort",
"worker.broadcastChannel",
"process.stdio",
// The typed write slot and the run-time-typed chunk write keep the loop
// live for the write callbacks they queue: both reach the same stream
// object as process.stdio and call back from a nextTick.
"process.stdioSlot",
"process.stdioWriteDyn",
"fetch.start",
"fetch.input",
"fetch.function",
@@ -1981,6 +2014,7 @@ export const USES_TIMERS_LIB_FNS = new Set<string>([
"writable.writeU",
"writable.writeDyn",
"writable.writeEnc",
"writable.writeEcb",
"writable.end",
"writable.uncork",
"stream.destroy",
@@ -455,6 +455,44 @@ export function emitStreamLibCall(host: LlvmEmitterContext, e: LibCallExpr): LlV
host.emitPendingCheck();
return e.type.kind === "void" ? { name: "", type: e.type } : { name: t, type: e.type };
}
if (e.fn === "writable.writeEcb") {
// (recv, mode, chunk, encoding, cb): mode 0 bytes, 1 string, 2 dyn. The
// callback MOVES into an error-first wrapper (scr_stream_ecb_wrap); the
// adapter turns the runtime's ScrError (NULL = success) into the
// callback's Error | null argument, as fs.rename callbacks do.
const modeArg = e.args[1]!;
if (modeArg.kind !== "numLit")
throw new InternalCompilerError("llvm emitter bug: writable.writeEcb mode not a literal");
const cbT = e.args[4]!.type;
if (cbT.kind !== "func")
throw new InternalCompilerError("llvm emitter bug: writable.writeEcb callback not a func");
const args = e.args.map((a) => host.emitExpr(a));
host.toHeap(args[4]!);
const adapter = host.fsRenameThunkFor(cbT);
host.declare(`declare ptr @scr_stream_ecb_wrap(ptr, ptr)`);
const wrapped = B.tmp();
B.line(`${wrapped} = call ptr @scr_stream_ecb_wrap(ptr ${args[4]!.name}, ptr @${adapter})`);
const entry =
modeArg.value === 0
? "scr_stream_write"
: modeArg.value === 1
? "scr_stream_write_str"
: null;
const t = B.tmp();
if (entry !== null) {
host.declare(`declare zeroext i1 @${entry}(ptr, ptr, ptr)`);
B.line(
`${t} = call zeroext i1 @${entry}(ptr ${args[0]!.name}, ptr ${args[2]!.name}, ptr ${wrapped})`,
);
} else {
host.declare(`declare zeroext i1 @scr_stream_write_enc(ptr, ptr, ptr, ptr)`);
B.line(
`${t} = call zeroext i1 @scr_stream_write_enc(ptr ${args[0]!.name}, ptr ${args[2]!.name}, ptr ${args[3]!.name}, ptr ${wrapped})`,
);
}
host.emitPendingCheck();
return e.type.kind === "void" ? { name: "", type: e.type } : { name: t, type: e.type };
}
if (e.fn === "writable.end") {
// (recv, flags[, chunk][, encoding][, cb]) — flags: 1 bytes chunk, 2
// string chunk, 8 dyn chunk (write first, Node's end(chunk)
@@ -3,11 +3,19 @@
* local arrays they rely on. */
import { findUnaliasedLocalArrays } from "../../ir/array-length-stability.js";
import type { IrFunction, IrStmt } from "../../ir/ir.js";
import { everyStmtList } from "../../ir/traverse.js";
import { findInitializedByteLoopBindings } from "./byte-windows.js";
import type { DenseArrayLoop } from "./dense-array-loops.js";
import type { LlEmitter } from "./emitter.js";
import { defineModuleState } from "./emitter-state.js";
/** The most IR nodes a function may have for its in-loop number-key
* lookups to inline the hashed probe (emitNumberMapEntry). Each inlined
* probe adds code to its function, and LLVM's passes over a large function
* take longer for it; a small function's loop of lookups, such as a k-mer
* count, gains the most from it. */
const MAX_INLINED_LOOKUP_NODES = 200;
/** Sets the loop state for `fn` at the start of its emission; `numericFn`
* is `fn` with its initializer bindings (emitFunction). */
export function beginFunctionLoops(
@@ -32,6 +40,15 @@ export function beginFunctionLoops(
loops.byteWindowsEnabled && numericFn.locals.some((l) => l.type.kind === "bytes")
? findInitializedByteLoopBindings(fn)
: new Map();
loops.inlinedLookups = fewNodes(fn.body, MAX_INLINED_LOOKUP_NODES);
}
/** Whether `body` has at most `max` statements and expressions. */
function fewNodes(body: IrStmt[], max: number): boolean {
let count = 0;
const visit = (): boolean => ++count <= max;
everyStmtList(body, { expr: visit, stmt: visit });
return count <= max;
}
/** The state LOOPS holds on each emitter. */
@@ -58,6 +75,9 @@ export interface LoopState {
/** Counted loops whose integer-form safety test an enclosing version
* guard already passed. */
readonly provenCountedGuards: Set<IrStmt>;
/** Number-key lookups in this function's loops inline the hashed probe:
* the function has at most MAX_INLINED_LOOKUP_NODES nodes. */
inlinedLookups: boolean;
}
export const LOOPS = defineModuleState<LoopState>(() => ({
@@ -73,4 +93,5 @@ export const LOOPS = defineModuleState<LoopState>(() => ({
bytesLoopEntries: null,
bytesLoopFunction: null,
provenCountedGuards: new Set<IrStmt>(),
inlinedLookups: false,
}));
@@ -117,8 +117,9 @@ test("string keys and reference values are retained inline, boolean slots compar
expect(text).toContain("call ptr @scr_map_iter_val_ref(");
});
/** work(map, k): map.has(k) on a number-keyed map. */
function lookupFixture(key: IrExpr = ref("k", F64)): IrModule {
/** work(map, k): map.has(k) on a number-keyed map, or with `loop`, the
* condition of `while (map.has(k)) {}`, after `filler` statements `k;`. */
function lookupFixture(key: IrExpr = ref("k", F64), loop = false, filler = 0): IrModule {
const map: IrType = { kind: "map", key: F64, value: F64 };
const params = [
{ localId: "map", name: "map", type: map },
@@ -138,7 +139,17 @@ function lookupFixture(key: IrExpr = ref("k", F64)): IrModule {
returnType: BOOL,
loc,
locals: params.map((p) => ({ id: p.localId, name: p.name, type: p.type, mutable: true })),
body: [{ kind: "return", value: has, loc }],
body: loop
? [
...Array.from({ length: filler }, (): IrStmt => ({
kind: "exprStmt",
expr: ref("k", F64),
loc,
})),
{ kind: "while", cond: has, body: [], loc },
{ kind: "return", value: { kind: "boolLit", value: false, type: BOOL, loc }, loc },
]
: [{ kind: "return", value: has, loc }],
};
return {
irVersion: 17,
@@ -159,6 +170,10 @@ test("hashed number-key lookups probe inline and call the runtime only for wide
module,
)![0];
expect(helper).toContain("call ptr @sc_map_entry_f64_hashed(ptr %m, double %k)");
// A key outside the direct index goes to the probe, whose small-map
// branch tells a map with the index (field 16) from one without it.
expect(helper).toMatch(/br i1 %ix\d+, label %slot, label %slow\n/);
expect(helper).not.toContain("dense.any");
const hashed =
/^define internal ptr @sc_map_entry_f64_hashed\(ptr %m, double %k\) noinline [^]*?^}/m.exec(
module,
@@ -171,7 +186,10 @@ test("hashed number-key lookups probe inline and call the runtime only for wide
// Filter (field 10) before the small-map scan of the entry count (field
// 8) only, and compact buckets (fields 13 and 14) tagged with the hash's
// top byte.
expect(hashed).toContain("br i1 %linear, label %filter, label %buckets");
expect(hashed).toContain("br i1 %linear, label %unbucketed, label %buckets");
expect(hashed).toMatch(
/^unbucketed:\n[^]*i32 0, i32 16\n[^]*br i1 %indexed, label %miss, label %filter\n/m,
);
expect(hashed).toContain("getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 10");
expect(hashed).toContain(`load i32, ptr %nep`);
expect(hashed).toContain(`load i32, ptr %nbp`);
@@ -194,22 +212,27 @@ test("a key with an integer view tests the direct index on the integer", () => {
integral.module,
)![0];
// An unsigned compare also rejects negative keys; the hashed probe takes
// the integer.
// the integer and its double.
expect(helper).toContain("icmp ult i64 %ki, %nd");
expect(helper).toContain(`getelementptr inbounds i32, ptr %dense, ${sz} %i`);
expect(helper).toContain("call ptr @sc_map_entry_f64_hashed_int(ptr %m, i64 %ki)");
expect(helper).toContain("call ptr @sc_map_entry_f64_hashed_int(ptr %m, i64 %ki, double %k)");
expect(helper).toContain("br i1 %inside, label %slot, label %slow");
expect(helper).not.toContain("fptosi");
// Its key word is the integer's double, which needs no NaN or -0
// normalization; the runtime lookup gets that double.
// Its key word is the bits of the integer's double, which needs no NaN
// normalization and is +0 for the integer 0; the runtime lookup gets
// that double. The probe converts nothing itself (cvtsi2sd would wait
// for its register's last value).
const hashed =
/^define internal ptr @sc_map_entry_f64_hashed_int\(ptr %m, i64 %ki\) noinline [^]*?^}/m.exec(
/^define internal ptr @sc_map_entry_f64_hashed_int\(ptr %m, i64 %ki, double %k\) noinline [^]*?^}/m.exec(
integral.module,
)![0];
expect(hashed).toContain("sitofp i64 %ki to double");
expect(hashed).toContain("bitcast double %k to i64");
expect(hashed).toContain("select i1 %kzero, i64 0, i64 %kbits");
expect(hashed).not.toContain("sitofp");
expect(hashed).not.toContain("fcmp");
expect(hashed).toContain("mul i64 %x1, -4658895280553007687");
expect(hashed).toContain(
"call ptr @scr_map_entry_f64(ptr %m, double %kf) memory(read) nounwind willreturn",
"call ptr @scr_map_entry_f64(ptr %m, double %k) memory(read) nounwind willreturn",
);
// A fractional key and a key without a proof keep the double lookup.
for (const key of [{ kind: "numLit", value: 0.5, type: F64, loc } as IrExpr, ref("k", F64)]) {
@@ -220,6 +243,36 @@ test("a key with an integer view tests the direct index on the integer", () => {
}
});
test("a lookup inside a loop calls the flavor that inlines the hashed probe", () => {
for (const key of [ref("k", F64), { kind: "numLit", value: 12, type: F64, loc } as IrExpr]) {
const { work, module } = emitted(lookupFixture(key, true), 64);
const name = key.kind === "numLit" ? "sc_map_entry_f64_int" : "sc_map_entry_f64";
expect(work).toContain(`call ptr @${name}.loop(`);
expect(work).not.toContain(`call ptr @${name}(`);
// They only read memory and always return, so a has() and get() of one
// key merge before they are inlined.
const helper = new RegExp(
`^define internal ptr @${name}\\.loop\\([^)]*\\) alwaysinline memory\\(read\\) nounwind willreturn [^]*?^}`,
"m",
).exec(module)![0];
const hashed =
key.kind === "numLit" ? "sc_map_entry_f64_hashed_int" : "sc_map_entry_f64_hashed";
expect(helper).toContain(`call ptr @${hashed}.loop(`);
expect(module).toMatch(
new RegExp(
`^define internal ptr @${hashed}\\.loop\\([^)]*\\) alwaysinline memory\\(read\\) nounwind willreturn `,
"m",
),
);
// A lookup outside a loop keeps the out-of-line probe, and so does a
// lookup in a loop of a function of more than 200 nodes.
expect(emitted(lookupFixture(key), 64).work).toContain(`call ptr @${name}(`);
const large = emitted(lookupFixture(key, true, 100), 64).work;
expect(large).toContain(`call ptr @${name}(`);
expect(large).not.toContain(".loop(");
}
});
test("a key view converted in a branch is not reused by a later lookup of the same constant", () => {
// work(map, k, taken): if (taken) return map.has((k = 2 ** 40)); return map.has(2 ** 40).
const wide = 2 ** 40;
@@ -32,6 +32,7 @@
import type { IrExpr, IrType } from "../../ir/ir.js";
import type { LlValue, LlvmEmitterContext } from "./expr-context.js";
import { exactInteger, widenInteger } from "./integer-values.js";
import { LOOPS } from "./loop-state.js";
import { mapKeyAccess } from "./shapes.js";
export const NUMBER_MAP_ENTRY_DECL = "declare ptr @scr_map_entry_f64(ptr, double)";
@@ -55,19 +56,22 @@ export function noteNumberMapKey(host: LlvmEmitterContext, key: LlValue, expr: I
/** The entry for `key`, or null when absent. Valid until the next mutation.
* A key with an integer view (noteNumberMapKey) tests the direct index on
* that integer instead of converting the double back. */
* that integer instead of converting the double back. A lookup inside a
* loop of a small function (LOOPS.inlinedLookups) calls the helpers' `.loop`
* flavor, which inlines the hashed probe. */
export function emitNumberMapEntry(host: LlvmEmitterContext, map: string, key: string): string {
host.declare(NUMBER_MAP_ENTRY_DECL);
const entry = host.B.tmp();
const flavor = host.loopDepth > 0 && LOOPS.of(host).inlinedLookups ? ".loop" : "";
const integer = host.numberMapKeyIntegers.get(key);
if (integer) {
const wide = widenInteger(host, integer);
host.B.line(
`${entry} = call ptr @sc_map_entry_f64_int(ptr ${map}, i64 ${wide}, double ${key})`,
`${entry} = call ptr @sc_map_entry_f64_int${flavor}(ptr ${map}, i64 ${wide}, double ${key})`,
);
return entry;
}
host.B.line(`${entry} = call ptr @sc_map_entry_f64(ptr ${map}, double ${key})`);
host.B.line(`${entry} = call ptr @sc_map_entry_f64${flavor}(ptr ${map}, double ${key})`);
return entry;
}
@@ -0,0 +1,68 @@
import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { expect, test } from "vitest";
import { compile, deserializeModule } from "../../index.js";
import { emitLlvmModule } from "./emitter.js";
async function llvmFor(source: string): Promise<string> {
const dir = await mkdtemp(join(tmpdir(), "scriptc-receiver-brackets-"));
try {
const entry = join(dir, "main.ts"),
output = join(dir, "main.ir.json");
await writeFile(entry, source);
const result = await compile(entry, { outDir: dir, outPath: output, outputKind: "ir" });
if (!result.ok)
throw new Error(result.diagnostics.map((item) => `${item.code}: ${item.message}`).join("\n"));
return emitLlvmModule(deserializeModule(await readFile(output, "utf8")));
} finally {
await rm(dir, { recursive: true, force: true });
}
}
function body(ir: string, name: string): string {
const match = new RegExp(`^define [^\\n]*@${name}\\(.*?^}`, "ms").exec(ir);
if (!match) throw new Error(`no function ${name}`);
return match[0];
}
test("a known signature adapter's direct entry brackets the receiver only when its original reads it", async () => {
const ir = await llvmFor(`
class Node {
kids: Node[] = [];
constructor(readonly kind: number) {}
}
function walk(root: Node, cb: (node: Node, depth: number) => void | "skip" | "stop"): void {
const stack: Node[] = [root];
while (stack.length > 0) {
const node = stack.pop()!;
const verdict = cb(node, stack.length);
if (verdict === "stop") return;
if (verdict === "skip") continue;
for (const kid of node.kids) stack.push(kid);
}
}
class Counter {
count = 0;
run(root: Node): number {
walk(root, (node) => {
if (node.kind === 3) this.count++;
});
return this.count;
}
}
const root = new Node(0);
root.kids.push(new Node(3));
console.log(new Counter().run(root));
`);
const walk = body(ir, "sc_bf_walk");
const blocks = walk.split(/^(?=[\w.]+:$)/m);
const direct = blocks.find((block) => block.startsWith("callback.direct"));
// The adapter forwards the receiver to the arrow function, which never
// reads it: the direct entry tests the closure's RECEIVER_FREE bit, and
// every receiver push is behind such a test.
expect(direct).toMatch(/ = and i32 %\S+, 16\n/);
const pushes = blocks.filter((block) => block.includes("@scr_dyn_this_push_dyn"));
expect(pushes.length).toBeGreaterThan(0);
for (const block of pushes) expect(block).toMatch(/^receiver\.push\d+:/);
});
@@ -103,6 +103,7 @@ const INERT_LIB_CALLS = new Set([
"json.parse",
"process.stderrWrite",
"process.stdoutWrite",
"process.stdioWriteDyn",
"process.envGet",
"process.envPairs",
"regex.groupEquals",
@@ -234,6 +234,17 @@ function powerOfTwo(e: IrExpr): boolean {
/** Typed-array element kinds whose elements are integers. */
const INTEGER_ELEMENTS = new Set(["u8", "u8c", "i8", "u16", "i16", "u32", "i32"]);
/** The values of each integer element kind. */
const ELEMENT_RANGES: Record<string, { min: number; max: number }> = {
u8: { min: 0, max: 255 },
u8c: { min: 0, max: 255 },
i8: { min: -128, max: 127 },
u16: { min: 0, max: 65535 },
i16: { min: -32768, max: 32767 },
u32: { min: 0, max: 2 ** 32 - 1 },
i32: { min: -(2 ** 31), max: 2 ** 31 - 1 },
};
/** A typed-array element read of an integer kind with no effect: an
* out-of-range read gives NaN, which fails its check. Float elements would
* only bail. */
@@ -904,6 +915,17 @@ export function emitSafeIntegerCheck(host: LlvmEmitterContext, value: string): [
return [integer, ok];
}
/** The condition that a bounded value's exponent is above `limit`. A
* value test it replaces can fail only past that limit; testing the
* exponent alone, which is usually the same in every iteration of a loop,
* keeps the value out of the test. Past the limit the write bails to the
* double copy, also for a value the replaced test would have passed. */
function exponentAbove(host: LlvmEmitterContext, bound: MaskBound, limit: number): string {
const above = host.B.tmp();
host.B.line(`${above} = icmp sgt i64 ${bound.exponent}, ${limit}`);
return above;
}
/** A condition that is true when an i64 lies outside the safe range. */
function unsafeInteger(host: LlvmEmitterContext, value: string): string {
const B = host.B;
@@ -932,6 +954,29 @@ function outsideRange(host: LlvmEmitterContext, value: string, low: number, high
return outside;
}
/** The range of an operand once its speculative evaluation continued: a
* literal's value, the range analysis' proof (well inside the safe
* integers), or for an integer element read, which bails on the NaN of an
* out-of-range index, the element kind's values. */
function operandRange(
host: LlvmEmitterContext,
e: IrExpr,
): { min: number; max: number } | undefined {
const range = staticRange(host, e);
if (range || !isSpeculativeByteRead(e) || e.kind !== "bytesIntrinsic") return range;
return e.receiver.type.kind === "bytes" ? ELEMENT_RANGES[e.receiver.type.elem] : undefined;
}
/** An i64 the speculative evaluation has bounded to [0, 2^(exponent +
* shift)): the floor-mod of a value by 2^exponent, shifted left by `shift`.
* `exponent` is an i64 already checked to be in [0, 52]; in a loop it is
* usually the same value on every iteration, so a test on it alone is
* loop-invariant. */
interface MaskBound {
exponent: string;
shift: number;
}
/** A literal's value or the range analysis' proof for an expression whose
* range lies well inside the safe integers. */
function staticRange(
@@ -962,6 +1007,7 @@ export function emitSpeculativeInteger(
bail: (condition: string) => void,
reads: ReadonlyMap<IrExpr, LlValue> = new Map(),
integral = false,
bounds = new Map<string, MaskBound>(),
): string {
const B = host.B;
// A value the range analysis already proves takes the ordinary integer
@@ -1000,12 +1046,13 @@ export function emitSpeculativeInteger(
// bits, also for a negative value: in [0, 2^n), and never -0.
const mod = floorModPowerOfTwo(e);
if (mod) {
const value = emitSpeculativeInteger(host, mod.value, bail, reads, integral);
const value = emitSpeculativeInteger(host, mod.value, bail, reads, integral, bounds);
const n = exponentOf(host, mod.power, bail, reads, integral);
const mask = B.tmp(),
low = B.tmp();
B.line(`${mask} = sub i64 ${shiftOne(host, n)}, 1`);
B.line(`${low} = and i64 ${value}, ${mask}`);
bounds.set(low, { exponent: n, shift: 0 });
return low;
}
// A product with 2^n shifts: it is safe iff the other factor's
@@ -1013,8 +1060,19 @@ export function emitSpeculativeInteger(
// integer and a positive power is +0.
if (e.op === "*" && (powerOfTwo(e.right) || powerOfTwo(e.left))) {
const [factor, power] = powerOfTwo(e.right) ? [e.left, e.right] : [e.right, e.left];
const value = emitSpeculativeInteger(host, factor, bail, reads, integral);
const value = emitSpeculativeInteger(host, factor, bail, reads, integral, bounds);
const n = exponentOf(host, power, bail, reads, integral);
// A factor below 2^(m + s) times 2^k is below 2^(m + s + k), so
// safe when m + s + k <= 53.
const bound = bounds.get(value);
const k = power.kind === "numLit" ? exactPowerOfTwo(power.value) : null;
if (bound && k !== null) {
bail(exponentAbove(host, bound, 53 - bound.shift - k));
const product = B.tmp();
B.line(`${product} = shl nuw nsw i64 ${value}, ${n}`);
bounds.set(product, { exponent: bound.exponent, shift: bound.shift + k });
return product;
}
const limit = B.tmp(),
magnitude = B.tmp(),
outside = B.tmp(),
@@ -1027,16 +1085,24 @@ export function emitSpeculativeInteger(
B.line(`${product} = shl nsw i64 ${value}, ${n}`);
return product;
}
const a = emitSpeculativeInteger(host, e.left, bail, reads, integral);
const b = emitSpeculativeInteger(host, e.right, bail, reads, integral);
const a = emitSpeculativeInteger(host, e.left, bail, reads, integral, bounds);
const b = emitSpeculativeInteger(host, e.right, bail, reads, integral, bounds);
const result = B.tmp();
if (e.op === "+" || e.op === "-") {
// With one operand's range known, the other is checked against the
// narrowed bounds before the operation: the check then depends on
// no new value, and LLVM keeps the operation itself simple.
const right = staticRange(host, e.right);
const left = e.op === "+" ? staticRange(host, e.left) : undefined;
if (right) {
const right = operandRange(host, e.right);
const left = e.op === "+" ? operandRange(host, e.left) : undefined;
// A bounded operand, in [0, 2^(m + s)), plus one in [lo, hi] with
// |lo|, hi <= 2^52 is safe when m + s <= 52.
const bound = (value: string): MaskBound | undefined =>
e.op === "+" ? bounds.get(value) : undefined;
const aBound = right ? bound(a) : undefined;
const bBound = !right && left ? bound(b) : undefined;
if (aBound) bail(exponentAbove(host, aBound, 52 - aBound.shift));
else if (bBound) bail(exponentAbove(host, bBound, 52 - bBound.shift));
else if (right) {
// a + b with b in [lo, hi] is safe iff a in [-SAFE - lo, SAFE - hi];
// a - b iff a in [-SAFE + hi, SAFE + lo].
const low = e.op === "+" ? -SAFE - right.min : -SAFE + right.max;
@@ -563,6 +563,9 @@ function helperProfileMatch(match: NativeProfileMatch): Record<string, number> {
missing: match.missing,
recovered: match.recovered,
unknown_partitions: match.unknownPartitions,
...(match.annotatedCounts === undefined || match.mismatchedCounts === undefined
? {}
: { annotated_counts: match.annotatedCounts, mismatched_counts: match.mismatchedCounts }),
};
}
@@ -34,6 +34,12 @@ export interface NativeLinkFeatures {
/** The program uses process.stdin.setEncoding, string 'data' listeners
* or stdio 'resize' events: links the SCR_STDIO_EXTENDED events unit. */
stdioExtended?: boolean;
/** A write completion callback takes the error argument: links the
* SCR_STREAM_WRITE_ERRORS build of the stream unit. */
streamWriteErrors?: boolean;
/** The program reaches the process output streams' write slot: links the
* SCR_STDIO_WRITE_SLOT build of the events unit. */
stdioWriteSlot?: boolean;
/** Calls into Node.js (@scriptc/node): scr_node_island.c and the
* SCR_NODE_ISLANDS variant of the event loop unit. */
nodeIsland?: boolean;
@@ -43,6 +49,9 @@ export interface NativeLinkFeatures {
/** A module namespace boxed for typed code links
* scr_module_namespace_dyn.c; absent (false) otherwise. */
moduleNamespaceDyn?: boolean;
/** Primitive wrapper objects link scr_js_wrapper.c; absent (false)
* otherwise. */
primitiveWrapper?: boolean;
zlib: boolean;
/** node:zlib's Brotli forms: scr_brotli.c and the vendored brotli archive. */
brotli?: boolean;
@@ -120,6 +129,8 @@ export interface NativeLinkFeatures {
* scr_text_codec_dyn.c. */
textCodecDyn: boolean;
watch: boolean;
/** fs.watchFile/fs.unwatchFile: scr_watch_file.c. */
watchFile: boolean;
foreignFfi: boolean;
nodeTest: boolean;
tls: boolean;
@@ -144,6 +155,10 @@ export interface NativeLinkFeatures {
x509: boolean;
/** perf_hooks event-loop metrics: scr_perf_loop.c. */
perfLoop: boolean;
/** Intl over ICU (scr_intl.c plus the ICU archive and its data
* file): the `icu.*` libCalls of the first slice. Absent (false) for
* programs without them, so their link features stay unchanged. */
intl?: boolean;
tlsCa: boolean;
/** crypto.subtle importKey/sign/verify/digest and CryptoKey values:
* scr_webcrypto.c. */
@@ -34,6 +34,7 @@ export async function createNativeLinkInfo(options: {
features: pack.features,
runtime: pack.selectedRuntimeArtifacts,
archives: pack.selectedArchiveArtifacts,
data: pack.selectedDataArtifacts,
systemLibraries: pack.systemLibraries,
},
});
@@ -145,6 +145,20 @@ test("the helper's profile match is read from its output", () => {
'{"ok":true,"profile":{"annotated":1,"mismatched":0,"missing":0,"recovered":-1,"unknown_partitions":0}}\n',
),
).toBeNull();
// With the profile's counters on the annotated and the mismatched functions.
expect(
parseNativeProfileMatch(
'{"ok":true,"profile":{"annotated":120,"mismatched":7,"missing":40,"recovered":0,"unknown_partitions":0,"annotated_counts":9000,"mismatched_counts":1000}}\n',
),
).toEqual({
annotated: 120,
mismatched: 7,
missing: 40,
recovered: 0,
unknownPartitions: 0,
annotatedCounts: 9000,
mismatchedCounts: 1000,
});
// A helper that predates the report prints nothing.
expect(parseNativeProfileMatch("")).toBeNull();
expect(parseNativeProfileMatch('{"ok":true}\n')).toBeNull();
@@ -111,13 +111,16 @@ export function nativeProfileHelperArgs(profile: NativeProfileOptions | undefine
* profile has no record of (new, or never run while it was collected),
* mismatched functions that the block-count file annotated (never more
* than the mismatched ones), and partitions reused from the cache without
* their counts. */
* their counts. Helpers that weigh the match also report the profile's
* counters on the annotated and on the mismatched functions. */
export interface NativeProfileMatch {
annotated: number;
mismatched: number;
missing: number;
recovered: number;
unknownPartitions: number;
annotatedCounts?: number;
mismatchedCounts?: number;
}
/** The profile match in a successful helper emit's output, or null when the
@@ -150,7 +153,18 @@ export function parseNativeProfileMatch(stdout: string): NativeProfileMatch | nu
unknownPartitions === null
)
continue;
return { annotated, mismatched, missing, recovered, unknownPartitions };
const annotatedCounts = count("annotated_counts");
const mismatchedCounts = count("mismatched_counts");
return {
annotated,
mismatched,
missing,
recovered,
unknownPartitions,
...(annotatedCounts === null || mismatchedCounts === null
? {}
: { annotatedCounts, mismatchedCounts }),
};
}
return null;
}
@@ -83,6 +83,17 @@ export interface CcOptions {
/** The program uses node:zlib's Brotli forms (with zlib): compiles
* scr_brotli.c and links the vendored brotli archive. */
brotli?: boolean;
/** The program uses an Intl form that only ICU can answer (the `icu.*`
* libCalls: default-locale number/date formats, word segmentation):
* compiles scr_intl.c and links the ICU archive, and the
* executable gets icudt78l.dat next to it. Every engine-free form keeps
* linking none of it. */
intl?: boolean;
/** With `intl`: the installed runtime pack's ICU archive and the include
* root of the ICU headers it ships. The C-source path then links that
* archive and compiles scr_intl.c against those headers, instead of
* building ICU from its fetched release with Zig's C++ driver. */
icuPack?: { archive: string; include: string };
/** The program uses node:assert (index.ts detects assert.* libCalls on
* the IR): compiles scr_assert.c — the zlib gating precedent, so
* assert-free binaries keep their exact size class. scr_regex.c calls
@@ -161,6 +172,12 @@ export interface CcOptions {
warningReplace?: boolean;
/** SCR_STDIO_EXTENDED: stdin encoding, string data listeners, resize. */
stdioExtended?: boolean;
/** SCR_STREAM_WRITE_ERRORS: the stream unit publishes the error each
* write completion callback receives (error-taking write callbacks). */
streamWriteErrors?: boolean;
/** SCR_STDIO_WRITE_SLOT: the events unit carries the typed write slot
* (the write method as a value, its replacement, run-time-typed chunks). */
stdioWriteSlot?: boolean;
/** Calls into Node.js (@scriptc/node): compiles scr_node_island.c and
* the event loop with SCR_NODE_ISLANDS (its helper slot). */
nodeIsland?: boolean;
@@ -170,6 +187,8 @@ export interface CcOptions {
/** A module namespace boxed for typed code: compiles
* scr_module_namespace_dyn.c. */
moduleNamespaceDyn?: boolean;
/** Primitive wrapper objects: compiles scr_js_wrapper.c. */
primitiveWrapper?: boolean;
/** The program uses the process-events surface (signal/exit listeners,
* stdin events, for-await over stdin — moduleUsesProcessEvents on the
* IR): compiles scr_events.c into the binary. Event-free binaries keep
@@ -258,6 +277,11 @@ export interface CcOptions {
* scr_watch.c into the binary — the net gating precedent, so watch-free
* binaries keep their exact link line. */
watch?: boolean;
/** The program uses fs.watchFile or fs.unwatchFile (moduleUsesFsWatchFile
* on the IR): compiles scr_watch_file.c into the binary — its own gated
* unit under the same precedent, so polling-free binaries keep their
* exact link line. */
watchFile?: boolean;
/** The executable manifest has a format-5 foreign callback descriptor:
* compiles the MPSC queue/self-pipe unit. Other FFI and non-FFI binaries
* keep their existing runtime size class. */
@@ -84,6 +84,9 @@ import {
vendorBuildCacheRoot,
tlsArchivePath,
brotliArchivePath,
ensureIcuSources,
icuArchivePath,
ensureIcuArchive,
lreObjectPaths,
zlibObjectPaths,
stageVendorInputs,
@@ -124,6 +127,10 @@ interface RuntimeUnitContext {
zlibObjects: readonly string[];
tlsArchive: string | null;
brotliArchive: string | null;
icuArchive: string | null;
/** The include roots of the ICU headers scr_intl.c compiles against: the
* runtime pack's, or the fetched release's common/ and i18n/. */
icuIncludeDirs: readonly string[];
}
/** An optional runtime input: runtime sources compiled when `when` holds,
@@ -174,9 +181,10 @@ const RUNTIME_UNITS: readonly RuntimeUnit[] = [
// SCR_MUSL flag and this target-selected TU travel together.
unit((u) => isMuslTarget(u.driver), "scr_musl.c"),
// scr_glob.c (path.matchesGlob) compiles its segments through the
// regex unit and scr_idna.c (node:url's IDNA) normalizes through it,
// so both ride the same switch; section GC drops them from binaries
// that never use them.
// regex unit, scr_idna.c (node:url's IDNA) normalizes through it and
// scr_path_win32_relative.c lowercases through its case mapping, so all
// three ride the same switch; section GC drops them from binaries that
// never use them.
unitArgs((u) =>
u.regex
? [
@@ -185,6 +193,7 @@ const RUNTIME_UNITS: readonly RuntimeUnit[] = [
u.rt(join(u.rtDir, "scr_regex.c")),
u.rt(join(u.rtDir, "scr_glob.c")),
u.rt(join(u.rtDir, "scr_idna.c")),
u.rt(join(u.rtDir, "scr_path_win32_relative.c")),
...u.lreObjects,
]
: [],
@@ -212,6 +221,8 @@ const RUNTIME_UNITS: readonly RuntimeUnit[] = [
unit((u) => u.opts.moduleNamespace === true, "scr_module_namespace.c"),
// A module namespace as the frozen null-prototype object typed code sees.
unit((u) => u.opts.moduleNamespaceDyn === true, "scr_module_namespace_dyn.c"),
// Primitive wrapper objects (new String/Number/Boolean, Object(primitive)).
unit((u) => u.opts.primitiveWrapper === true, "scr_js_wrapper.c"),
unit(
(u) =>
u.opts.dynAsync ||
@@ -253,6 +264,20 @@ const RUNTIME_UNITS: readonly RuntimeUnit[] = [
]
: [],
),
// Intl over ICU (scr_intl.c): ICU's C API only, at the default locale,
// reading its data file from beside the executable. The ICU headers come
// from the installed runtime pack beside its archive, or, when ICU is
// built from its fetched release, from that source tree (common/unicode
// for the core API, i18n/unicode for udat.h and unumberformatter.h).
unitArgs((u) =>
u.icuIncludeDirs.length === 0 || u.icuArchive === null
? []
: [
...u.icuIncludeDirs.flatMap((dir) => ["-I", dir]),
u.rt(join(u.rtDir, "scr_intl.c")),
u.icuArchive,
],
),
unit((u) => u.opts.events, "scr_events.c", "scr_readline.c"),
// readline's terminal mode: rl.terminal sets the regex unit it
// measures with.
@@ -344,6 +369,9 @@ const RUNTIME_UNITS: readonly RuntimeUnit[] = [
unit((u) => u.opts.dgram && u.opts.dgramDyn, "scr_dgram_dyn.c"),
unit((u) => u.opts.textCodecDyn, "scr_text_codec_dyn.c"),
unit((u) => u.opts.watch, "scr_watch.c"),
// fs.watchFile's stat poller: its own gated unit (no loop hook — the
// re-armed timers are the liveness), linked only by programs that poll.
unit((u) => u.opts.watchFile, "scr_watch_file.c"),
unit((u) => u.opts.foreignFfi, "scr_ffi_queue.c"),
unit((u) => u.opts.nodeTest, "scr_test.c"),
unit((u) => u.opts.webcrypto, "scr_webcrypto.c"),
@@ -389,6 +417,7 @@ const RUNTIME_UNITS: readonly RuntimeUnit[] = [
"scr_node_island_handle.c",
"scr_node_island_export.c",
"scr_node_island_boot.c",
"scr_node_island_gc.c",
),
// Readable.fromWeb reads a web ReadableStream through its reader.
unit((u) => u.nativeFetch && u.opts.stream, "scr_stream_from_web.c"),
@@ -417,12 +446,19 @@ const RUNTIME_DEFINES: readonly {
{ define: "SCR_JS_TIER", when: (opts) => opts.dynTier === true },
{ define: "SCR_WARNING_REPLACE", when: (opts) => opts.warningReplace },
{ define: "SCR_STDIO_EXTENDED", when: (opts) => opts.stdioExtended },
// scr_stream.c's write-callback error publication, which the compiler's
// error-first write adapter reads.
{ define: "SCR_STREAM_WRITE_ERRORS", when: (opts) => opts.streamWriteErrors },
// scr_events.c's typed process output write slot.
{ define: "SCR_STDIO_WRITE_SLOT", when: (opts) => opts.stdioWriteSlot },
// scr_async.c's native promise reactions, which the web streams unit uses,
// scr_json.c's and scr_dyn_builtin_class.c's instanceof over its classes,
// and scr_fetch.c's bodies of its streams.
{ define: "SCR_NATIVE_REACTIONS", when: (opts) => opts.webStreams },
{ define: "SCR_HANDLE_CLASSES", when: (opts) => opts.webStreams },
{ define: "SCR_YIN", when: (opts) => opts.youngNursery },
// scr_lib.c's path stat and zero Stats for the fs.watchFile poll.
{ define: "SCR_FS_WATCH_FILE", when: (opts) => opts.watchFile },
{ define: "SCR_NODE_ISLANDS", when: (opts) => opts.nodeIsland },
];
@@ -455,6 +491,7 @@ export async function compileCInternal(
const pki = opts.pki ?? false;
const mbedtls = tls || cipher || ecdh || dh || pki;
const brotli = (opts.zlib ?? false) && opts.brotli === true;
const icu = opts.intl ?? false;
const driver = resolveCc();
const darwinDebugSymbols = needsDarwinDebugSymbols(
targetPlatform(driver),
@@ -697,6 +734,24 @@ export async function compileCInternal(
let brotliArchive = brotli
? brotliArchivePath(sanitize, driver, vendorBuildIdentity, vendorCacheRoot)
: null;
// The installed runtime pack's ICU archive is the one this path would
// build (uninstrumented either way), so a pack that ships it and its
// headers spares the Zig C++ build and the source download.
const icuPack = icu ? (opts.icuPack ?? null) : null;
let icuArchive = icu
? (icuPack?.archive ?? icuArchivePath(driver, vendorBuildIdentity, vendorCacheRoot))
: null;
// scr_intl.c's ICU headers. They are known before the runtime-object flags
// are: those flags compile every runtime unit, so without them a cached
// scr_intl.o would take whatever ICU headers the host has.
const icuSource =
icu && icuPack === null ? join(await ensureIcuSources(vendorCacheRoot), "source") : null;
const icuIncludeDirs: readonly string[] =
icuPack !== null
? [icuPack.include]
: icuSource !== null
? [join(icuSource, "common"), join(icuSource, "i18n")]
: [];
// ours would collide; see scr_regex.c) — the standalone objects are for
// static builds only.
@@ -751,6 +806,18 @@ export async function compileCInternal(
: await stageVendorInputs(materialize, join(stageRoot, "brotli"));
brotliArchive = paths[0]!;
}
if (icu && icuPack === null) {
const cachedArchive = icuArchivePath(driver, vendorBuildIdentity, materializeCacheRoot);
protectCachedArtifact(cacheWarmPaths, cachedArchive);
const materialize = async (): Promise<string[]> => [
await ensureIcuArchive(driver, vendorBuildIdentity, materializeCacheRoot),
];
const paths =
stageRoot === undefined
? await materialize()
: await stageVendorInputs(materialize, join(stageRoot, "icu"));
icuArchive = paths[0]!;
}
if (regex) {
for (const object of lreObjectPaths(
sanitize,
@@ -838,6 +905,8 @@ export async function compileCInternal(
zlibObjects,
tlsArchive,
brotliArchive,
icuArchive,
icuIncludeDirs,
}),
// A .ll program TU (the LLVM backend) deliberately carries no target
// triple (byte-stable output; clang supplies the host/SCRIPTC_TARGET
@@ -886,6 +955,7 @@ export async function compileCInternal(
...(regex ? ["-I", vendorEngineDir()] : []),
...(zlibObjects.length > 0 ? ["-I", vendorZlibDir()] : []),
...(tlsArchive !== null ? ["-I", join(vendorTlsDir(), "include")] : []),
...icuIncludeDirs.flatMap((dir) => ["-I", dir]),
];
// Keep sanitizer runtime objects optimized in both postures. Large dev
// programs still compile at O0, avoiding an expensive LLVM optimization
@@ -104,7 +104,7 @@ export async function compileLibArchive(opts: LibArchiveOptions): Promise<void>
// Zig's musl sysroot does not provide arc4random_buf. Keep the fallback
// inside the archive so library embedders need no extra system library.
...(isMuslTarget(driver) ? ["scr_musl.c"] : []),
...(regex ? ["scr_regex.c", "scr_glob.c"] : []),
...(regex ? ["scr_regex.c", "scr_glob.c", "scr_path_win32_relative.c"] : []),
...(opts.assert || regex || opts.symbol ? ["scr_assert.c"] : []),
...(opts.inspect ? ["scr_inspect.c", "scr_console_native.c"] : []),
...(opts.symbol ? ["scr_symbol.c"] : []),
@@ -3,7 +3,13 @@ import { createRequire } from "node:module";
import { readdir, readFile, realpath, stat } from "node:fs/promises";
import { basename, dirname, extname, join, relative, resolve } from "node:path";
import { fileURLToPath } from "node:url";
import { BROTLI_VERSION, MBEDTLS_VERSION, QJS_COMMIT, ZLIB_VERSION } from "../vendor-inputs.js";
import {
BROTLI_VERSION,
ICU_VERSION,
MBEDTLS_VERSION,
QJS_COMMIT,
ZLIB_VERSION,
} from "../vendor-inputs.js";
import { stableTestMemo } from "./session.js";
const implementationPath = fileURLToPath(import.meta.url);
@@ -139,6 +145,7 @@ async function runtimeFingerprintFresh(rtDir: string): Promise<string> {
.update(MBEDTLS_VERSION)
.update(ZLIB_VERSION)
.update(BROTLI_VERSION)
.update(ICU_VERSION)
.update("\0backend-recipe\0");
for (const path of NATIVE_RECIPE_IMPLEMENTATION_PATHS) {
h.update(basename(path))
@@ -159,19 +166,29 @@ async function runtimeFingerprintFresh(rtDir: string): Promise<string> {
return h.digest("hex");
}
interface RuntimeFingerprintGroup {
label: string;
dir: string;
}
async function runtimeFingerprintInputGroups(
rtDir: string,
): Promise<{ label: string; dir: string; names: string[] }[]> {
return Promise.all(
[
{ label: "runtime", dir: rtDir },
{ label: "ryu", dir: join(rtDir, "..", "vendor", "ryu") },
{ label: "quickjs-ng", dir: join(rtDir, "..", "vendor", "quickjs-ng") },
{ label: "mbedtls", dir: join(rtDir, "..", "vendor", "mbedtls") },
{ label: "zlib", dir: join(rtDir, "..", "vendor", "zlib") },
{ label: "brotli", dir: join(rtDir, "..", "vendor", "brotli", "c") },
{ label: "curl", dir: join(rtDir, "..", "vendor", "curl") },
].map(async (group) => {
(
[
{ label: "runtime", dir: rtDir },
{ label: "ryu", dir: join(rtDir, "..", "vendor", "ryu") },
{ label: "quickjs-ng", dir: join(rtDir, "..", "vendor", "quickjs-ng") },
{ label: "mbedtls", dir: join(rtDir, "..", "vendor", "mbedtls") },
{ label: "zlib", dir: join(rtDir, "..", "vendor", "zlib") },
{ label: "brotli", dir: join(rtDir, "..", "vendor", "brotli", "c") },
// ICU is fetched, not vendored: its pinned version and sha256 key the
// archive's cache path (vendor-archives.ts icuArchivePath), so it is
// not part of the runtime tree's fingerprint.
{ label: "curl", dir: join(rtDir, "..", "vendor", "curl") },
] satisfies RuntimeFingerprintGroup[]
).map(async (group) => {
const names = (await nativeSourceFiles(group.dir, true))
.map((path) => relative(group.dir, path))
.sort();
@@ -9,6 +9,9 @@ export const {
vendorBrotliDir,
brotliArchivePath,
ensureBrotliArchive,
ensureIcuSources,
icuArchivePath,
ensureIcuArchive,
vendorBuildCacheRoot,
vendorCacheTargetFlavor,
vendorCacheBuildIdentity,
@@ -34,7 +34,7 @@ interface RuntimePackUnit {
}
interface RuntimePackArchive extends RuntimePackArtifact {
id: "quickjs" | "libregexp" | "zlib" | "brotli" | "mbedtls";
id: "quickjs" | "libregexp" | "zlib" | "brotli" | "mbedtls" | "icu";
predicate: RuntimePredicate;
}
@@ -74,6 +74,15 @@ export interface RuntimePackManifest {
};
flavors: Partial<Record<RuntimePackFlavor, RuntimePackFlavorManifest>>;
archives: RuntimePackArchive[];
/** Files the linked executable needs at RUN time rather than at link time,
* installed beside it on every build. ICU's data file (icudt78l.dat, 31.6
* MiB) is the only one; Intl programs copy it, every other program links
* neither ICU nor its data. */
data?: RuntimePackArtifact[];
/** C headers under one include root (`<root>/unicode/*.h`): the ICU C API
* headers scr_intl.c includes, which a sanitized build compiles that unit
* against (native/sanitizer.ts). Only a pack that ships ICU lists them. */
headers?: RuntimePackArtifact[];
system_libraries: { name: string; predicate: RuntimePredicate }[];
licenses: { path: string; license: string }[];
}
@@ -101,10 +110,17 @@ export interface RuntimeFeatureSet extends NativeLinkFeatures {
computedRequire: boolean;
warningReplace: boolean;
stdioExtended: boolean;
/** The program's write completion callbacks take the error argument:
* the SCR_STREAM_WRITE_ERRORS build of the stream unit. */
streamWriteErrors: boolean;
/** The program reaches the process output streams' write slot: the
* SCR_STDIO_WRITE_SLOT build of the events unit. */
stdioWriteSlot: boolean;
nodeIsland: boolean;
brotli: boolean;
moduleNamespace: boolean;
moduleNamespaceDyn: boolean;
primitiveWrapper: boolean;
nativeFetch: boolean;
abortSignalOnly: boolean;
netEffective: boolean;
@@ -245,6 +261,10 @@ export function parseRuntimePackManifest(value: unknown): RuntimePackManifest {
validArtifact(raw) && typeof archive?.id === "string" && validPredicate(archive.predicate)
);
}) ||
(manifest.data !== undefined &&
(!Array.isArray(manifest.data) || !manifest.data.every(validArtifact))) ||
(manifest.headers !== undefined &&
(!Array.isArray(manifest.headers) || !manifest.headers.every(validArtifact))) ||
!Array.isArray(manifest.system_libraries) ||
!manifest.system_libraries.every((raw) => {
const library = object(raw);
@@ -312,10 +332,13 @@ export function effectiveRuntimeFeatures(
computedRequire: features.computedRequire === true,
warningReplace: features.warningReplace === true,
stdioExtended: features.stdioExtended === true,
streamWriteErrors: features.streamWriteErrors === true,
stdioWriteSlot: features.stdioWriteSlot === true,
nodeIsland: features.nodeIsland === true,
brotli: features.brotli === true,
moduleNamespace: features.moduleNamespace === true,
moduleNamespaceDyn: features.moduleNamespaceDyn === true,
primitiveWrapper: features.primitiveWrapper === true,
nativeFetch,
// The fetch unit has AbortController and AbortSignal; scr_abort.c is
// the same implementation without the fetch transport.
@@ -395,6 +418,8 @@ export interface RuntimePackArtifacts {
/** Selected unit variants; each may carry its bitcode artifact. */
runtime: (RuntimePackArtifact & { bitcode?: RuntimePackArtifact })[];
archives: RuntimePackArtifact[];
/** Files to install beside the executable (ICU's data file for Intl). */
data: RuntimePackArtifact[];
systemLibraries: string[];
}
@@ -447,12 +472,26 @@ export function selectRuntimePackArtifacts(
`runtime pack has no ${key} flavor; reinstall the matching runtime package`,
"invalid",
);
// Intl needs the ICU objects only the host-flavored packs carry (ICU is
// C++, so a pack builds it with Zig's C++ driver for its own glibc target);
// say so instead of linking a program whose scr_intl.c references would go
// undefined.
if (
features.intl === true &&
!selectedFlavor.runtime_units.some((unit) => unit.source === "scr_intl.c")
)
throw new RuntimePackError(
`the ${manifest.target.name} runtime pack has no ICU objects, which Intl needs; reinstall a runtime package that supports Intl`,
"unsupported",
);
return {
features,
runtime: selectedRuntimeUnits(selectedFlavor, features).map(({ variant }) => variant),
archives: manifest.archives.filter((archive) =>
evaluateRuntimePredicate(archive.predicate, features),
),
// ICU's data file is 31.6 MiB: only a program that links ICU installs it.
data: features.intl === true ? (manifest.data ?? []) : [],
systemLibraries: manifest.system_libraries
.filter((entry) => evaluateRuntimePredicate(entry.predicate, features))
.map((entry) => entry.name),
@@ -1,8 +1,12 @@
import { createHash } from "node:crypto";
import {
chmodSync,
existsSync,
mkdirSync,
readdirSync,
readFileSync,
renameSync,
rmSync,
statSync,
symlinkSync,
unlinkSync,
@@ -148,10 +152,12 @@ async function packFixture() {
store,
);
let stages = 0;
const stage = (selection: NativeRuntimeSelection) => {
const directory = join(root, `stage-${stages++}`);
const stage = (selection: NativeRuntimeSelection, with_: RuntimePackStore = store) => {
const directory = join(tmpdir(), `scriptc-runtime-pack-stage-${process.pid}-${stages++}`);
rmSync(directory, { recursive: true, force: true });
mkdirSync(directory);
return stageNativeRuntimeSelection(selection, directory, store);
dirs.push(directory);
return stageNativeRuntimeSelection(selection, directory, with_);
};
return { root, object, manifestText, entries, select, stage };
}
@@ -241,3 +247,180 @@ test.skipIf(process.platform === "win32")(
expect(readFileSync(staged.runtimeObjects[0]!, "utf8")).toBe("base object");
},
);
/** A store that keeps nothing: a build with an empty cache. */
function emptyStore(): RuntimePackStore {
const entries = new Map<string, string>();
return {
read: (key) => entries.get(key) ?? null,
write: (key, text) => void entries.set(key, text),
};
}
const packRecord = (root: string) => join(root, "runtime-pack.verified.json");
const hashes = () => vi.mocked(createHash).mock.calls.length;
test("a build with an empty cache trusts the record beside the pack", async () => {
const f = await packFixture();
const selection = f.select();
let start = hashes();
f.stage(selection);
const firstStage = hashes() - start;
const record = JSON.parse(readFileSync(packRecord(f.root), "utf8"));
expect(record.schema).toBe(1);
expect(record.artifacts.map((artifact: { path: string }) => artifact.path)).toEqual([
"artifacts/base.o",
]);
start = hashes();
const staged = f.stage(selection, emptyStore());
// Only the store key is hashed.
expect(hashes() - start).toBe(firstStage - 1);
expect(readFileSync(staged.runtimeObjects[0]!, "utf8")).toBe("base object");
});
test.each(["replaced", "rewritten in place", "resized"])(
"the record beside the pack does not trust an artifact that was %s",
async (change) => {
const f = await packFixture();
const selection = f.select();
f.stage(selection);
const before = statSync(f.object);
const changed = change === "replaced" ? f.object + ".new" : f.object;
writeFileSync(changed, "x".repeat(before.size + (change === "resized" ? 1 : 0)));
utimesSync(changed, before.atime, before.mtime);
if (changed !== f.object) renameSync(changed, f.object);
expect(() => f.stage(selection, emptyStore())).toThrow(
"runtime pack artifact hash mismatch: artifacts/base.o",
);
},
);
test("an artifact whose mtime changed is hashed again and recorded again", async () => {
const f = await packFixture();
const selection = f.select();
f.stage(selection);
const before = statSync(f.object);
utimesSync(f.object, before.atime, new Date(before.mtimeMs + 5000));
const start = hashes();
f.stage(selection, emptyStore());
// The store key and the artifact.
expect(hashes() - start).toBe(2);
const [entry] = JSON.parse(readFileSync(packRecord(f.root), "utf8")).artifacts;
expect(entry.mtime).toBe(statSync(f.object).mtimeMs);
const again = hashes();
f.stage(selection, emptyStore());
expect(hashes() - again).toBe(1);
});
test.each([
["truncated", () => "{"],
["null", () => "null"],
["another schema", (text: string) => text.replace('"schema":1', '"schema":2')],
[
"an entry with a string ctime",
(text: string) => text.replace(/"ctime":([\d.]+)/, '"ctime":"$1"'),
],
["an entry without an inode", (text: string) => text.replace(/"ino":[\d.]+,/, "")],
[
"another digest",
(text: string) => text.replace(/"sha256":"[0-9a-f]+"/, `"sha256":"${"0".repeat(64)}"`),
],
])("a record beside the pack that is %s is not trusted", async (_name, rewrite) => {
const f = await packFixture();
const selection = f.select();
f.stage(selection);
const text = readFileSync(packRecord(f.root), "utf8");
const rewritten = rewrite(text);
expect(rewritten).not.toBe(text);
writeFileSync(packRecord(f.root), rewritten);
const start = hashes();
const staged = f.stage(selection, emptyStore());
expect(hashes() - start).toBe(2);
expect(readFileSync(staged.runtimeObjects[0]!, "utf8")).toBe("base object");
// The record is written again, whole.
const [entry] = JSON.parse(readFileSync(packRecord(f.root), "utf8")).artifacts;
expect(entry.ino).toBe(statSync(f.object).ino);
});
test("an unreadable record beside the pack is not trusted, and staging still succeeds", async () => {
const f = await packFixture();
const selection = f.select();
// A directory where the record would be: it can be neither read nor replaced.
mkdirSync(packRecord(f.root));
const start = hashes();
f.stage(selection, emptyStore());
f.stage(selection, emptyStore());
expect(hashes() - start).toBe(4);
expect(readdirSync(f.root).filter((name) => name.endsWith(".tmp"))).toEqual([]);
});
test.skipIf(process.platform === "win32")(
"a pack directory that cannot be written keeps the record in the cache only",
async (context) => {
const f = await packFixture();
const selection = f.select();
chmodSync(f.root, 0o555);
try {
// A process that may write any directory cannot run this case.
try {
writeFileSync(join(f.root, "probe"), "");
rmSync(join(f.root, "probe"));
context.skip();
} catch {
/* The directory is read-only for this process. */
}
const store = emptyStore();
let start = hashes();
f.stage(selection, store);
expect(hashes() - start).toBe(2);
expect(existsSync(packRecord(f.root))).toBe(false);
expect(readdirSync(f.root).filter((name) => name.endsWith(".tmp"))).toEqual([]);
// The cache's record still answers the next build with that cache.
start = hashes();
f.stage(selection, store);
expect(hashes() - start).toBe(1);
} finally {
chmodSync(f.root, 0o755);
}
},
);
test("writing the record beside the pack keeps the entries another build wrote", async () => {
const f = await packFixture();
const selection = f.select();
const other = {
path: "artifacts/other.o",
sha256: "1".repeat(64),
dev: 1,
ino: 2,
size: 3,
mtime: 4,
ctime: 5,
};
writeFileSync(packRecord(f.root), JSON.stringify({ schema: 1, artifacts: [other] }));
f.stage(selection, emptyStore());
const record = JSON.parse(readFileSync(packRecord(f.root), "utf8"));
expect(record.artifacts.map((artifact: { path: string }) => artifact.path).sort()).toEqual([
"artifacts/base.o",
"artifacts/other.o",
]);
expect(
record.artifacts.find((artifact: { path: string }) => artifact.path === other.path),
).toEqual(other);
expect(readdirSync(f.root).filter((name) => name.endsWith(".tmp"))).toEqual([]);
});
test("a cache that already trusts the artifacts writes the record beside the pack", async () => {
const f = await packFixture();
const selection = f.select();
f.stage(selection);
rmSync(packRecord(f.root));
const start = hashes();
f.stage(selection);
// The cache's record answers; only the store key is hashed.
expect(hashes() - start).toBe(1);
expect(existsSync(packRecord(f.root))).toBe(true);
const empty = hashes();
f.stage(selection, emptyStore());
expect(hashes() - empty).toBe(1);
});
@@ -2,22 +2,27 @@
* selected artifact is verified and copied from the same read into private
* storage. With a store, a pack's selections and the identities of the
* artifacts it verified are kept between builds: an artifact whose file
* identity is unchanged since it was hashed is not hashed again. */
* identity is unchanged since it was hashed is not hashed again. The
* identities are also kept in a record beside the pack when its directory
* can be written, which builds with an empty store read too. */
import type { NativeOptimization } from "./optimization.js";
import { createHash } from "node:crypto";
import { createHash, randomBytes, randomUUID } from "node:crypto";
import {
closeSync,
constants,
copyFileSync,
existsSync,
fstatSync,
mkdirSync,
openSync,
readFileSync,
renameSync,
rmSync,
statSync,
writeSync,
type Stats,
} from "node:fs";
import { dirname, join } from "node:path";
import { basename, dirname, join, resolve } from "node:path";
import type { NativeLinkFeatures } from "./native-link-info.js";
import type { NativeTargetSpec } from "./targets.js";
import {
@@ -94,7 +99,9 @@ function storeKey(text: string): string {
/** A stored selection: what selecting from the pack's files, with these
* identities, returned. */
interface StoredSelection {
schema: 1;
/** 2 adds the selection's data artifacts: a stored selection from before
* them would make an Intl build stage none of them, so it is discarded. */
schema: 2;
licenses: RuntimePackManifest["licenses"];
selected: RuntimePackArtifacts;
}
@@ -172,7 +179,7 @@ export function selectNativeRuntimePack(
sameIdentity(fileIdentity(packagePath), identities[0]!) &&
sameIdentity(fileIdentity(manifestPath), identities[1]!)
) {
const entry: StoredSelection = { schema: 1, licenses: manifest.licenses, selected };
const entry: StoredSelection = { schema: 2, licenses: manifest.licenses, selected };
store.write(key, JSON.stringify(entry));
}
} catch {
@@ -201,13 +208,14 @@ function readStoredSelection(text: string | null): StoredSelection | null {
if (value === null || typeof value !== "object") return null;
const entry = value as Partial<StoredSelection>;
if (
entry.schema !== 1 ||
entry.schema !== 2 ||
!Array.isArray(entry.licenses) ||
entry.selected === undefined ||
entry.selected === null ||
typeof entry.selected !== "object" ||
!Array.isArray(entry.selected.runtime) ||
!Array.isArray(entry.selected.archives) ||
!Array.isArray(entry.selected.data) ||
!Array.isArray(entry.selected.systemLibraries)
)
return null;
@@ -239,21 +247,53 @@ function readVerified(
try {
const text = store?.read(key) ?? null;
if (text === null) return verified;
const stored = JSON.parse(text) as StoredVerification;
if (stored.schema === 1)
for (const artifact of stored.artifacts) verified.set(artifact.path, artifact);
for (const artifact of storedArtifacts(text)) verified.set(artifact.path, artifact);
} catch {
/* A malformed entry verifies everything again. */
}
return verified;
}
/** The well-formed entries of a stored verification; none when the text is
* not a stored verification. */
function storedArtifacts(text: string): VerifiedArtifact[] {
const stored = JSON.parse(text) as Partial<StoredVerification> | null;
if (stored === null || stored.schema !== 1 || !Array.isArray(stored.artifacts)) return [];
const number = (value: unknown): boolean => typeof value === "number" && Number.isFinite(value);
return stored.artifacts.filter(
(artifact): artifact is VerifiedArtifact =>
artifact !== null &&
typeof artifact === "object" &&
typeof artifact.path === "string" &&
typeof artifact.sha256 === "string" &&
number(artifact.dev) &&
number(artifact.ino) &&
number(artifact.size) &&
number(artifact.mtime) &&
number(artifact.ctime),
);
}
/** `known` records `artifact`'s digest and size for a file with `identity`. */
function verifiedAs(
known: VerifiedArtifact | undefined,
artifact: { sha256: string; size: number },
identity: FileIdentity,
): boolean {
return (
known !== undefined &&
known.sha256 === artifact.sha256 &&
known.size === artifact.size &&
sameIdentity(known, identity)
);
}
function stageArtifact(
root: string,
stage: string,
artifact: RuntimePackArtifact,
verified: Map<string, VerifiedArtifact>,
newlyVerified: VerifiedArtifact[],
records: readonly Map<string, VerifiedArtifact>[],
verified: VerifiedArtifact[],
): string {
const source = join(root, artifact.path);
const destination = join(stage, artifact.path);
@@ -276,26 +316,17 @@ function stageArtifact(
}
// The bytes are those last hashed when the file kept the identity it had
// then for the whole read.
const known = verified.get(artifact.path);
const unchanged = sameIdentity(before, after);
const trusted =
known !== undefined &&
known.sha256 === artifact.sha256 &&
known.size === artifact.size &&
unchanged &&
known.dev === before.dev &&
known.ino === before.ino &&
known.size === before.size &&
known.mtime === before.mtime &&
known.ctime === before.ctime;
unchanged && records.some((record) => verifiedAs(record.get(artifact.path), artifact, before));
if (
bytes.length !== artifact.size ||
(!trusted && createHash("sha256").update(bytes).digest("hex") !== artifact.sha256)
) {
throw new RuntimePackError(`runtime pack artifact hash mismatch: ${artifact.path}`, "invalid");
}
if (!trusted && unchanged)
newlyVerified.push({
if (unchanged)
verified.push({
path: artifact.path,
sha256: artifact.sha256,
dev: before.dev,
@@ -353,10 +384,12 @@ export function stageNativeRuntimeSelection(
const packagePath = join(root, "package.json");
const manifestPath = join(root, "runtime-pack.json");
const key = storeKey(JSON.stringify({ schema: 1, verified: root }));
const verified = readVerified(store, key);
const newlyVerified: VerifiedArtifact[] = [];
const cacheRecord = readVerified(store, key);
const packRecordPath = join(root, PACK_VERIFIED_FILE);
const packRecord = readPackVerified(packRecordPath);
const verified: VerifiedArtifact[] = [];
const stage = (artifact: RuntimePackArtifact): string =>
stageArtifact(root, stageRoot, artifact, verified, newlyVerified);
stageArtifact(root, stageRoot, artifact, [cacheRecord, packRecord], verified);
const runtimeObjects = selected.runtime.map(stage);
const archives = selected.archives.map(stage);
for (const license of licenses) {
@@ -372,14 +405,112 @@ export function stageNativeRuntimeSelection(
) {
throw new RuntimePackError("runtime pack changed while staging verified artifacts", "invalid");
}
if (store !== undefined && newlyVerified.length > 0) {
for (const artifact of newlyVerified) verified.set(artifact.path, artifact);
const entry: StoredVerification = { schema: 1, artifacts: [...verified.values()] };
// Each record learns the artifacts verified here that it does not list
// with their current identity.
const missing = (record: Map<string, VerifiedArtifact>): VerifiedArtifact[] =>
verified.filter((artifact) => !verifiedAs(record.get(artifact.path), artifact, artifact));
const cacheMissing = missing(cacheRecord);
if (store !== undefined && cacheMissing.length > 0) {
for (const artifact of cacheMissing) cacheRecord.set(artifact.path, artifact);
const entry: StoredVerification = { schema: 1, artifacts: [...cacheRecord.values()] };
try {
store.write(key, JSON.stringify(entry));
} catch {
/* A failed write only means hashing again next time. */
}
}
const packMissing = missing(packRecord);
if (packMissing.length > 0) writePackVerified(packRecordPath, packMissing);
return { runtimeObjects, archives, systemLibraries: selected.systemLibraries };
}
/** Install the selected RUN-time data artifacts (ICU's icudt78l.dat) beside an
* executable this host linked. The in-process linker does the same from its
* own selection (backend/linker.ts); this host writes executables
* synchronously and has no link plan, so it installs them here.
*
* A file already present at the pinned size is left alone (31.6 MiB of pure
* cost otherwise), and the copy is atomic — a private temporary, then a
* rename — so two concurrent builds of one program never read a partial
* file. */
export function installNativeRuntimeData(
selection: NativeRuntimeSelection,
outputPath: string,
): void {
const data = selection.selected.data ?? [];
if (data.length === 0) return;
const outputDir = dirname(resolve(outputPath));
for (const artifact of data) {
const destination = join(outputDir, basename(artifact.path));
const existing = statSync(destination, { throwIfNoEntry: false });
if (existing?.isFile() === true && existing.size === artifact.size) continue;
const temporary = join(
outputDir,
`.${basename(destination)}.scriptc-${process.pid}-${randomUUID()}`,
);
copyFileSync(join(selection.root, artifact.path), temporary);
try {
renameSync(temporary, destination);
} catch {
rmSync(destination, { force: true });
renameSync(temporary, destination);
}
}
}
/** The verification record kept beside the runtime pack, so a build with an
* empty cache does not hash the pack's artifacts again. Like the cache's
* record, it trusts an artifact only while its file keeps the identity
* (device, inode, size, mtime, ctime) the record lists. */
export const PACK_VERIFIED_FILE = "runtime-pack.verified.json";
/** The pack record's well-formed entries; none when it is missing,
* unreadable or malformed. */
function readPackVerified(path: string): Map<string, VerifiedArtifact> {
const verified = new Map<string, VerifiedArtifact>();
try {
for (const artifact of storedArtifacts(readFileSync(path, "utf8")))
verified.set(artifact.path, artifact);
} catch {
/* The artifacts are hashed again. */
}
return verified;
}
/** Adds `artifacts` to the pack record: the record as it is now, with these
* entries replacing the ones for the same paths, written to a new file and
* renamed over the record, so a reader sees a whole record. Of two builds
* writing at once, the last rename wins; the entries the other added are
* hashed again by a later build. When the pack directory cannot be written,
* the cache's record is the only one. */
function writePackVerified(path: string, artifacts: readonly VerifiedArtifact[]): void {
const temporary = `${path}.${process.pid}.${randomBytes(6).toString("hex")}.tmp`;
try {
const record = readPackVerified(path);
for (const artifact of artifacts) record.set(artifact.path, artifact);
const entry: StoredVerification = { schema: 1, artifacts: [...record.values()] };
const bytes = Buffer.from(JSON.stringify(entry));
const fd = openSync(
temporary,
constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL,
0o644,
);
try {
let offset = 0;
while (offset < bytes.length) {
const written = writeSync(fd, bytes, offset, bytes.length - offset);
if (written === 0) throw new Error("the pack record was not written");
offset += written;
}
} finally {
closeSync(fd);
}
renameSync(temporary, path);
} catch {
try {
rmSync(temporary, { force: true });
} catch {
/* Cleanup is best effort. */
}
}
}
@@ -120,6 +120,7 @@ const features = {
dgramDyn: false,
textCodecDyn: false,
watch: false,
watchFile: false,
foreignFfi: false,
nodeTest: false,
tls: false,
@@ -62,6 +62,7 @@ const BASE: NativeLinkFeatures = {
dgramDyn: false,
textCodecDyn: false,
watch: false,
watchFile: false,
foreignFfi: false,
nodeTest: false,
tls: false,
+32 -5
View File
@@ -40,12 +40,19 @@ export interface RuntimePackSelection {
features: RuntimeFeatureSet;
runtimeObjects: string[];
archives: string[];
/** Files the executable loads at run time, installed beside it. */
dataFiles: string[];
/** The ICU headers an Intl selection ships (manifest `headers`), all under
* `<include root>/unicode/`, verified like the other artifacts. */
headerFiles: string[];
systemLibraries: string[];
dependencyPaths: string[];
/** Exact installed inputs observed while the selected artifacts were verified. */
sourceDependencies: NativeArtifactDependency[];
selectedRuntimeArtifacts: RuntimePackArtifact[];
selectedArchiveArtifacts: RuntimePackArtifact[];
selectedDataArtifacts: RuntimePackArtifact[];
selectedHeaderArtifacts: RuntimePackArtifact[];
}
async function verifyArtifact(root: string, artifact: RuntimePackArtifact): Promise<string> {
@@ -209,18 +216,27 @@ export async function loadRuntimePack(options: {
const features = selected.features;
const selectedVariants = selected.runtime;
const selectedArchives = selected.archives;
const selectedArtifactPaths = [...selectedVariants, ...selectedArchives].map((artifact) =>
join(root, artifact.path),
);
const selectedData = selected.data;
// The C-source path compiles scr_intl.c against these headers and links
// the pack's ICU archive (index.ts runtimePackIcuInputs).
const selectedHeaders = features.intl === true ? (manifest.headers ?? []) : [];
const selectedArtifactPaths = [
...selectedVariants,
...selectedArchives,
...selectedData,
...selectedHeaders,
].map((artifact) => join(root, artifact.path));
let artifactDependencies: NativeArtifactDependency[];
try {
artifactDependencies = await snapshotDependencies(selectedArtifactPaths);
} catch {
throw new RuntimePackError("runtime pack artifact set changed during selection", "invalid");
}
const [runtimeObjects, archives] = await Promise.all([
const [runtimeObjects, archives, dataFiles, headerFiles] = await Promise.all([
Promise.all(selectedVariants.map((artifact) => verifyArtifact(root, artifact))),
Promise.all(selectedArchives.map((artifact) => verifyArtifact(root, artifact))),
Promise.all(selectedData.map((artifact) => verifyArtifact(root, artifact))),
Promise.all(selectedHeaders.map((artifact) => verifyArtifact(root, artifact))),
]);
await Promise.all(manifest.licenses.map((license) => readFile(join(root, license.path)))).catch(
() => {
@@ -242,10 +258,21 @@ export async function loadRuntimePack(options: {
features,
runtimeObjects,
archives,
dataFiles,
headerFiles,
systemLibraries: selected.systemLibraries,
dependencyPaths: [packagePath, manifestPath, ...runtimeObjects, ...archives],
dependencyPaths: [
packagePath,
manifestPath,
...runtimeObjects,
...archives,
...dataFiles,
...headerFiles,
],
sourceDependencies,
selectedRuntimeArtifacts: selectedVariants,
selectedArchiveArtifacts: selectedArchives,
selectedDataArtifacts: selectedData,
selectedHeaderArtifacts: selectedHeaders,
};
}
@@ -1,6 +1,6 @@
import { execFile } from "node:child_process";
import { createHash } from "node:crypto";
import { mkdir, mkdtemp, readdir, rm } from "node:fs/promises";
import { mkdir, mkdtemp, readFile, readdir, rename, rm, stat, writeFile } from "node:fs/promises";
import { availableParallelism, tmpdir } from "node:os";
import { basename, join, resolve } from "node:path";
import { promisify } from "node:util";
@@ -11,9 +11,32 @@ import {
validCachedFile,
} from "./build-cache.js";
import type { CcDriver } from "./native/driver.js";
import { NativeCodegenError } from "./native-codegen-core.js";
const execFileAsync = promisify(execFile);
/** One C++ translation unit of the ICU archive: the source, the archive member
* name it compiles to, and the flags of its group. */
interface IcuUnit {
path: string;
member: string;
flags: string[];
define?: string;
}
/** Zig's library directory, which carries the libc++/libc++abi sources the
* ICU archive merges: `ZIG_LIB_DIR` when the caller pins one, else the pinned
* toolchain's own answer. */
async function zigLibDir(): Promise<string> {
const pinned = process.env["ZIG_LIB_DIR"];
if (pinned !== undefined && pinned !== "") return pinned;
const { stdout } = await execFileAsync("zig", ["env"]);
const match = /\.lib_dir\s*=\s*"([^"]+)"/.exec(stdout);
if (match === null)
throw new Error("Intl needs Zig 0.16.0's C++ toolchain: `zig env` named no lib_dir");
return match[1]!;
}
/** The section flags the libregexp/libunicode objects build with (the
* runtime pack's executable_section_elimination flags). */
const LRE_SECTION_FLAGS = ["-ffunction-sections", "-fdata-sections"];
@@ -24,13 +47,33 @@ function vendorObjectName(source: string): string {
return source.replaceAll("/", "-").replace(/\.c$/, ".o");
}
/** The sha256 of a buffer, as lowercase hex. */
function sha256(bytes: Uint8Array): string {
return createHash("sha256").update(bytes).digest("hex");
}
/** Whether `path` is a non-empty file. */
async function presentFile(path: string): Promise<boolean> {
try {
return (await stat(path)).size > 0;
} catch {
return false;
}
}
import {
QJS_COMMIT,
MBEDTLS_VERSION,
ZLIB_VERSION,
BROTLI_VERSION,
ICU_VERSION,
ICU_SOURCE_URL,
ICU_SOURCE_SHA256,
ICU_SOURCE_SIZE,
ICU_CXX_TARGET,
LRE_SOURCES,
ZLIB_SOURCES,
icuSourceList,
zlibInputs,
} from "./vendor-inputs.js";
export {
@@ -38,6 +81,7 @@ export {
MBEDTLS_VERSION,
ZLIB_VERSION,
BROTLI_VERSION,
ICU_VERSION,
LRE_SOURCES,
ZLIB_SOURCES,
} from "./vendor-inputs.js";
@@ -532,9 +576,271 @@ export function createVendorArchives(context: VendorArchiveContext) {
return archive;
}
/** The fetched ICU release root (`icu/`, holding `source/` and LICENSE),
* downloaded once per cache root and reused after that. ICU is not
* vendored: the release's own source tarball is pinned by version and
* sha256 in vendor-inputs.ts, verified on every use, and extracted by the
* system `tar` as the libnode build does. An existing extracted tree is
* enough, so an offline build works from a pre-populated cache. */
const icuSourcesByCacheRoot = new Map<string, Promise<string>>();
function ensureIcuSources(cacheRoot: string = vendorBuildCacheRoot()): Promise<string> {
const pending = icuSourcesByCacheRoot.get(cacheRoot);
if (pending !== undefined) return pending;
const pendingFetch = (async (): Promise<string> => {
const tree = join(cacheRoot, `icu4c-${ICU_VERSION}-source`);
if (await presentFile(join(tree, "source", "common", "sources.txt"))) return tree;
const tarball = join(cacheRoot, `icu4c-${ICU_VERSION}-sources.tgz`);
let archive: Buffer;
try {
archive = await readFile(tarball);
if (archive.length !== ICU_SOURCE_SIZE || sha256(archive) !== ICU_SOURCE_SHA256)
throw new Error("cached ICU source tarball does not match its pin");
} catch {
let response: Response;
try {
response = await fetch(ICU_SOURCE_URL);
} catch (cause) {
throw new Error(
`could not download ICU's sources (${ICU_SOURCE_URL}); ` +
`populate ${tarball} or allow network access`,
{ cause },
);
}
if (!response.ok)
throw new Error(
`could not download ICU's sources (${ICU_SOURCE_URL}): ` +
`${response.status} ${response.statusText}`,
);
archive = Buffer.from(await response.arrayBuffer());
if (archive.length !== ICU_SOURCE_SIZE || sha256(archive) !== ICU_SOURCE_SHA256)
throw new Error(
`ICU source tarball from ${ICU_SOURCE_URL} is ${archive.length} bytes with sha256 ` +
`${sha256(archive)}, expected ${ICU_SOURCE_SIZE} bytes with sha256 ` +
`${ICU_SOURCE_SHA256}`,
);
await mkdir(cacheRoot, { recursive: true });
await writeFile(tarball, archive);
}
await mkdir(cacheRoot, { recursive: true });
const stage = await mkdtemp(join(cacheRoot, ".icu-sources-"));
try {
await execFileAsync("tar", ["-xzf", tarball, "-C", stage]);
if (!(await presentFile(join(stage, "icu", "source", "common", "sources.txt"))))
throw new Error(`ICU source tarball ${ICU_SOURCE_URL} does not hold icu/source`);
await rm(tree, { recursive: true, force: true });
await rename(join(stage, "icu"), tree);
} finally {
await rm(stage, { recursive: true, force: true });
}
return tree;
})();
icuSourcesByCacheRoot.set(cacheRoot, pendingFetch);
pendingFetch.catch(() => icuSourcesByCacheRoot.delete(cacheRoot));
return pendingFetch;
}
/** The archive is the same for sanitized and plain builds: it is never
* instrumented (see ensureIcuArchive), so neither the path nor the build
* carries a sanitizer flavor. */
function icuArchivePath(
driver: CcDriver,
buildIdentity: string,
cacheRoot: string = vendorBuildCacheRoot(),
): string {
const flavor = `plain-${vendorCacheTargetFlavor(driver)}-${buildIdentity}`;
return join(cacheRoot, `icu-${ICU_VERSION}-${flavor}`, "libicu.a");
}
/** The ICU archive for one flavor, compiled lazily on the first
* Intl-using build (~470 C++ TUs, parallelized) and cached under
* <build-cache>/vendor/icu-<version>-<flavor>-<target>-<toolchain>/
* libicu.a with the mbedTLS archive's atomic-rename publish.
*
* Three things make this archive different from every other vendored one:
*
* - It is C++, so the pack's C driver cannot compile it: the TUs (and the
* standard library they use) build with Zig's C++ driver for the glibc
* target the runtime packs pin. That target is the point. In C++ mode,
* glibc 2.38+ headers redirect strtol/strtoul to the C23 __isoc23_*
* names, and a host libstdc++ binds those names too, so a host-compiled
* C++ archive would raise the glibc floor above the packs'. Zig's
* bundled libc++ and libc++abi compile from the toolchain's own sources
* for the target and bind the target's names only.
* - ICU requires RTTI (`typeid` in unifiedcache.h and ustrenum.cpp,
* `dynamic_cast` in uniset_props.cpp), so the design note's
* `-fno-rtti` is not available; `-fno-exceptions` is, and ICU, libc++,
* and libc++abi all build so. libc++abi's Itanium personality routine
* (cxa_personality.cpp) is the one unit that needs exceptions, so it is
* left out: nothing can throw into it.
* - Both the standard library and the C++ runtime it needs are merged INTO
* the archive, so the final link stays C-only: a plain `clang`/`zig` C
* link with the runtime pack's `-lm` resolves every undefined symbol.
* Members are prefixed by source (`cxx-libcxx-`, `cxx-libcxxabi-`) so
* their names stay unique.
*
* ICU builds with symbol renaming on (version suffixes `_78`), which is
* what lets the zero-cost gate assert that only Intl programs link ICU
* symbols.
*
* Sanitized builds link this same uninstrumented archive. AddressSanitizer
* cannot instrument it: Zig's bundled libc++ compiles with
* `-fsanitize=address` only against a `sanitizer/asan_interface.h` its
* tree does not ship, and compiling ~600 C++ TUs again for the sanitizer
* lane would add minutes to the first Intl program of every sanitized
* build. The runtime's own ICU code (scr_intl.c) still compiles with the
* sanitizer, and AddressSanitizer's interceptors cover ICU's allocations,
* so a sanitized Intl program reports heap errors from ICU data as it
* does everywhere else — the ICU code itself is simply not instrumented. */
async function ensureIcuArchive(
driver: CcDriver,
buildIdentity: string,
cacheRoot: string = vendorBuildCacheRoot(),
): Promise<string> {
// The archive is host-targeted ELF for the packs' glibc floor; a cross
// target would need a Zig C++ link recipe this slice has not validated.
// Both refusals are compile diagnostics (SC3002), not internal errors:
// an Intl program on any other host or target is a documented "not
// here yet", the same answer the missing runtime pack gives.
if (driver.target !== null)
throw new NativeCodegenError(
"SC3002",
`Intl needs ICU, which scriptc builds for the host's linux-x64 gnu target only: ` +
`the ${driver.target} target is not supported yet`,
);
if (process.platform !== "linux" || process.arch !== "x64")
throw new NativeCodegenError(
"SC3002",
"Intl needs ICU, which this compiler builds for linux-x64 only: " +
`the ${process.platform}-${process.arch} host is not supported yet`,
);
const flavor = `plain-${vendorCacheTargetFlavor(driver)}-${buildIdentity}`;
const archive = icuArchivePath(driver, buildIdentity, cacheRoot);
if (await validVendorArtifact(archive)) return archive;
await mkdir(cacheRoot, { recursive: true });
const buildDir = await mkdtemp(join(tmpdir(), `scriptc-vendor-icu-${flavor}-`));
// The tarball extracts to `icu/source/...` with ICU's LICENSE beside it.
const vendor = join(await ensureIcuSources(cacheRoot), "source");
const libDir = await zigLibDir();
const cxx = ["zig", "c++", "-target", ICU_CXX_TARGET];
const optimize = ["-Os"];
const commonFlags = [
"-std=c++17",
...optimize,
"-fno-exceptions",
"-ffunction-sections",
"-fdata-sections",
];
const icuFlags = [
...commonFlags,
"-DU_STATIC_IMPLEMENTATION",
"-DU_ATTRIBUTE_DEPRECATED=",
"-DNDEBUG",
"-Wno-deprecated-declarations",
"-I",
join(vendor, "common"),
"-I",
join(vendor, "i18n"),
];
// libc++ 20 sources need C++20 (`constinit`, ranges) and libc++'s
// charconv reads a header from `libcxx/libc`.
const cxxLibraryFlags = [
"-std=c++23",
...optimize,
"-fno-exceptions",
"-ffunction-sections",
"-fdata-sections",
"-I",
join(libDir, "libcxx", "include"),
"-I",
join(libDir, "libcxx", "src"),
"-I",
join(libDir, "libcxx", "libc"),
"-I",
join(libDir, "libcxxabi", "include"),
"-D_LIBCPP_BUILDING_LIBRARY",
"-D_LIBCXXABI_BUILDING_LIBRARY",
"-DLIBCXX_BUILDING_LIBCXXABI",
"-Wno-unknown-warning-option",
];
const cxxLibraryUnits = async (
subdirectory: string,
memberPrefix: string,
skip: ReadonlySet<string>,
): Promise<IcuUnit[]> => {
const sources = (await readdir(join(libDir, subdirectory, "src")))
.filter((name) => name.endsWith(".cpp") && !skip.has(name))
.sort();
return sources.map((source) => ({
path: join(libDir, subdirectory, "src", source),
member: `${memberPrefix}${source}`,
flags: cxxLibraryFlags,
}));
};
try {
const units: IcuUnit[] = [
...icuSourceList(await readFile(join(vendor, "common", "sources.txt"), "utf8")).map(
(source) => ({
path: join(vendor, "common", source),
member: `common-${source}`,
flags: icuFlags,
define: "U_COMMON_IMPLEMENTATION",
}),
),
{
path: join(vendor, "stubdata", "stubdata.cpp"),
member: "stubdata-stubdata.cpp",
flags: icuFlags,
define: "U_COMMON_IMPLEMENTATION",
},
...icuSourceList(await readFile(join(vendor, "i18n", "sources.txt"), "utf8")).map(
(source) => ({
path: join(vendor, "i18n", source),
member: `i18n-${source}`,
flags: icuFlags,
define: "U_I18N_IMPLEMENTATION",
}),
),
...(await cxxLibraryUnits("libcxx", "cxx-libcxx-", new Set())),
...(await cxxLibraryUnits("libcxxabi", "cxx-libcxxabi-", new Set(["cxa_personality.cpp"]))),
];
const width = availableParallelism();
for (let i = 0; i < units.length; i += width) {
await Promise.all(
units.slice(i, i + width).map(async (unit) => {
const output = join(buildDir, unit.member.replace(/\.cpp$/, ".o"));
await execFileAsync(cxx[0]!, [
...cxx.slice(1),
...unit.flags,
...(unit.define === undefined ? [] : [`-D${unit.define}`]),
"-g0",
"-c",
unit.path,
"-o",
output,
]);
return output;
}),
);
}
await execFileAsync("ar", [
"rcs",
join(buildDir, "libicu.a"),
...units.map((unit) => join(buildDir, unit.member.replace(/\.cpp$/, ".o"))),
]);
await publishVendorArtifact(join(buildDir, "libicu.a"), archive);
} finally {
await rm(buildDir, { recursive: true, force: true });
}
return archive;
}
return {
vendorEngineDir,
vendorTlsDir,
ensureIcuSources,
icuArchivePath,
ensureIcuArchive,
vendorBrotliDir,
brotliArchivePath,
ensureBrotliArchive,
@@ -2,7 +2,21 @@ import { existsSync } from "node:fs";
import { join } from "node:path";
import { fileURLToPath, pathToFileURL } from "node:url";
import { expect, test } from "vitest";
import { ZLIB_SOURCES, zlibInputs } from "./vendor-inputs.js";
import {
ZLIB_SOURCES,
ICU_VERSION,
ICU_SOURCE_URL,
ICU_SOURCE_SHA256,
ICU_SOURCE_SIZE,
ICU_DATA_URL,
ICU_DATA_FILE,
ICU_DATA_SHA256,
ICU_DATA_SIZE,
ICU_DATA_ZIP_SHA256,
ICU_DATA_ZIP_SIZE,
icuSourceList,
zlibInputs,
} from "./vendor-inputs.js";
const repoRoot = fileURLToPath(new URL("../../../..", import.meta.url));
const zlibDir = join(repoRoot, "packages/runtime/vendor/zlib");
@@ -78,3 +92,39 @@ test("zlib inputs follow Node's deps/zlib/zlib.gyp", () => {
expect(wasm.sources).not.toContain("contrib/optimizations/inflate.c");
expect(wasm.defines).toEqual(["HAVE_HIDDEN", "CPU_NO_SIMD"]);
});
test("the pinned ICU is the release the oracle Node bundles", () => {
// Node's own Intl answers from the ICU it was built with, so the fetched
// release must be that version exactly: the differential corpus compares
// scriptc's ICU output against Node's, byte for byte.
expect(process.versions.icu).toBe(ICU_VERSION);
});
test("the compiler's ICU pins match the runtime packs'", async () => {
const pack = (await import(
pathToFileURL(join(repoRoot, "packages/runtime-pack-common/scripts/icu-inputs.mjs")).href
)) as Record<string, string | number>;
expect(pack["ICU_VERSION"]).toBe(ICU_VERSION);
expect(pack["ICU_SOURCE_URL"]).toBe(ICU_SOURCE_URL);
expect(pack["ICU_SOURCE_SHA256"]).toBe(ICU_SOURCE_SHA256);
expect(pack["ICU_SOURCE_SIZE"]).toBe(ICU_SOURCE_SIZE);
expect(pack["ICU_DATA_URL"]).toBe(ICU_DATA_URL);
expect(pack["ICU_DATA_FILE"]).toBe(ICU_DATA_FILE);
expect(pack["ICU_DATA_SHA256"]).toBe(ICU_DATA_SHA256);
expect(pack["ICU_DATA_SIZE"]).toBe(ICU_DATA_SIZE);
expect(pack["ICU_DATA_ZIP_SHA256"]).toBe(ICU_DATA_ZIP_SHA256);
expect(pack["ICU_DATA_ZIP_SIZE"]).toBe(ICU_DATA_ZIP_SIZE);
// Both pins name the official release assets for the pinned version.
expect(ICU_SOURCE_URL).toBe(
`https://github.com/unicode-org/icu/releases/download/release-${ICU_VERSION}/icu4c-${ICU_VERSION}-sources.tgz`,
);
expect(ICU_DATA_URL).toBe(
`https://github.com/unicode-org/icu/releases/download/release-${ICU_VERSION}/icu4c-${ICU_VERSION}-data-bin-l.zip`,
);
expect(ICU_SOURCE_SHA256).toMatch(/^[0-9a-f]{64}$/);
expect(ICU_DATA_SHA256).toMatch(/^[0-9a-f]{64}$/);
});
test("ICU source lists parse to the units a build compiles", () => {
expect(icuSourceList("# a comment\n\n b.cpp \nc.cpp\n")).toEqual(["b.cpp", "c.cpp"]);
});
@@ -6,6 +6,48 @@ export const MBEDTLS_VERSION = "3.6.7";
export const ZLIB_VERSION = "1.3.2.1-motley-285e94b";
/** The brotli release Node v24.22.0 bundles (packages/runtime/vendor/brotli). */
export const BROTLI_VERSION = "1.2.0";
/** ICU4C release 78.3 — the exact version Node v24.22.0 bundles
* (`process.versions.icu` there), from the `release-78.3` tag. ICU is not
* vendored: the build fetches the release's own source tarball and data zip
* from unicode-org/icu's GitHub release, pinning each by version and sha256,
* and caches the download. Its data file icudt78l.dat (33,107,232 B) ships
* beside the executable; see packages/runtime-pack-common/scripts/
* icu-data.mjs. */
export const ICU_VERSION = "78.3";
const ICU_RELEASE_URL = `https://github.com/unicode-org/icu/releases/download/release-${ICU_VERSION}`;
/** The release's source tarball (icu, not icu4j), the only build input that
* is not vendored. */
export const ICU_SOURCE_URL = `${ICU_RELEASE_URL}/icu4c-${ICU_VERSION}-sources.tgz`;
export const ICU_SOURCE_SHA256 = "3a2e7a47604ba702f345878308e6fefeca612ee895cf4a5f222e7955fabfe0c0";
export const ICU_SOURCE_SIZE = 27_977_255;
/** The release's prebuilt little-endian data zip. 969 source files are
* reviewable; 33 MiB of compiled CLDR tables are not, so the data is fetched
* too, pinned the same way. */
export const ICU_DATA_URL = `${ICU_RELEASE_URL}/icu4c-${ICU_VERSION}-data-bin-l.zip`;
export const ICU_DATA_ZIP_SHA256 =
"982619632b78887f1895b063e96e8c3cc7f99283337c8abbd05aa71635de613c";
export const ICU_DATA_ZIP_SIZE = 12_836_659;
export const ICU_DATA_FILE = "icudt78l.dat";
export const ICU_DATA_SHA256 = "d5cf2a40dccbe471781ec7af85693bff542ff12f0b670c9630c4e72d60714b8b";
export const ICU_DATA_SIZE = 33_107_232;
/** The glibc target the ICU archive compiles for: the floor the linux-x64-gnu
* runtime pack builds against (packages/runtime-pack-common/scripts/
* glibc-toolchain.mjs GLIBC_RUNTIME_FLOOR), spelled for Zig's C++ driver.
* ICU is the one C++ archive, and C++ compiled against a newer host glibc
* binds the C23 strtol names (__isoc23_*) that floor forbids. */
export const ICU_CXX_TARGET = "x86_64-linux-gnu.2.34";
/** The translation units one of ICU's own source lists names, in its order.
* The fetched tarball carries the lists (`source/common/sources.txt`,
* `source/i18n/sources.txt`) verbatim, so the units a build compiles are
* exactly the units ICU's own autoconf build compiles for this release.
* Entries are bare file names. The pack build reads the same files
* (packages/runtime-pack-common/scripts/icu-data.mjs). */
export function icuSourceList(text: string): string[] {
return text
.split("\n")
.map((line) => line.trim())
.filter((line) => line.length > 0 && !line.startsWith("#"));
}
/** Exact translation-unit membership shared by cache builds and external
* source-pack recipes. */
export const LRE_SOURCES = ["libregexp.c", "libunicode.c"] as const;
@@ -63,6 +63,7 @@ export async function warmNativeCaches(
dgramDyn: false,
textCodecDyn: false,
watch: false,
watchFile: false,
foreignFfi: false,
nodeTest: false,
tls: false,
+28 -23
View File
@@ -155,6 +155,12 @@ const typedInterfaceUnsupported = "typed source has no compiler bridge for this
const metadataExclusion =
"WebIDL brand metadata is observable reflection, not an executable compatibility operation";
/** ReadableStream methods a typed program reaches through the web stream
* classes (scr_webstreams.c): on a stream it constructs with them, and on
* the native stream through its web view (scr_fetch.c, sf_stream_web_view).
* A program that calls one links the classes. */
export const READABLE_STREAM_WEB_VIEW_CALLS = ["pipeThrough", "pipeTo", "tee"] as const;
function buildFetchCompatProfile() {
return {
schemaVersion: 1,
@@ -233,7 +239,7 @@ function buildFetchCompatProfile() {
],
responseCalls: ["json", "text", "bytes", "clone"],
readableStreamReads: ["locked"],
readableStreamCalls: ["cancel", "getReader"],
readableStreamCalls: ["cancel", "getReader", ...READABLE_STREAM_WEB_VIEW_CALLS],
},
operations: [
{
@@ -447,6 +453,13 @@ function buildFetchCompatProfile() {
facets: ["webidl-conversion", "identity", "state-machine", "error-shape"],
evidence: [generated("stream-traces"), fixture("static-stream")],
},
...READABLE_STREAM_WEB_VIEW_CALLS.map((member): FetchCompatOperation => ({
id: `stdlib.readable-stream.${member}`,
name: `ReadableStream.${member}`,
kind: "method",
facets: ["identity", "promise-settlement", "state-machine", "error-shape"],
evidence: [fixture("static-web-streams")],
})),
{
id: "stdlib.readable-stream-default-reader.closed",
name: "ReadableStreamDefaultReader.closed",
@@ -757,25 +770,15 @@ function buildFetchCompatProfile() {
"getReader",
"prototype",
),
...["pipeThrough", "values"].map((member) =>
unsupportedEntry(
`stdlib.readable-stream.${member}`,
"ReadableStream",
member,
"prototype",
member === "values"
? "direct for-await consumption compiles statically; storing or driving the first-class iterator handle is unsupported"
: "the wider Web Streams graph is outside the native readable-stream slice",
),
...READABLE_STREAM_WEB_VIEW_CALLS.map((member) =>
staticEntry(`stdlib.readable-stream.${member}`, "ReadableStream", member, "prototype"),
),
...["pipeTo", "tee"].map((member) =>
unsupportedEntry(
`stdlib.readable-stream.${member}`,
"ReadableStream",
member,
"prototype",
"this operation is outside the native readable-stream implementation",
),
unsupportedEntry(
"stdlib.readable-stream.values",
"ReadableStream",
"values",
"prototype",
"direct for-await consumption compiles statically; storing or driving the first-class iterator handle is unsupported",
),
unsupportedEntry(
"stdlib.readable-stream.symbol.asyncIterator",
@@ -954,7 +957,8 @@ export interface StaticFetchMembers {
readableStreamReads: ReadonlySet<string>;
readableStreamCalls: ReadonlySet<string>;
/** ReadableStream prototype members the native stream does not
* implement, symbol-keyed ones as `[Symbol.name]`. */
* implement itself, symbol-keyed ones as `[Symbol.name]`: the unsupported
* ones, and the calls it answers through its web view. */
readableStreamUnsupported: ReadonlySet<string>;
}
@@ -979,8 +983,8 @@ export function staticFetchMembers(): StaticFetchMembers {
),
readableStreamReads: new Set(profile.members.readableStreamReads),
readableStreamCalls: new Set(profile.members.readableStreamCalls),
readableStreamUnsupported: new Set(
profile.inventory.entries
readableStreamUnsupported: new Set([
...profile.inventory.entries
.filter(
(entry) =>
entry.owner === "ReadableStream" &&
@@ -988,7 +992,8 @@ export function staticFetchMembers(): StaticFetchMembers {
entry.status === "unsupported",
)
.map((entry) => entry.member),
),
...READABLE_STREAM_WEB_VIEW_CALLS,
]),
};
return staticFetchMemberSets;
}
@@ -98,7 +98,7 @@ function coverageNotes(): string[] {
"Entries with status 'unsupported' describe where the named code is raised: forms of the construct outside the supported subset are refused with that code — not that every form of the named feature is refused. Supported forms appear as their own static entries where a table projects them.",
`The engine-free fetch projection targets Node ${fetchTarget.node} with bundled Undici ${fetchTarget.undici}. Each projected row names the differential evidence that guards it; changing the pinned Node or Undici version is an explicit profile update.`,
"The fetch profile also contains a runtime-reflected census of the selected fetch, abort, Headers, and readable-stream interfaces plus RequestInit/ResponseInit dictionary reads. Static and unsupported census rows are projected here; its explicitly out-of-scope metadata rows and adjacent-interface exclusions remain in the profile so absence is deliberate rather than ambiguous.",
"Process-level diagnostic codes are not surface entries: SC0001-SC0004 are preflight gates, SC1110 is a comptime evaluation failure, SC3001/SC3002 are backend/target tier refusals, SC9001/SC9002 are internal errors.",
"Process-level diagnostic codes are not surface entries: SC0001-SC0007 are preflight gates (SC0005-SC0007 are the ECMAScript early errors), SC1110 is a comptime evaluation failure, SC3001/SC3002 are backend/target tier refusals, SC9001/SC9002 are internal errors.",
"Entry statuses are projected for the desktop targets. The mobile targets (aarch64-apple-ios, aarch64-apple-ios-simulator, aarch64-linux-android) compile library-mode archives only: the library-admissible surface (what SC4005's async_free requirement and the library link set admit) is supported there, the executable lane refuses those triples with SC3002, and no entry outside the library-admissible surface carries a mobile support claim. iOS archives build for iOS 15.0 on darwin hosts; Android archives build against NDK API level 26.",
"No scheduling metadata is published; entry ids are the stable diff keys across releases.",
];
@@ -19,6 +19,9 @@ export const DIAGNOSTIC_CODES = {
SC0002: "project configuration incompatible with scriptc",
SC0003: "malformed imported JSON module",
SC0004: "TypeScript checker panic",
SC0005: "duplicate lexical declaration in a block or switch statement list",
SC0006: "lexical declaration colliding with a var in the same block",
SC0007: "declaration in a single-statement position",
SC1010: "package imports outside the supported set",
SC1012: "default exports/imports (residual edges)",
SC1013: "namespace imports (residual edges)",
@@ -40,6 +43,7 @@ export const DIAGNOSTIC_CODES = {
SC1071: "generators",
SC1080: "'this' outside a class method",
SC1090: "this syntax (generic construct fence)",
SC1091: "write to an undeclared name in a JavaScript source",
SC1100: "operations on 'unknown' values",
SC1101: "converting typed values to 'unknown'",
SC1110: "comptime evaluation failure",
@@ -7,8 +7,12 @@
* Code bands:
* SC0xxx preflight gates: TypeScript errors passed through (SC0001),
* project-config incompatibilities (SC0002), malformed imported
* JSON modules (SC0003), and upstream tsgo checker panics
* surfaced as source-anchored diagnostics (SC0004)
* JSON modules (SC0003), upstream tsgo checker panics surfaced
* as source-anchored diagnostics (SC0004), and the ECMAScript
* early errors the checker does not model: a duplicated or
* `var`-colliding lexical declaration in a block or switch
* statement list (SC0005/SC0006) and a declaration where the
* grammar admits only a Statement (SC0007)
* SC1xxx supported-TypeScript-not-yet: valid TS outside the current subset
* SC2xxx scriptc type rules (types we cannot compile yet)
* SC3xxx backend/target coverage: the program is valid, but the selected
@@ -366,6 +370,24 @@ export function sharedModuleDuplicatedDiag(
};
}
/** SC8020 — the program replaces a process output stream's write method
* while a Node island exists: the island's `process.stdout` is its own
* object, so the replacement and the island's writes cannot see each other.
* H2 shares the native stream object with the islands and removes this. */
export function stdioPatchAcrossIslandsDiag(
stream: "stdout" | "stderr" | null,
loc: SrcLoc,
): ScrDiagnostic {
const what =
stream === null ? "process.stdout.write or process.stderr.write" : `process.${stream}.write`;
return {
code: "SC8020",
message: `${what} is replaced while a Node island exists: the island's process output stream would not see this patch, and island writes would bypass it`,
loc,
hint: "the island's process.stdout would not see this patch",
};
}
/** SC8022 — island code extends a class of a module the program compiles.
* A native instance cannot take an island subclass's overrides, so the
* module is not owned by the program: a package runs in Node for both
@@ -687,6 +709,11 @@ export const UNSUPPORTED: Record<string, UnsupportedEntry> = {
// remains is `this` outside any method (top level, plain functions).
SC1080: { feature: "'this' outside a class method", milestone: "later" },
SC1090: { feature: "this syntax", milestone: "later" }, // generic fallback; message names the construct
SC1091: {
feature: "writing an undeclared name in a JavaScript source",
milestone: "later",
hint: "a write to an undeclared name has no native lowering: sloppy mode makes it a property on Node's global object, which a native binary has none of, and strict mode makes it throw; declare the name, or keep the module in a Node island (sloppy-mode programs route there)",
},
SC1100: {
feature: "operations on 'unknown' values",
milestone: "later",
@@ -60,6 +60,7 @@ const native: EarlyExecutableNativeFeatures = {
dgramDyn: false,
textCodecDyn: false,
watch: false,
watchFile: false,
foreignFfi: false,
nodeTest: false,
tls: false,
@@ -65,9 +65,14 @@ export interface EarlyExecutableNativeFeatures {
wasm?: boolean;
warningReplace?: boolean;
stdioExtended?: boolean;
/** Error-taking write completion callbacks: SCR_STREAM_WRITE_ERRORS. */
streamWriteErrors?: boolean;
/** The process output write slot: SCR_STDIO_WRITE_SLOT. */
stdioWriteSlot?: boolean;
nodeIsland?: boolean;
moduleNamespace?: boolean;
moduleNamespaceDyn?: boolean;
primitiveWrapper?: boolean;
zlib: boolean;
brotli?: boolean;
assert: boolean;
@@ -111,6 +116,7 @@ export interface EarlyExecutableNativeFeatures {
dgramDyn: boolean;
textCodecDyn: boolean;
watch: boolean;
watchFile: boolean;
foreignFfi: boolean;
nodeTest: boolean;
tls: boolean;
@@ -123,6 +129,8 @@ export interface EarlyExecutableNativeFeatures {
perfLoop: boolean;
tlsCa: boolean;
webcrypto: boolean;
/** Intl over ICU (scr_intl.c, the ICU archive and its data file). */
intl?: boolean;
}
interface EarlyExecutableCacheStamp {
@@ -299,6 +307,7 @@ const BOOLEAN_NATIVE_KEYS = [
"dgramDyn",
"textCodecDyn",
"watch",
"watchFile",
"foreignFfi",
"nodeTest",
"tls",
+18 -1
View File
@@ -8,6 +8,7 @@ import {
npmStaticRefusalDiag,
iceDiag,
isCheckerPanic,
stdioPatchAcrossIslandsDiag,
type ScrDiagnostic,
} from "../diagnostics/diagnostic.js";
import type { IrModule, SrcLoc } from "../ir/ir.js";
@@ -20,6 +21,7 @@ import {
} from "../backend/target-diagnostics.js";
import type { LowerResult, LowerStats } from "../frontend/lowering/lowerer.js";
import type { FrontendFactory } from "../frontend/pipeline.js";
import type { UnitStore } from "../frontend/lowering/unit-reuse.js";
import type { CompilationTiming } from "../timing.js";
import { readProjectConfig } from "../project-config.js";
import { nodeIslandPolicy } from "../node-islands/policy.js";
@@ -42,7 +44,9 @@ export interface PreparedExecutableModule {
warnings: ScrDiagnostic[];
}
/** Release the parser/checker before native object generation begins. */
/** Release the parser/checker before native object generation begins.
* `unitStore` keeps the lowered units of this build for the next one
* (unit-reuse.ts). */
export function prepareExecutableModule(
entryPath: string,
opts: CompileRequestOptions,
@@ -52,6 +56,7 @@ export function prepareExecutableModule(
timing: CompilationTiming = () => {},
lowerIslands: ReadonlyMap<string, IslandPackage> = new Map(),
retryFences = true,
unitStore: UnitStore | null = null,
): PreparedExecutableModule | CompileFailure {
// The island decision reads the detector's scan unless islands are off.
const islandScan =
@@ -115,6 +120,7 @@ export function prepareExecutableModule(
timing,
grown,
retryFences,
unitStore,
);
}
return fail(fe.preflight);
@@ -127,6 +133,7 @@ export function prepareExecutableModule(
targetPlatform: buildPlatform,
...(ffi !== null ? { ffiImports: ffi.functions } : {}),
...(opts.tier !== undefined ? { tierMode: opts.tier } : {}),
...(unitStore !== null ? { unitStore } : {}),
},
(module) => validateModule(module).length === 0,
);
@@ -165,6 +172,7 @@ export function prepareExecutableModule(
timing,
grown,
retryFences,
unitStore,
);
}
// A failed program whose compiled packages need the runtime or a
@@ -195,6 +203,14 @@ export function prepareExecutableModule(
return fail(lowered.diagnostics);
}
// A program that replaces a process output stream's write method has
// nowhere to route an island's writes: the island's process.stdout is
// its own object. Until islands share the native stream object (H2),
// refuse the combination instead of silently missing the patch.
if (islandDecision !== null && lowered.stdioPatches !== undefined) {
return fail(lowered.stdioPatches.map((p) => stdioPatchAcrossIslandsDiag(p.stream, p.loc)));
}
const validation = validateModule(lowered.module);
timing("ir-validate");
if (validation.length > 0) {
@@ -267,6 +283,7 @@ export function prepareExecutableModule(
timing,
islands,
false,
unitStore,
);
if (islanded.ok) return islanded;
firstError ??= islanded.diagnostics[0] ?? null;
@@ -0,0 +1,43 @@
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { expect, test } from "vitest";
import { checkPreflight, loadProgram } from "./program-node.js";
/** The preflight diagnostics of a CommonJS script (sloppy unless it says
* "use strict"). */
function diagnostics(source: string) {
const directory = mkdtempSync(join(tmpdir(), "scriptc-early-errors-"));
writeFileSync(join(directory, "main.cjs"), source);
const load = loadProgram(join(directory, "main.cjs"));
try {
return checkPreflight(load).map((diagnostic) => diagnostic.code);
} finally {
load.dispose();
rmSync(directory, { recursive: true, force: true });
}
}
test.for([
// A function body's top-level function declarations are var-scoped, so
// duplicates are legal in strict code too.
'function outer() { "use strict"; function d() {} function d() {} }\nouter();',
"class C { static { function s() {} function s() {} } }\nnew C();",
// Object literal methods and accessors are sloppy unless they say otherwise.
"var o = { m() { if (true) function f() {} } };\no.m();",
"var o = { get g() { { function h() {} function h() {} } return 1; } };\no.g;",
// A function's own directive makes its body strict, not the code declaring it.
'if (true) function k() { "use strict"; }',
'{ function z() { "use strict"; } function z() {} }',
])("Node accepts: %s", (source) => {
expect(diagnostics(source)).toEqual([]);
});
test.for([
["function outer() { let d; function d() {} }\nouter();", "SC0006"],
['(() => { "use strict"; if (true) function f() {} })();', "SC0007"],
["class C { m() { if (true) function f() {} } }\nnew C().m();", "SC0007"],
['var o = { m() { "use strict"; { function h() {} function h() {} } } };\no.m();', "SC0005"],
])("Node refuses: %s", ([source, code]) => {
expect(diagnostics(source!)).toContain(code);
});
@@ -0,0 +1,384 @@
/* ECMAScript early errors the TypeScript checker does not model.
*
* Static semantics reject a program before it runs, and until this pass
* existed scriptc borrowed that rejection from whatever tsc happened to
* report (`Let x` declarations at an odd position, a redeclared block name
* TypeScript refuses). Test262's negative-parse suites show where the
* borrowing stops: 1,198 programs the spec makes SyntaxErrors compile — the
* constructs below are the ones the corpus covers, in descending variant
* count (plan §7.7 D-4 option (c)).
*
* One code per construct, so C1's checker relaxation stays safe: relaxing a
* tsc code that doubles as an early error removes the ONLY rejection some
* parse-negative test had, which is why these codes live beside
* JS_RELAXED_TSC_CODES in tsc-codes.ts and never depend on it.
*
* Scope: JavaScript sources only. A TypeScript source is type-checked, and
* its checker diagnostics (TS2451 and friends) are the gate; re-reporting
* the same collision under a second code would only duplicate them.
*
* The pass runs at preflight, before any lowering, and reports through
* SC0xxx — the preflight band — so a rejected program fails the same way a
* tsc error does. */
import * as ts from "./ts7/adapter.js";
import type { ScrDiagnostic } from "../diagnostics/diagnostic.js";
import type { SrcLoc } from "../ir/ir.js";
import { isJsSourceFileName } from "./tsc-codes.js";
/** The early-error codes. SC0005 and SC0006 are the Block/CaseBlock static
* semantics (a statement list may not declare one lexical name twice, and a
* lexical name may not be `var`-declared in the same list); SC0007 is a
* declaration where the grammar admits only a Statement. */
export const EARLY_ERROR_CODES = ["SC0005", "SC0006", "SC0007"] as const;
const NO_DUPLICATE_LEXICAL = "SC0005";
const LEXICAL_VAR_COLLISION = "SC0006";
const NOT_A_STATEMENT = "SC0007";
/** Whether the code around a node is strict — the one input this pass needs
* from outside: Node's own module classification (program.ts's
* isNodeEsmFile7: fixed extensions, package type, then syntax markers). The
* checker's external-module bit cannot answer it, because
* moduleDetection=force gives every file module scope, CommonJS included. */
export type IsStrictCode = (sf: ts.SourceFile, node: ts.Node) => boolean;
export function earlyErrorDiagnostics7(
files: readonly ts.SourceFile[],
isModuleCode: (sf: ts.SourceFile) => boolean,
): ScrDiagnostic[] {
const diags: ScrDiagnostic[] = [];
const isStrict: IsStrictCode = (sf, node) => isStrictCode(sf, node, isModuleCode);
for (const sf of files) {
if (!isJsSourceFileName(sf.fileName)) continue;
// Iterative walk: a deeply nested program must not overflow the stack
// before the lowering's own nesting fence runs.
const stack: ts.Node[] = [sf];
while (stack.length > 0) {
const node = stack.pop()!;
checkStatementListScope(sf, node, diags, isStrict);
checkStatementSlot(node, diags, isStrict);
ts.forEachChild(node, (child) => {
stack.push(child);
return undefined;
});
}
}
return diags;
}
function locOf(node: ts.Node): SrcLoc {
const sf = ts.getSourceFileOfNode(node);
return { file: sf.fileName, start: node.getStart(sf), end: node.end };
}
function report(diags: ScrDiagnostic[], code: EarlyErrorCode, node: ts.Node, message: string) {
diags.push({ code, message, loc: locOf(node) });
}
type EarlyErrorCode = (typeof EARLY_ERROR_CODES)[number];
/* ── SC0005 / SC0006: Block and CaseBlock static semantics ───────────────
*
* Block : { StatementList }
* CaseBlock : { CaseClauses_opt DefaultClause CaseClauses_opt }
*
* * It is a Syntax Error if the LexicallyDeclaredNames of StatementList
* contains any duplicate entries.
* * It is a Syntax Error if any element of the LexicallyDeclaredNames of
* StatementList also occurs in the VarDeclaredNames of StatementList.
*
* A case block is ONE lexical scope across its clauses, and VarDeclaredNames
* reach into nested blocks (a `var` anywhere below the block belongs to it)
* but never into a function or class body. The duplicate rule's one carve-out
* is Annex B's B.3.3.1: two plain function declarations of the same name in
* the same block are legal in non-strict code.
*
* A function body and a class static block have the same two rules over
* their top-level declarations (FunctionStatementList and
* ClassStaticBlockStatementList), but there a function declaration is
* var-scoped: it is one of the VarDeclaredNames, so two of them are legal in
* strict code too, while `let f; function f() {}` still collides. */
function checkStatementListScope(
sf: ts.SourceFile,
node: ts.Node,
diags: ScrDiagnostic[],
isStrict: IsStrictCode,
): void {
let statements: readonly ts.Statement[] | null = null;
if (ts.isBlock(node)) statements = node.statements.nodes;
else if (ts.isCaseBlock(node)) {
statements = node.clauses.nodes.flatMap((clause) => clause.statements.nodes);
}
if (statements === null) return;
if (statements.length < 2) return;
const functionBody = isFunctionBody(node);
const lexical = new Map<string, ts.Node[]>();
for (const stmt of statements) {
for (const [name, decl] of lexicalDeclarationsOf(stmt, functionBody)) {
const list = lexical.get(name);
if (list === undefined) lexical.set(name, [decl]);
else list.push(decl);
}
}
if (lexical.size === 0) return;
const varNames = new Set<string>();
for (const stmt of statements) {
collectVarDeclaredNames(stmt, varNames);
if (functionBody && ts.isFunctionDeclaration(stmt) && stmt.name !== undefined)
varNames.add(stmt.name.text);
}
for (const [name, decls] of lexical) {
if (varNames.has(name)) {
report(
diags,
LEXICAL_VAR_COLLISION,
decls[0]!,
functionBody
? `the lexical declaration '${name}' collides with a 'var' or function declaration of the same name in this function body`
: `the lexical declaration '${name}' collides with a 'var' declaration of the same name in this block`,
);
continue;
}
if (decls.length < 2) continue;
// Annex B B.3.3.1: duplicate plain function declarations are the web's
// legacy, and only non-strict code keeps them.
if (decls.every((d) => ts.isFunctionDeclaration(d) && !isEnhancedFunction(d))) {
if (!isStrict(sf, decls[0]!)) continue;
}
report(
diags,
NO_DUPLICATE_LEXICAL,
decls[1]!,
`the declaration '${name}' is already declared in this block's statement list`,
);
}
}
/** Whether a block is the body of a function or a class static block, whose
* top-level function declarations are var-scoped. */
function isFunctionBody(node: ts.Node): boolean {
const parent = node.parent;
if (parent === undefined || !ts.isBlock(node)) return false;
if (ts.isClassStaticBlockDeclaration(parent)) return true;
return ts.isFunctionLike(parent) && (parent as { body?: ts.Node }).body === node;
}
/** The names a DIRECT statement of a statement list declares lexically
* (let/const/using, class, and every function declaration form outside a
* function body), paired with the node to point the diagnostic at. */
function lexicalDeclarationsOf(
stmt: ts.Statement,
functionBody: boolean,
): Array<[string, ts.Node]> {
const names: Array<[string, ts.Node]> = [];
if (ts.isVariableStatement(stmt)) {
if ((stmt.declarationList.flags & ts.NodeFlags.BlockScoped) === 0) return names;
for (const decl of stmt.declarationList.declarations.nodes)
collectBindingNames(decl.name, names);
return names;
}
if (ts.isClassDeclaration(stmt) || (ts.isFunctionDeclaration(stmt) && !functionBody)) {
const name = stmt.name;
if (name !== undefined && ts.isIdentifier(name)) names.push([name.text, stmt]);
}
return names;
}
function collectBindingNames(name: ts.BindingName, into: Array<[string, ts.Node]>): void {
if (ts.isIdentifier(name)) {
into.push([name.text, name]);
return;
}
for (const element of name.elements.nodes) {
if (ts.isOmittedExpression(element)) continue;
if (ts.isBindingElement(element) && element.name !== undefined)
collectBindingNames(element.name, into);
}
}
/** VarDeclaredNames of a statement: every `var`-declared name below it,
* stopping at function and class boundaries (their declarations are not
* var-scoped here). */
function collectVarDeclaredNames(root: ts.Node, into: Set<string>): void {
const stack: ts.Node[] = [root];
while (stack.length > 0) {
const node = stack.pop()!;
if (ts.isFunctionLike(node) || ts.isClassDeclaration(node) || ts.isClassExpression(node))
continue;
if (ts.isVariableDeclarationList(node) && (node.flags & ts.NodeFlags.BlockScoped) === 0) {
const names: Array<[string, ts.Node]> = [];
for (const decl of node.declarations.nodes) collectBindingNames(decl.name, names);
for (const [name] of names) into.add(name);
}
ts.forEachChild(node, (child) => {
stack.push(child);
return undefined;
});
}
}
/** `async function` and `function*` are Annex B's allowance never — only the
* plain form is. */
function isEnhancedFunction(decl: ts.FunctionDeclaration): boolean {
if (decl.asteriskToken !== undefined) return true;
return (ts.getCombinedModifierFlags(decl) & ts.ModifierFlags.Async) !== 0;
}
/* ── SC0007: a declaration where the grammar admits only a Statement ─────
*
* Every `Statement` slot below admits Statement productions only, and only
* Annex B widens three of them to a plain FunctionDeclaration in non-strict
* code: the body of an `if`/`else` and a labelled statement. Async functions,
* generators, classes and lexical declarations are never admitted, and a
* LABELLED function is never admitted as an if/else body (`IsLabelledFunction`
* is an explicit early error there, so `if (x) l: function f(){}` fails even
* in sloppy code). */
type StatementSlot = "if" | "loop" | "with" | "label";
function checkStatementSlot(node: ts.Node, diags: ScrDiagnostic[], isStrict: IsStrictCode): void {
if (ts.isIfStatement(node)) {
checkStatementPosition(node.thenStatement, "if", diags, isStrict);
if (node.elseStatement !== undefined)
checkStatementPosition(node.elseStatement, "if", diags, isStrict);
return;
}
if (
ts.isWhileStatement(node) ||
ts.isDoStatement(node) ||
ts.isForStatement(node) ||
ts.isForInStatement(node) ||
ts.isForOfStatement(node)
) {
checkStatementPosition(node.statement, "loop", diags, isStrict);
return;
}
if (ts.isWithStatement(node)) {
checkStatementPosition(node.statement, "with", diags, isStrict);
return;
}
if (ts.isLabeledStatement(node)) checkStatementPosition(node.statement, "label", diags, isStrict);
}
function checkStatementPosition(
stmt: ts.Statement,
slot: StatementSlot,
diags: ScrDiagnostic[],
isStrict: IsStrictCode,
): void {
const sf = ts.getSourceFileOfNode(stmt);
// A labelled function is an early error as an if-clause, loop, or 'with'
// body in BOTH modes (Node agrees); only a plain labelled STATEMENT is
// Annex B's, and that is the `slot === "label"` case, whose terminal
// function the function branches below handle.
const labelTerminal =
slot === "label" || !ts.isLabeledStatement(stmt) ? null : labelledFunctionOf(stmt);
if (labelTerminal !== null) {
report(
diags,
NOT_A_STATEMENT,
labelTerminal,
"a labelled function declaration cannot be the body of a loop, a 'with' statement, or an 'if' clause",
);
return;
}
if (ts.isVariableStatement(stmt)) {
if ((stmt.declarationList.flags & ts.NodeFlags.BlockScoped) !== 0)
report(
diags,
NOT_A_STATEMENT,
stmt,
"a lexical declaration cannot appear in a single-statement position",
);
return;
}
if (ts.isClassDeclaration(stmt)) {
report(
diags,
NOT_A_STATEMENT,
stmt,
"a class declaration cannot appear in a single-statement position",
);
return;
}
if (!ts.isFunctionDeclaration(stmt)) return;
if (isEnhancedFunction(stmt)) {
report(
diags,
NOT_A_STATEMENT,
stmt,
"an async function or generator declaration can only appear at the top level or inside a block",
);
return;
}
// A plain function declaration: Annex B admits it as an if/else body and as
// a labelled statement, in non-strict code only, and never as a loop's or a
// 'with' statement's body.
if (slot === "loop" || slot === "with") {
report(
diags,
NOT_A_STATEMENT,
stmt,
"a function declaration can only appear at the top level, inside a block, or as the body of an if statement",
);
return;
}
if (isStrict(sf, stmt))
report(
diags,
NOT_A_STATEMENT,
stmt,
"strict mode code may not declare a function in a single-statement position",
);
}
/** The function declaration at the end of a `label:` chain, when `stmt` is a
* labelled statement around one. */
function labelledFunctionOf(stmt: ts.Statement): ts.FunctionDeclaration | null {
let cur: ts.Statement = stmt;
while (ts.isLabeledStatement(cur)) cur = cur.statement;
return ts.isFunctionDeclaration(cur) ? cur : null;
}
/* ── strictness of the code around a node ───────────────────────────────
*
* A directive prologue's "use strict" applies to the function (or script) it
* opens, an arrow function's block body included, and every part of a class is strict by
* construction. Object literal methods and accessors are ordinary functions:
* strict only by their own prologue or their surroundings. Module sources are
* strict whether they say so or not, and Node's classification — not the
* checker's forced external-module scope — decides which sources are modules.
*
* The walk starts at the node's parent: a function declaration's own "use
* strict" makes its body strict, not the code that declares it. */
function isStrictCode(
sf: ts.SourceFile,
node: ts.Node,
isModuleCode: (sf: ts.SourceFile) => boolean,
): boolean {
if (isModuleCode(sf)) return true;
for (let n: ts.Node | undefined = node.parent; n !== undefined; n = n.parent) {
if (ts.isSourceFile(n)) return hasUseStrictPrologue(n.statements.nodes);
if (ts.isClassDeclaration(n) || ts.isClassExpression(n)) return true;
if (ts.isFunctionLike(n)) {
const body = n.body;
if (body !== undefined && ts.isBlock(body) && hasUseStrictPrologue(body.statements.nodes))
return true;
}
}
return false;
}
function hasUseStrictPrologue(statements: readonly ts.Statement[]): boolean {
for (const stmt of statements) {
if (!ts.isExpressionStatement(stmt)) return false;
const expr = stmt.expression;
if (!ts.isStringLiteral(expr)) return false;
if (expr.text === "use strict") return true;
}
return false;
}
@@ -94,3 +94,23 @@ test.for([
invalid,
);
});
test("JavaScript async generators may delegate over a runtime async-iterator shape", () => {
expect(
diagnostics(
`export async function* delegated() { yield* { [Symbol.asyncIterator]: 42 }; }
export async function* looped() { for await (const value of { [Symbol.asyncIterator]() { return "text"; } }) console.dir(value); }`,
'import { delegated, looped } from "./library.js"; delegated(); looped();',
),
).toEqual([]);
});
test("TypeScript keeps the async-iterator demand", () => {
expect(
diagnostics(
"export async function* delegated() { yield* { [Symbol.asyncIterator]: 42 }; }",
'import { delegated } from "./library.js"; delegated();',
"ts",
).some((diagnostic) => diagnostic.message.includes("Symbol.asyncIterator")),
).toBe(true);
});
@@ -6,7 +6,8 @@ import { isGenericCallableMemberType } from "../type-mapper.js";
/** True when nothing in `sym`'s DECLARING FILE ever writes it after the
* initializer: assignments (plain and compound, destructuring targets
* included), ++/--, and for-of/for-in expression targets all count.
* included), ++/--, for-of/for-in expression targets, a `var` in a
* for-of/for-in head, and redeclarations that initialize all count.
* Sound file-locally for module-scope bindings because ESM import
* bindings are read-only — no other file can write one. Cached per
* symbol (the scan walks the whole file once). */
@@ -31,6 +32,7 @@ export function fileWritesBinding(
file: ts.SourceFile,
resolve: (ident: ts.Identifier) => ts.Symbol | null | undefined,
): boolean {
if (declarationsWriteBinding(sym)) return true;
let written = false;
// Text pre-check keeps the file walk cheap: only same-named
// identifiers pay a symbol resolution.
@@ -78,6 +80,77 @@ export function fileWritesBinding(
return written;
}
/** True when `sym`'s own declarations write it after it holds a value:
* a `var` in a for-in or for-of head is written on every iteration, a
* `var` with an initializer inside a loop body is written each time the
* body runs, and a binding declared more than once (`var x = 1; ... var x
* = 2;`, a parameter or function declaration that a `var` redeclares) is
* written by each initializing declaration that runs. Only value
* declarations count, and of functions only implementations: overload
* signatures have no body. A redeclaration without an initializer is
* counted too, which only over-approximates. */
function declarationsWriteBinding(sym: ts.Symbol): boolean {
let values = 0;
let initializing = 0;
for (const d of sym.declarations) {
let root: ts.Node = d;
while (
root.parent !== undefined &&
(ts.isBindingElement(root) ||
ts.isObjectBindingPattern(root) ||
ts.isArrayBindingPattern(root))
)
root = root.parent;
if (ts.isVariableDeclaration(root)) {
const list = root.parent;
if (
ts.isVariableDeclarationList(list) &&
(list.flags & ts.NodeFlags.BlockScoped) === 0 &&
(ts.isForInStatement(list.parent) || ts.isForOfStatement(list.parent))
)
return true;
values++;
if (root.initializer !== undefined) {
if (insideLoopBody(root)) return true;
initializing++;
}
} else if (
ts.isParameter(root) ||
(ts.isFunctionDeclaration(d) && d.body !== undefined) ||
ts.isClassDeclaration(d)
) {
values++;
initializing++;
}
}
return values > 1 && initializing > 0;
}
/** Whether a declaration sits in the body of a loop of its own function,
* where it runs once per iteration. */
function insideLoopBody(node: ts.Node): boolean {
let child = node;
for (let parent = node.parent; parent !== undefined; parent = parent.parent) {
if (
ts.isFunctionLike(parent) ||
ts.isClassStaticBlockDeclaration(parent) ||
ts.isSourceFile(parent)
)
return false;
if (
(ts.isForStatement(parent) ||
ts.isForInStatement(parent) ||
ts.isForOfStatement(parent) ||
ts.isWhileStatement(parent) ||
ts.isDoStatement(parent)) &&
parent.statement === child
)
return true;
child = parent;
}
return false;
}
/** Strips the value-preserving wrappers off an expression: parens,
* non-null assertions, `as`/`satisfies`/angle-bracket casts. What
* remains is the expression that actually evaluates. */
@@ -1,4 +1,5 @@
import { lowerReadlineCursorCall } from "./builtins/readline.js";
import { lowerFsWatchFileCall } from "./builtins/filesystem.js";
import { lowerMonitorEventLoopDelayCall } from "./builtins/performance.js";
import {
lowerEnvironmentData,
@@ -134,6 +135,13 @@ export function lowerBuiltinCall(
if (streamServed) return streamServed;
const fsTs = lowerer.lowerFsToUnixTimestampCall(expr, bi, loc);
if (fsTs) return fsTs;
// fs.watchFile/fs.unwatchFile have no table entry (their checked-dynamic
// form is the fs.watchFileChk ladder below), so the typed polling forms
// dispatch here; anything else falls through to the ladder and the fence.
if (bi.module === "fs" && (bi.member === "watchFile" || bi.member === "unwatchFile")) {
const polling = lowerFsWatchFileCall(lowerer, expr, loc, bi.member === "unwatchFile");
if (polling) return polling;
}
const fsLadder = lowerer.lowerFsLadderCall(expr, bi, loc);
if (fsLadder) return fsLadder;
const cryptoServed = lowerer.lowerCryptoModuleCall(expr, bi, loc);
@@ -72,7 +72,7 @@ const DATE_TEXT_FNS: ReadonlyMap<string, IrLibFn> = new Map([
]);
export const DATE_METHOD_HINT =
"getTime(), valueOf(), toISOString(), toUTCString(), toJSON(), toString(), toDateString(), toTimeString(), the local/UTC calendar getters and setters, setTime(), and getTimezoneOffset() are supported; locale formatters have no lowering";
"getTime(), valueOf(), toISOString(), toUTCString(), toJSON(), toString(), toDateString(), toTimeString(), toLocaleString(), toLocaleTimeString(), the local/UTC calendar getters and setters, setTime(), and getTimezoneOffset() are supported; locale and options arguments, toLocaleDateString() and the Intl.DateTimeFormat surface have no lowering";
/** Date statics and methods on Date objects (a TimeClip'd millisecond
* time value with identity): getters, text forms and setters.
@@ -208,6 +208,19 @@ export function dateValueMethod(name: string, value: IrExpr, loc: SrcLoc): IrExp
if (textFn !== undefined) {
return { kind: "libCall", fn: textFn, args: [value], type: STRING, loc };
}
// The default locale's forms. The spec makes them DateTimeFormat(locale,
// options).format(this); with neither argument that is ICU's default
// locale over the host time zone, which scr_intl.c formats over the
// ICU release exactly as V8 does.
if (name === "toLocaleString" || name === "toLocaleTimeString") {
return {
kind: "libCall",
fn: name === "toLocaleString" ? "icu.dateLocaleString" : "icu.dateLocaleTimeString",
args: [value],
type: STRING,
loc,
};
}
return null;
}
@@ -8,6 +8,7 @@ import {
builtinModuleFnOf,
FS_READDIR_DOCUMENTED_OPTIONS,
FS_WATCH_DOCUMENTED_OPTIONS,
FS_WATCH_FILE_DOCUMENTED_OPTIONS,
fenceOrDropOptionKey,
} from "../surfaces.js";
import { defaultAfterUndefined, lowerStaticallyUndefinedArgument } from "../optional-arguments.js";
@@ -1127,13 +1128,37 @@ export function lowerStatsMethodCall(
);
}
/** The watch-listener shapes: () , (eventType: string), and
* (eventType: string, filename: string | null) — the filename parameter
* must admit null (Node passes null when the platform names nothing), or
* be the checked dynamic. Returns the parameter count, or null when the
* shape has no lowering. */
function watchListenerArity(lowerer: Lowerer, cb: IrExpr): 0 | 1 | 2 | null {
if (cb.type.kind !== "func" || cb.type.ret.kind !== "void" || cb.type.params.length > 2)
return null;
const [event, name] = cb.type.params;
if (event !== undefined && event.kind !== "string" && event.kind !== "dyn") return null;
if (name !== undefined && name.kind !== "dyn") {
const def = name.kind === "union" ? lowerer.unions.get(name.unionId) : undefined;
if (
!def ||
!def.arms.some((a) => a.kind === "nullT") ||
!def.arms.some((a) => a.kind === "string") ||
!def.arms.every((a) => a.kind === "nullT" || a.kind === "undefinedT" || a.kind === "string")
)
return null;
}
return cb.type.params.length as 0 | 1 | 2;
}
/** `fs.watch(path[, options][, listener])` → the fs.watch libCall
* (scr_watch.c — kqueue EVFILT_VNODE on the opened path; an unopenable
* path THROWS Node's fs error synchronously, the polling-fallback catch
* shape). The listener fires with "rename"/"change" and takes zero
* parameters or the eventType string — the filename parameter has no
* lowering (kqueue watches the inode, not the directory entry; you
* watched one path). The options record follows the options-record
* parameters, the eventType string, or (eventType, filename: string |
* null) — the filename is the entry name on a directory watch and the
* watched file's basename otherwise (Linux inotify names both; a macOS
* directory watch has no name source until F3). The options record follows the options-record
* stance: persistent: true, recursive: false, and encoding: "utf8"
* state the lowered behavior and are accepted; persistent: false
* (a watcher that does NOT hold the loop), recursive: true (kqueue
@@ -1204,7 +1229,7 @@ export function lowerFsWatchCall(lowerer: Lowerer, expr: ts.CallExpression, loc:
lowerer.noLowering(
"fs.watch with a non-utf8 encoding",
prop,
"the encoding applies to the filename argument, which has no lowering — utf8 (the default) is accepted",
"the filename argument is a string — utf8 (the default) is the accepted encoding",
);
}
if (key === "signal") {
@@ -1236,19 +1261,11 @@ export function lowerFsWatchCall(lowerer: Lowerer, expr: ts.CallExpression, loc:
: null;
if (listenerArg !== null) {
const cb = lowerer.lowerExpr(listenerArg);
if (cb.type.kind !== "func" || cb.type.ret.kind !== "void" || cb.type.params.length > 1) {
if (watchListenerArity(lowerer, cb) === null) {
lowerer.noLowering(
"fs.watch with this listener shape",
listenerArg,
"the listener takes () or (eventType: string) — the filename parameter has no lowering (you watched one path)",
);
}
const param = cb.type.params[0];
if (param !== undefined && param.kind !== "string") {
lowerer.unsupported(
"SC1090",
listenerArg,
`watch listeners whose parameter is not the eventType string (got '${lowerer.fmt(param)}')`,
"the listener takes (), (eventType: string), or (eventType: string, filename: string | null)",
);
}
args.push(cb);
@@ -1262,10 +1279,134 @@ export function lowerFsWatchCall(lowerer: Lowerer, expr: ts.CallExpression, loc:
};
}
/** Method calls on FSWatcher receivers: close() — idempotent, statement
* position (Node returns void there too). Everything else @types/node
* declares (ref/unref, the EventEmitter surface) fences member-
* qualified. Null for non-watcher receivers. */
/** `fs.watchFile(path[, options], listener)` and
* `fs.unwatchFile(path[, listener])` → the polling libCalls
* (scr_watch_file.c: libuv's uv_fs_poll shape — one poller per RESOLVED
* path, shared by every listener on it, re-armed every `interval` ms,
* 5007 by default, persistent by default). The listener takes
* (current: Stats, previous: Stats) — zero, one or both — and runs when a
* poll sees a difference; the first poll only records the baseline, and a
* file that cannot be stat'd polls as all-zero Stats. The options record
* follows the options-record stance: interval, persistent: true/false and
* bigint: false are accepted; bigint: true fences (BigIntStats); other
* documented keys fence by name; undocumented keys drop like Node.
* Statement position only — Node returns a StatWatcher, which has no
* lowering. Returns null for any untyped shape so the checked-dynamic
* ladder (fs.watchFileChk) keeps its current IR. */
export function lowerFsWatchFileCall(
lowerer: Lowerer,
expr: ts.CallExpression,
loc: SrcLoc,
unwatch: boolean,
): IrExpr | null {
const nodes = expr.arguments.nodes;
if (nodes.length < 1 || nodes.length > 3 || nodes.some(ts.isSpreadElement)) return null;
if (lowerer.mapTypeOf(lowerer.typeOf(nodes[0]!))?.kind !== "string") return null;
const path = lowerer.lowerExprExpecting(nodes[0]!, STRING);
if (unwatch) {
if (nodes.length > 2) return null;
const args: IrExpr[] = [path];
if (nodes.length === 2) {
if (lowerer.mapTypeOf(lowerer.typeOf(nodes[1]!))?.kind !== "func") return null;
args.push(lowerer.lowerExpr(nodes[1]!));
}
return { kind: "libCall", fn: "fs.unwatchFile", args, type: VOID, loc };
}
const hasOptions = nodes.length >= 2 && ts.isObjectLiteralExpression(nodes[1]!);
if (nodes.length === 3 && !hasOptions) return null;
if (nodes.length === 2 && hasOptions) return null; /* no listener */
const listenerArg = nodes[nodes.length - 1]!;
if (lowerer.mapTypeOf(lowerer.typeOf(listenerArg))?.kind !== "func") return null;
const parent = expr.parent;
if (!ts.isExpressionStatement(parent) && !(ts.isArrowFunction(parent) && parent.body === expr)) {
lowerer.unsupported(
"SC1090",
expr,
"using the returned StatWatcher (register the fs.watchFile call as its own statement)",
);
}
let interval: IrExpr | null = null;
let persistent: IrExpr | null = null;
if (hasOptions) {
for (const prop of (nodes[1] as ts.ObjectLiteralExpression).properties.nodes) {
if (!ts.isPropertyAssignment(prop) && !ts.isShorthandPropertyAssignment(prop)) {
lowerer.noLowering(
"fs.watchFile options with computed keys or spreads",
prop,
"each option must be a plain `name: value` entry with a literal key",
);
}
if (!ts.isIdentifier(prop.name) && !ts.isStringLiteral(prop.name)) {
lowerer.noLowering(
"fs.watchFile options with computed keys",
prop,
"each option must be a plain `name: value` entry with a literal key",
);
}
const key = prop.name.text;
const value = ts.isPropertyAssignment(prop)
? prop.initializer
: ts.isShorthandPropertyAssignment(prop)
? prop.name
: null;
if (key === "interval" && value !== null) {
interval = lowerer.lowerExprExpecting(value, F64);
continue;
}
if (key === "persistent" && value !== null) {
persistent = lowerer.lowerExprExpecting(value, BOOL);
continue;
}
if (key === "bigint") {
// The fallback's option record declares `bigint: false`, so the
// false literal (or `false` as a value) states the lowered Stats
// shape; BigIntStats has no lowering.
if (value !== null && value.kind === ts.SyntaxKind.FalseKeyword) continue;
lowerer.noLowering(
"fs.watchFile with bigint: true",
prop,
"the listener receives Stats (double integers); BigIntStats has no lowering",
);
}
fenceOrDropOptionKey(
lowerer,
prop,
key,
"fs.watchFile",
FS_WATCH_FILE_DOCUMENTED_OPTIONS,
"interval, persistent: true/false, and bigint: false are the accepted options",
);
// An undocumented key, dropped like Node drops it.
}
}
const cb = lowerer.lowerExpr(listenerArg);
if (
cb.type.kind !== "func" ||
cb.type.ret.kind !== "void" ||
cb.type.params.length > 2 ||
!cb.type.params.every((p) => p.kind === "stats")
) {
lowerer.noLowering(
"fs.watchFile with this listener shape",
listenerArg,
"the listener takes (current: Stats, previous: Stats)",
);
}
return {
kind: "libCall",
fn: "fs.watchFile",
args: [path, interval ?? numLit(5007, loc), persistent ?? boolLit(true, loc), cb],
type: VOID,
loc,
};
}
/** Method calls on FSWatcher receivers: close() (idempotent, statement
* position — Node returns void there too) and the EventEmitter surface for
* 'change', 'error' and 'close' (on/once/addListener/prependListener/
* prependOnceListener/off/removeListener, registered as their own
* statement). Everything else @types/node declares (ref/unref, other
* events) fences member-qualified. Null for non-watcher receivers. */
export function lowerWatcherMethodCall(
lowerer: Lowerer,
call: ts.CallExpression,
@@ -1280,10 +1421,82 @@ export function lowerWatcherMethodCall(
const receiver = lowerer.lowerExpr(access.expression);
return { kind: "libCall", fn: "watcher.close", args: [receiver], type: VOID, loc };
}
// The EventEmitter surface for the three events the watcher emits:
// "change" (the listener shapes fs.watch takes), "error", and "close".
const ON = ["on", "once", "addListener", "prependListener", "prependOnceListener"];
if (
(ON.includes(name) || name === "off" || name === "removeListener") &&
call.arguments.nodes.length === 2
) {
const evT = lowerer.typeOf(call.arguments.nodes[0]!);
const event = evT.isStringLiteralType() ? evT.value : null;
const code = event === "change" ? 0 : event === "error" ? 1 : event === "close" ? 2 : -1;
if (code < 0) {
lowerer.noLowering(
`FSWatcher.${name}(${event === null ? "non-literal event" : `"${event}"`}, ...)`,
call.arguments.nodes[0]!,
'"change", "error", and "close" are the supported FSWatcher events (as literals)',
);
}
const parent = call.parent;
if (
!ts.isExpressionStatement(parent) &&
!(ts.isArrowFunction(parent) && parent.body === call)
) {
lowerer.unsupported(
"SC1090",
call,
`using the result of FSWatcher.${name} (register it as its own statement)`,
);
}
const receiver = lowerer.lowerExpr(access.expression);
const cb = lowerer.lowerExpr(call.arguments.nodes[1]!);
if (name === "off" || name === "removeListener") {
return {
kind: "libCall",
fn: "watcher.off",
args: [receiver, numLit(code, loc), cb],
type: VOID,
loc,
};
}
const ok =
code === 0
? watchListenerArity(lowerer, cb) !== null
: code === 1
? cb.type.kind === "func" &&
cb.type.ret.kind === "void" &&
cb.type.params.length <= 1 &&
(cb.type.params[0] === undefined ||
cb.type.params[0].kind === "dyn" ||
(cb.type.params[0].kind === "object" && cb.type.params[0].className === "%Error"))
: cb.type.kind === "func" && cb.type.ret.kind === "void" && cb.type.params.length === 0;
if (!ok) {
lowerer.noLowering(
`FSWatcher.${name}("${event}", ...) with this listener shape`,
call.arguments.nodes[1]!,
code === 0
? "the listener takes (), (eventType: string), or (eventType: string, filename: string | null)"
: code === 1
? "the listener takes () or (error: Error)"
: "the listener takes ()",
);
}
const flags =
(name === "once" || name === "prependOnceListener" ? 1 : 0) |
(name.startsWith("prepend") ? 2 : 0);
return {
kind: "libCall",
fn: "watcher.on",
args: [receiver, numLit(code, loc), numLit(flags, loc), cb],
type: VOID,
loc,
};
}
lowerer.noLowering(
`FSWatcher.${name}`,
call,
"close() is the supported FSWatcher member",
'close(), and on/once/addListener/prependListener/prependOnceListener/off/removeListener for "change", "error", and "close" are the supported FSWatcher members',
lowerer.checker.getSymbolAtLocation(access.name),
);
}
@@ -32,6 +32,7 @@ import { lowerUtilTypeCall } from "./util-types.js";
import { InternalCompilerError } from "../../../errors.js";
import { dynUndefinedExpr, varRef } from "../../../ir/build.js";
import * as ts from "../../ts7/adapter.js";
import { jsCheckedBuiltinCall } from "../lower-builtin-values.js";
import { type Lowerer } from "../lowerer.js";
import { type BuiltinModuleFn } from "../surfaces.js";
import { type ParamShape } from "../call-signatures.js";
@@ -72,6 +73,10 @@ export function lowerBuiltinModuleCall(
);
if (fallible) return fallible;
checkFsUrlPaths(lowerer, expr, bi);
{
const checked = jsCheckedBuiltinCall(lowerer, expr, fn, `${bi.module}.${bi.member}`, loc);
if (checked) return checked;
}
if (bi.module === "util/types") {
return lowerUtilTypeCall(lowerer, expr, bi, loc, name);
}
@@ -17,8 +17,9 @@ import {
} from "../../../ir/ir.js";
/** `os.userInfo()` — the passwd-entry snapshot, Node's uv_os_get_passwd
* behind the call site's own mapped record shape: username = pw_name,
* uid/gid = getuid/getgid, shell = pw_shell (as the `string | null`
* for the effective uid behind the call site's own mapped record shape:
* username = pw_name, uid/gid = pw_uid/pw_gid (the user's primary group,
* not the process gid; -1 on Windows), shell = pw_shell (as the `string | null`
* union @types/node declares — POSIX always answers the string arm;
* null is Node's Windows answer), homedir = pw_dir (the PASSWD home,
* NOT os.homedir's $HOME-first cascade — Node's own split). The record
@@ -56,12 +57,12 @@ export function lowerOsUserInfoCall(
} else if (f.name === "uid" && f.type.kind === "f64") {
fields.push({
name: f.name,
value: { kind: "libCall", fn: "process.getuid", args: [], type: F64, loc },
value: { kind: "libCall", fn: "os.userUid", args: [], type: F64, loc },
});
} else if (f.name === "gid" && f.type.kind === "f64") {
fields.push({
name: f.name,
value: { kind: "libCall", fn: "process.getgid", args: [], type: F64, loc },
value: { kind: "libCall", fn: "os.userGid", args: [], type: F64, loc },
});
} else if (f.name === "homedir" && f.type.kind === "string") {
fields.push({
@@ -34,6 +34,7 @@ import { builtinFunctionValue } from "../lower-exprs.js";
import { canonicalBuiltinModule } from "../../builtin-modules.js";
import { lowerProcessIpcSend, ipcMessageListener, ipcDisconnectListener } from "./ipc.js";
import { lowerDiagnosticsSubscriber, lowerTracingArguments } from "./async-context.js";
import { isCheckerPanic } from "../../../diagnostics/diagnostic.js";
/** Whether the process has an IPC channel, Node's test for defining
* process.send and process.disconnect: a channel this fork child opened
@@ -173,6 +174,70 @@ export function programReadsExitCode(lowerer: Lowerer): boolean {
return found;
}
/** A write-method replacement on a process output stream, its source site.
* `stream` is null when the receiver is a first-class alias, whose type does
* not say which of the two output streams it holds. */
export interface StdioPatchSite {
stream: "stdout" | "stderr" | null;
loc: SrcLoc;
}
/** Whether any program file replaces a process output stream's write
* method: `process.stdout.write = …` / `process.stderr.write = …`, or the
* same assignment through a first-class alias of the stream (a value whose
* type maps to the process stream). Such a program routes its typed
* stdout/stderr writes through the stream object, so the replacement sees
* them with their arguments, as in Node. Cached per program; the first
* site found is recorded for the executable planner's SC8020 check. */
export function programPatchesStdio(lowerer: Lowerer): boolean {
if (lowerer.patchesStdio !== null) return lowerer.patchesStdio;
let found = false;
// undefined: not an output stream; null: a first-class alias of one.
const streamOf = (recv: ts.Expression): "stdout" | "stderr" | null | undefined => {
if (ts.isPropertyAccessExpression(recv)) {
const direct = lowerer.stdlibGlobalMember(recv, "process");
if (direct === "stdout" || direct === "stderr") return direct;
}
// An arbitrary receiver's mapping is a query, never a commit: a checker
// panic (the type has no readable form) is not a patch site. The type
// cannot say WHICH output stream an alias holds.
try {
return lowerer.mapTypeOf(lowerer.typeOf(recv))?.kind === "procStream" ? null : undefined;
} catch (e) {
if (!isCheckerPanic(e)) throw e;
return undefined;
}
};
const walk = (node: ts.Node): void => {
if (found) return;
if (
ts.isBinaryExpression(node) &&
node.operatorToken.kind === ts.SyntaxKind.EqualsToken &&
ts.isPropertyAccessExpression(node.left) &&
node.left.name.text === "write"
) {
const recv = node.left.expression;
const direct = ts.isPropertyAccessExpression(recv)
? lowerer.stdlibGlobalMember(recv, "process")
: null;
const stream = direct === "stdout" || direct === "stderr" ? direct : streamOf(recv);
if (stream !== undefined) {
found = true;
lowerer.stdioPatchSites.push({ stream, loc: locOf(node) });
return;
}
}
ts.forEachChild(node, walk);
};
for (const sf of lowerer.program.getImplementationSourceFiles()) {
if (sf.isDeclarationFile) continue;
walk(sf);
if (found) break;
}
lowerer.patchesStdio = found;
return found;
}
/** The pinned Node release the runtime reports as process.versions.node
* (packages/runtime/src/scr_lib.c SCR_NODE_COMPAT_VERSION, kept equal to
* internal/compatibility/node-v24.json by test/node-pin.test.ts). */
@@ -341,6 +406,26 @@ export function lowerProcessStreamProperty(
loc: locOf(expr),
};
}
// The write method as a VALUE in TypeScript (`const original =
// process.stdout.write`, `process.stdout.write.bind(process.stdout)`):
// the stream object's current write slot, a checked-dynamic function
// (the native method until a program replaces it).
if (member === "write" && (direct === "stdout" || direct === "stderr")) {
const loc = locOf(expr);
return {
kind: "dynKeyGet",
value: {
kind: "libCall",
fn: "process.stdioSlot",
args: [numLit(direct === "stdout" ? 1 : 2, loc)],
type: DYN,
loc,
},
key: strLit("write", locOf(expr.name)),
type: DYN,
loc,
};
}
// `if (process.stdin.setRawMode)`: Node's stdin has the method only
// when it is a TTY. The runtime's stream object answers the same
// presence (the method or undefined); calls keep their direct lowering.
@@ -357,7 +442,9 @@ export function lowerProcessStreamProperty(
member !== "isTTY" &&
member !== "columns" &&
member !== "rows" &&
member !== "writableLength"
member !== "writableLength" &&
member !== "writableHighWaterMark" &&
member !== "writableNeedDrain"
)
return null;
let recv: ts.Expression = expr.expression;
@@ -391,19 +478,31 @@ export function lowerProcessStreamProperty(
// nothing is ever queued and Node's number is always 0. The other
// members stay keyed reads — the runtime stream answers their live
// values (isatty columns/rows, stdin readableLength).
if (member === "writableLength") {
if (
member === "writableLength" ||
member === "writableHighWaterMark" ||
member === "writableNeedDrain"
) {
// The receiver still EVALUATES: a call or property read that hands
// back the stream runs its own side effects — and throws where Node
// throws — before the constant answers, exactly as the read of the
// member would. A pure receiver (the alias above) folds away.
const answer: IrExpr = { kind: "numLit", value: 0, type: F64, loc: locOf(expr) };
const answer: IrExpr =
member === "writableNeedDrain"
? { kind: "boolLit", value: false, type: BOOL, loc: locOf(expr) }
: {
kind: "numLit",
value: member === "writableLength" ? 0 : 65536,
type: F64,
loc: locOf(expr),
};
const effect = lowerer.lowerExpr(expr.expression);
if (isSafeToDiscard(effect)) return answer;
return {
kind: "seqExpr",
stmts: [{ kind: "exprStmt", expr: effect, loc: locOf(expr.expression) }],
result: answer,
type: F64,
type: answer.type,
loc: locOf(expr),
};
}
@@ -435,6 +534,16 @@ export function lowerProcessStreamProperty(
// true for the same reason). Backpressure probes read 0 and stay
// correct; the write itself never defers.
if (member === "writableLength") return { kind: "numLit", value: 0, type: F64, loc };
// Node's default high-water mark for the process streams, and the
// drain flag a synchronous write never sets.
if (member === "writableHighWaterMark") {
if (stream === "stdin") return null;
return { kind: "numLit", value: 65536, type: F64, loc };
}
if (member === "writableNeedDrain") {
if (stream === "stdin") return null;
return { kind: "boolLit", value: false, type: BOOL, loc };
}
if (member === "isTTY") {
const type = lowerer.withUndefinedArm(BOOL);
return {
@@ -569,9 +678,10 @@ export function lowerProcessProperty(
// The capability-probe members that honestly DON'T EXIST in a
// compiled binary — each reads undefined (their declared types carry
// the undefined arm), so feature probes take their documented
// fallbacks: no gyp build config (process.config.variables.* — no ICU,
// no QUIC — and process.config.target_defaults), no feature flags
// (process.features.* — no inspector, not a debug build).
// fallbacks: no gyp build config (process.config.variables.* — no QUIC
// — and process.config.target_defaults; v8_enable_i18n_support is the
// exception below, answered from the binary's linked ICU), no feature
// flags (process.features.* — no inspector, not a debug build).
if (!expr.questionDotToken && ts.isPropertyAccessExpression(expr.expression)) {
const container = lowerer.stdlibGlobalMember(expr.expression, "process");
if (container === "features") {
@@ -594,6 +704,13 @@ export function lowerProcessProperty(
expr.expression.name.text === "variables" &&
lowerer.stdlibGlobalMember(expr.expression.expression, "process") === "config"
) {
// The one variable a compiled binary can answer truthfully:
// v8_enable_i18n_support. Node bundles ICU and answers the number 1;
// this read selects the `icu` runtime feature (scr_intl.c), so a
// binary that reports ICU is a binary that carries it.
if (expr.name.text === "v8_enable_i18n_support") {
return { kind: "libCall", fn: "icu.i18nSupport", args: [], type: F64, loc: locOf(expr) };
}
return { kind: "unitLit", unit: "undefined", type: UNDEFINED_T, loc: locOf(expr) };
}
}
@@ -920,6 +1037,58 @@ export function lowerProcessMethodCall(
}
return { kind: "libCall", fn: "process.stdinSetRawMode", args: [mode], type: VOID, loc };
}
// process.stdout/stderr.on("drain", fn) and its spellings: Node emits
// 'drain' only after a write() returned false, and these writes are
// synchronous (they never return false), so the listener can never run.
// Registering it is therefore observably a no-op: the receiver and the
// listener are evaluated for their effects and nothing is retained.
// listenerCount/listeners/eventNames on the process streams stay
// refused, so the missing registration cannot be observed.
if (
[
"on",
"once",
"off",
"addListener",
"removeListener",
"prependListener",
"prependOnceListener",
].includes(access.name.text) &&
ts.isPropertyAccessExpression(access.expression) &&
call.arguments.nodes.length === 2 &&
!call.arguments.nodes.some(ts.isSpreadElement)
) {
const stream = lowerer.stdlibGlobalMember(access.expression, "process");
const evT = lowerer.typeOf(call.arguments.nodes[0]!);
if (
(stream === "stdout" || stream === "stderr") &&
evT.isStringLiteralType() &&
evT.value === "drain"
) {
const loc = locOf(call);
const parent = call.parent;
const discarded =
ts.isExpressionStatement(parent) || (ts.isArrowFunction(parent) && parent.body === call);
if (!discarded) {
lowerer.unsupported(
"SC1090",
call,
`using the result of process.${stream}.${access.name.text}("drain", ...) (register it as its own statement)`,
);
}
const stmts: IrStmt[] = [];
for (const arg of call.arguments.nodes) {
const value = lowerer.lowerExprExpecting(arg, DYN);
if (!isSafeToDiscard(value)) stmts.push({ kind: "exprStmt", expr: value, loc: value.loc });
}
// The call's value is discarded (statement position or a void arrow
// body), so the lowering is the evaluated arguments alone; a bare
// unitLit has no standalone IR value, so the no-op result is the
// dynamic undefined a discarded call leaves.
const result: IrExpr = dynUndefinedExpr(loc);
return stmts.length === 0 ? result : { kind: "seqExpr", stmts, result, type: DYN, loc };
}
}
// process.stdout.on("resize", cb) (and once/off/removeListener/
// addListener, stderr too): the terminal resize event, registered on the
// runtime's process stream object (a SIGWINCH listener that emits when a
@@ -1078,6 +1247,33 @@ export function lowerProcessMethodCall(
if (stream === "stdout" || stream === "stderr") {
const loc = locOf(call);
const args = call.arguments.nodes;
// In a program that replaces a process stream's write, a write call is
// what Node makes it: a call of the stream's CURRENT write property,
// with the arguments as passed (the replacement sees byte chunks,
// encodings and callbacks). The native method behind the slot is the
// same synchronous write, so an unpatched stream behaves as before.
if (programPatchesStdio(lowerer) && !args.some(ts.isSpreadElement)) {
const handle: IrExpr = {
kind: "libCall",
fn: "process.stdioSlot",
args: [numLit(stream === "stdout" ? 1 : 2, loc)],
type: DYN,
loc,
};
const invoke: IrExpr = {
kind: "dynInvoke",
recv: handle,
method: "write",
calleeName: ts.getTextOfNode(access),
args: args.map((arg) => lowerer.lowerExprExpecting(arg, DYN)),
type: DYN,
loc,
};
// The declared boolean: a replacement returning something else
// throws at a use that needs the boolean (types do not lie); a
// discarded result is not converted.
return resultIsDiscarded(call) ? invoke : lowerer.coerceToExpected(invoke, BOOL);
}
if (args.length < 1 || args.length > 3 || args.some(ts.isSpreadElement)) {
lowerer.noLowering(
`process.${stream}.write with ${args.length} arguments`,
@@ -1174,6 +1370,24 @@ export function lowerProcessMethodCall(
if (data.type.kind === "dyn" && isJsSourceFile(ts.getSourceFileOfNode(call))) {
data = { kind: "dynCheck", value: data, type: STRING, loc };
}
// A chunk known only at run time (`any`, or a string | Uint8Array
// union): the runtime dispatches on the value — a string or a
// Uint8Array/Buffer writes, null throws ERR_STREAM_NULL_VALUES, any
// other value throws ERR_INVALID_ARG_TYPE, as Node's Writable.write
// does before writing anything. Only the one-argument form.
if (
args.length === 1 &&
(data.type.kind === "dyn" || data.type.kind === "union") &&
(data.type.kind === "dyn" || lowerer.dynConvertible(data.type))
) {
return {
kind: "libCall",
fn: "process.stdioWriteDyn",
args: [numLit(stream === "stdout" ? 1 : 2, loc), lowerer.coerceToExpected(data, DYN)],
type: BOOL,
loc,
};
}
const isBytes = data.type.kind === "bytes" && data.type.elem === "u8";
if (!isBytes && data.type.kind !== "string") {
lowerer.noLowering(
@@ -30,11 +30,11 @@ const HEAP_STATISTICS_FIELDS = [
];
/** `v8.getHeapStatistics()`: one memoryUsage sample (the runtime heap's
* total and used bytes), with the machine's memory as the heap limit (the
* runtime heap has no fixed ceiling) and the available size the limit
* minus used bytes. V8-only figures (executable code space, malloced
* memory, global handles, external memory, zapping) read 0, and one
* native context exists. The record is the call site's interned HeapInfo
* total and used bytes, and external memory, memoryUsage's `external`),
* with the machine's memory as the heap limit (the runtime heap has no
* fixed ceiling) and the available size the limit minus used bytes.
* V8-only figures (executable code space, malloced memory, global handles,
* zapping) read 0, and one native context exists. The record is the call site's interned HeapInfo
* shape, built by a shared helper per shape. */
export function lowerV8HeapStatisticsCall(
lowerer: Lowerer,
@@ -73,6 +73,8 @@ export function lowerV8HeapStatisticsCall(
return sample(1);
case "used_heap_size":
return sample(2);
case "external_memory":
return sample(3);
case "heap_size_limit":
return limit();
case "total_available_size":
@@ -1,4 +1,7 @@
import { staticFetchMembers } from "../../../compat/fetch-profile.js";
import {
READABLE_STREAM_WEB_VIEW_CALLS,
staticFetchMembers,
} from "../../../compat/fetch-profile.js";
import { numLit } from "../../../ir/build.js";
import { everyStmt } from "../../../ir/traverse.js";
import {
@@ -16,6 +19,7 @@ import { constituentTypes } from "../../ts7/checker.js";
import { ENTRY_NAME, type Lowerer } from "../lowerer.js";
import { stdlibGlobalNameOf } from "../surfaces.js";
import { streamWebImportOf } from "./module-bindings.js";
import { lowerDynObjectLiteral } from "../expressions/object-literals.js";
/* The web stream classes in JavaScript sources: the globals and the
* node:stream/web exports, which are the same constructors. Each reference
@@ -92,6 +96,60 @@ export function programNamesWebStreamClass(lowerer: Lowerer): boolean {
return found;
}
/** The classes a TypeScript source constructs through the web stream
* classes once it uses an operation the native streams lack. */
const TYPED_CLASSES: ReadonlySet<string> = new Set([
"ReadableStream",
"WritableStream",
"TransformStream",
]);
const WEB_VIEW_CALLS: ReadonlySet<string> = new Set(READABLE_STREAM_WEB_VIEW_CALLS);
/** Whether a TypeScript source of the program uses a web stream operation
* the native streams lack: a ReadableStream tee(), pipeTo() or
* pipeThrough() call, a WritableStream constructor, or a TransformStream
* constructor with arguments (a transformer or queuing strategies). Such a
* program builds its ReadableStream, WritableStream and TransformStream
* values with the web stream classes, as JavaScript does; a program that
* uses only getReader(), cancel() and the identity transform keeps the
* native streams. */
export function programUsesTypedWebStreams(lowerer: Lowerer): boolean {
if (lowerer.typedWebStreams !== undefined) return lowerer.typedWebStreams;
let found = false;
for (const file of lowerer.program.getSourceFiles()) {
if (file.isDeclarationFile || isJsSourceFile(file)) continue;
ts.walkPreorder(file, (node) => {
const member = staticMemberName(node);
if (member !== null && WEB_VIEW_CALLS.has(member) && isReadableStreamMember(lowerer, node))
found = true;
else if (ts.isNewExpression(node)) {
const name = stdlibGlobalNameOf(lowerer, unparenthesized(node.expression));
if (
name === "WritableStream" ||
(name === "TransformStream" && (node.arguments?.nodes.length ?? 0) > 0)
)
found = true;
}
return found ? "stop" : undefined;
});
if (found) break;
}
lowerer.typedWebStreams = found;
return found;
}
/** The class a TypeScript `new` expression constructs through the web
* stream classes (programUsesTypedWebStreams), as its index in
* WEB_STREAM_CLASSES; null otherwise. */
function typedWebStreamClassKindOf(lowerer: Lowerer, node: ts.Expression): number | null {
const expr = unparenthesized(node);
if (!ts.isIdentifier(expr) || isJsSourceFile(ts.getSourceFileOfNode(expr))) return null;
const name = streamWebImportOf(lowerer, expr) ?? stdlibGlobalNameOf(lowerer, expr);
if (name === null || !TYPED_CLASSES.has(name) || !programUsesTypedWebStreams(lowerer))
return null;
return CLASS_NAMES.indexOf(name);
}
export function webStreamClassValue(kind: number, loc: SrcLoc): IrExpr {
return { kind: "libCall", fn: "webstream.class", args: [numLit(kind, loc)], type: DYN, loc };
}
@@ -102,13 +160,24 @@ export function lowerWebStreamClassRead(lowerer: Lowerer, node: ts.Expression):
return kind === null ? null : webStreamClassValue(kind, locOf(node));
}
/** `new X(...args)` on one of the classes in JavaScript. */
/** `new X(...args)` on one of the classes in JavaScript, and on
* ReadableStream, WritableStream and TransformStream in a TypeScript
* program that uses the web stream classes (programUsesTypedWebStreams).
* A TypeScript source's object literal argument (a source, sink,
* transformer or strategy) becomes a plain object with its methods. */
export function lowerWebStreamNew(lowerer: Lowerer, expr: ts.NewExpression): IrExpr | null {
const kind = webStreamClassKindOf(lowerer, expr.expression);
const typed = webStreamClassKindOf(lowerer, expr.expression) === null;
const kind = typed
? typedWebStreamClassKindOf(lowerer, expr.expression)
: webStreamClassKindOf(lowerer, expr.expression);
if (kind === null) return null;
const nodes = expr.arguments?.nodes ?? [];
if (nodes.some(ts.isSpreadElement)) return null;
const loc = locOf(expr);
const argument = (arg: ts.Expression): IrExpr =>
typed && ts.isObjectLiteralExpression(arg)
? lowerDynObjectLiteral(lowerer, arg)
: lowerer.lowerExprExpecting(arg, DYN);
return {
kind: "libCall",
fn: "dyn.construct",
@@ -116,7 +185,7 @@ export function lowerWebStreamNew(lowerer: Lowerer, expr: ts.NewExpression): IrE
webStreamClassValue(kind, loc),
{
kind: "dynArrLit",
elems: nodes.map((arg) => lowerer.lowerExprExpecting(arg, DYN)),
elems: nodes.map(argument),
type: DYN,
loc,
},
@@ -244,10 +313,29 @@ function isReadableStreamType(lowerer: Lowerer, type: ts.Type): boolean {
);
}
/** Whether a member access resolves to a member of the stdlib
* ReadableStream interface. */
function isReadableStreamMember(lowerer: Lowerer, access: ts.PropertyAccessExpression): boolean {
const member = lowerer.checker.getSymbolAtLocation(access.name);
type MemberAccess = ts.PropertyAccessExpression | ts.ElementAccessExpression;
/** The member a property access names, or an element access with a string
* literal key (`body.tee` and `body["tee"]`); null for any other node. */
function staticMemberName(node: ts.Node): string | null {
if (ts.isPropertyAccessExpression(node)) return node.name.text;
if (ts.isElementAccessExpression(node)) {
const key = node.argumentExpression;
if (ts.isStringLiteral(key) || ts.isNoSubstitutionTemplateLiteral(key)) return key.text;
}
return null;
}
/** Whether a member access (staticMemberName) resolves to a member of the
* stdlib ReadableStream interface. */
function isReadableStreamMember(lowerer: Lowerer, access: ts.Node): boolean {
const key = ts.isPropertyAccessExpression(access)
? access.name
: ts.isElementAccessExpression(access)
? access.argumentExpression
: null;
if (key === null) return false;
const member = lowerer.checker.getSymbolAtLocation(key);
if (member === undefined || !lowerer.isStdlibSymbol(member)) return false;
let owner: ts.Node | undefined = member.declarations[0]?.parent;
while (owner !== undefined && !ts.isInterfaceDeclaration(owner)) owner = owner.parent;
@@ -285,6 +373,7 @@ function programLinksFetch(functions: IrFunction[]): boolean {
* count in a program that links the fetch unit, since the value can be a
* fetch body. */
function programUsesWebStreamOnlyMembers(lowerer: Lowerer, functions: IrFunction[]): boolean {
if (programUsesTypedWebStreams(lowerer)) return true;
const members = staticFetchMembers().readableStreamUnsupported;
const iterates = members.has("[Symbol.asyncIterator]");
let untyped = false;
@@ -293,9 +382,10 @@ function programUsesWebStreamOnlyMembers(lowerer: Lowerer, functions: IrFunction
let found = false;
ts.walkPreorder(file, (node) => {
let receiver: ts.Type | null = null;
if (ts.isPropertyAccessExpression(node) && members.has(node.name.text)) {
const member = staticMemberName(node);
if (member !== null && members.has(member)) {
if (isReadableStreamMember(lowerer, node)) found = true;
else receiver = lowerer.typeOf(node.expression);
else receiver = lowerer.typeOf((node as MemberAccess).expression);
} else if (iterates && ts.isForOfStatement(node) && node.awaitModifier !== undefined) {
receiver = lowerer.typeOf(node.expression);
if (isReadableStreamType(lowerer, receiver)) found = true;
@@ -310,7 +400,9 @@ function programUsesWebStreamOnlyMembers(lowerer: Lowerer, functions: IrFunction
}
/** Links the web stream classes into a program whose JavaScript uses a
* ReadableStream member the fetch unit's own stream lacks: a fetch body's
* ReadableStream member the fetch unit's own stream lacks, or whose
* TypeScript uses the web stream classes (programUsesTypedWebStreams): a
* fetch body's or a native stream's
* handle stands for a web ReadableStream once such a member is used
* (scr_fetch.c, sf_stream_web_view). %main installs the classes first. A
* program that names one of the classes links them already. */
@@ -630,6 +630,41 @@ export function checkDefaultParamBodyType(
}
}
/** Whether a lambda's OWN signature maps to its completed ABI position for
* position — the contract its reads carry (mapType spells `x?: T` as the
* `T | undefined` slot). Two positions can escape the mapped-TYPE
* comparison in requireExactArityValue: a parameter the checker spells
* `any` holds the checked-dynamic value that mapType leaves unmapped when
* its spelling is context-free, and a *contextually* typed lambda's own
* signature need not be the slot's (a comparator with a surplus optional
* parameter under a two-parameter slot). */
function lambdaSpellsCompletedAbi(
lowerer: Lowerer,
lambda: ts.ArrowFunction | ts.FunctionExpression,
funcType: IrType,
): boolean {
if (funcType.kind !== "func" || funcType.rest === true) return false;
const sigs = lowerer.checker.getCallSignatures(lowerer.typeOf(lambda));
if (sigs.length !== 1) return false;
const sig = sigs[0]!;
const params = sig.getParameters();
if (params.length !== funcType.params.length) return false;
for (let i = 0; i < params.length; i++) {
const pTs = lowerer.checker.getTypeOfSymbol(params[i]!);
const slot = funcType.params[i]!;
if ((pTs.flags & ts.TypeFlags.Any) !== 0) {
if (slot.kind !== "dyn") return false;
continue;
}
const mappedParam = lowerer.mapTypeOf(pTs);
if (!mappedParam || !typeEquals(mappedParam, slot)) return false;
}
const retTs = lowerer.checker.getReturnTypeOfSignature(sig);
if ((retTs.flags & ts.TypeFlags.Any) !== 0) return funcType.ret.kind === "dyn";
const mappedRet = lowerer.mapTypeOf(retTs);
return mappedRet !== null && typeEquals(mappedRet, funcType.ret);
}
/** DECISION (docs/ir.md): function VALUES carry one completed native ABI.
* Optional/default parameters appear as literal `T | undefined` slots;
* typed rest parameters appear as one trailing typed-array slot plus the
@@ -675,6 +710,20 @@ export function requireExactArityValue(
? lowerer.mapTypeOf(lowerer.typeOf(contextual))
: null;
if (mapped && typeEquals(mapped, funcType)) return;
// A lambda whose OWN signature already spells the completed ABI is a
// value: every read of it types it that way, and its slot converts it
// (the surplus-parameter and undefined-arm adapters). The mapped-type
// comparison above misses two such lambdas — one whose `any` parameter
// mapType leaves unmapped (`(a: any, b: any, loose?: boolean) => number`
// beside a two-parameter slot), and one whose signature is wider than
// the contextual slot it is written under.
if (
contextual !== null &&
(ts.isArrowFunction(contextual) || ts.isFunctionExpression(contextual)) &&
lambdaSpellsCompletedAbi(lowerer, contextual, funcType)
) {
return;
}
// An 'any' slot (`C.prototype.m = function (a, b) {…}` through an
// untyped receiver), or a slot whose type maps to a checked-dynamic
// value (a member of a literal lowered with a generic function's type
@@ -1062,6 +1111,9 @@ export function lambdaSignature(
type: DYN,
mode: "required",
}));
// `arguments` holds the call's own tuple: a console line passes one
// or two arguments whatever the replacement declares.
appendArgumentsParameter(lowerer, node, shapes, "arguments");
return { shapes, funcType: funcTypeFromParamShapes(shapes, DYN) };
}
}
@@ -38,6 +38,7 @@ import { isCjsRequireGlobal } from "./lower-node-module.js";
import { DYN, type IrExpr } from "../../ir/ir.js";
import { nodeThrowExpr } from "../../ir/build.js";
import { probeNodeRequireRefusal } from "../npm.js";
import { lowerIslandRequireCall } from "./lower-node-island.js";
export interface InlineRequire {
call: ts.CallExpression;
@@ -537,6 +538,9 @@ export function lowerUncompiledRequireCall(
);
}
if (canonicalBuiltinModule(spec) !== null) return null;
// A package that runs in a Node island loads at the call (scope B).
const island = lowerIslandRequireCall(lowerer, call);
if (island !== null) return island;
if (!spec.startsWith(".") && !spec.startsWith("/") && !spec.startsWith("#")) {
const refusal = probeNodeRequireRefusal(sf.fileName, spec);
if (refusal !== null) {
@@ -100,6 +100,21 @@ export function initializeRuntimeStatics(
}
if (!member.name) continue;
const memberLoc = locOf(member);
// A private static field: its native global, assigned in order.
if (ts.isPrivateIdentifier(member.name)) {
const name = member.name.text;
const field = info.staticFields.find((candidate) => candidate.name === name);
if (!ts.isPropertyDeclaration(member) || !field) {
lowerer.unsupported("SC1090", member, "private static fields without an initializer");
}
statements.push({
kind: "assign",
localId: field.globalId,
value: lowerer.lowerExprExpecting(member.initializer!, field.type),
loc: memberLoc,
});
continue;
}
const key: IrExpr = ts.isComputedPropertyName(member.name)
? computedKeys.get(member)!
: lowerer.coerceToExpected(
@@ -114,10 +114,28 @@ export function coercibleValue(lowerer: Lowerer, src: IrType, dst: IrType): bool
return false;
}
/** Whether an omitted parameter position completes with the native
* undefined value: a checked-dynamic slot takes undefined as-is, and an
* undefined-armed union (`x?: T`, `x: T = e`) is the completed ABI
* spelling. */
export function omittableTrailingParam(lowerer: Lowerer, type: IrType): boolean {
if (type.kind === "dyn" || typeEquals(type, UNDEFINED_T)) return true;
return type.kind === "union" && lowerer.armTag(type.unionId, UNDEFINED_T) >= 0;
}
/** Whether all parameters and the result adapt without a trap-only path.
* Rest signatures are excluded, except a typed-rest source in a slot with
* fixed parameters (the rest packing adapter); extra destination
* parameters are ignored. */
* parameters are ignored. A source with MORE parameters than the slot is
* clean when every extra trailing one is omittable — the adapter passes
* that absent value explicitly (funcCoerceAdapter's omittedParams), the
* same way a call through the slot would. A parameter the slot coercion
* converts through a STRUCTURAL width counts (cleanParamPair), which
* recurses through a nested function type: a callback parameter of the
* value taking a narrower record where the slot's takes its `unknown` view
* converts the argument the same way a top-level call does. The result side
* likewise accepts what funcCoerceAdapter converts (cleanResult), including
* a promise whose payload widens. */
export function cleanFuncAdaptable(
lowerer: Lowerer,
src: IrType & { kind: "func" },
@@ -125,13 +143,35 @@ export function cleanFuncAdaptable(
): boolean {
if (restPackingAdapterFits(lowerer, src, dst)) return true;
if (src.rest === true || dst.rest === true) return false;
if (src.params.length > dst.params.length) return false;
for (let i = 0; i < src.params.length; i++) {
if (!lowerer.coercibleValue(dst.params[i]!, src.params[i]!)) return false;
// Recursive function types (`type Handler = (h: Handler) => void`) reach
// the same pair again one parameter down: assume it converts, the same
// coinductive answer recordWidthPlan gives its own back-edges.
const key = `funcClean:${typeKey(src)}:${typeKey(dst)}`;
if (lowerer.coercions.planning.has(key)) return true;
lowerer.coercions.planning.add(key);
try {
for (let i = dst.params.length; i < src.params.length; i++) {
if (!omittableTrailingParam(lowerer, src.params[i]!)) return false;
}
for (let i = 0; i < src.params.length && i < dst.params.length; i++) {
if (!cleanParamPair(lowerer, dst.params[i]!, src.params[i]!)) return false;
}
return cleanResult(lowerer, src.ret, dst.ret);
} finally {
lowerer.coercions.planning.delete(key);
}
if (dst.ret.kind === "void") return true;
if (lowerer.coercibleValue(src.ret, dst.ret)) return true;
return src.ret.kind === "void" && dst.ret.kind === "dyn";
}
/** Whether the slot's coercion converts the value's RESULT without a
* trap-only path: the dispositions funcCoerceAdapter's result side accepts
* (a void slot drops the value; coercibleValue; a structural width, which
* includes the promise payload widening and a nested function adaptation;
* the void-into-unknown drop). */
function cleanResult(lowerer: Lowerer, actual: IrType, expected: IrType): boolean {
if (expected.kind === "void") return true;
if (lowerer.coercibleValue(actual, expected)) return true;
if (actual.kind === "void" && expected.kind === "dyn") return true;
return lowerer.widthLiftPlan(actual, expected) !== null;
}
/** Whether the slot's argument type `actual` converts into the function's
@@ -56,6 +56,7 @@ import { typeKey } from "../../type-mapper.js";
import {
lowerArrayCallback,
requireProducedArrayElement,
unitOnlyArrayElement,
callbackArrayElement,
truthyFilterCallback,
} from "./callback-arguments.js";
@@ -69,6 +70,7 @@ import {
import { dynamicReceiverThrows } from "./dynamic-receivers.js";
import { defaultAfterUndefined, lowerStaticallyUndefinedArgument } from "../optional-arguments.js";
import { checkedFilterNarrowing } from "./filter-predicates.js";
import { omittableTrailingParam } from "../coercions/function-adapters.js";
/** `a.map(fn)` / `a.filter(fn)` / `a.forEach(fn)` desugar to a direct
* call of a synthetic module function — one per method + element/result
@@ -108,7 +110,14 @@ export function lowerArrayHofCall(
}
const callback = booleanFilter
? { fnArg: truthyFilterCallback(lowerer, valueT, loc), arity: 1, unread: false }
: lowerArrayCallback(lowerer, argNode, [valueT], arrayOf(elem));
: lowerArrayCallback(
lowerer,
argNode,
[valueT],
arrayOf(elem),
false,
method === "map" ? (unitOnlyArrayElement(lowerer, call) ?? undefined) : undefined,
);
const { fnArg, arity } = callback;
// Only an array that can hold elements outside its element type needs to
// leave an element its callback ignores unread.
@@ -659,7 +668,14 @@ export function lowerArrayUnionHofCall(
? valueArms[0]!
: { kind: "union", unionId: lowerer.unions.intern(valueArms) };
const argNode = call.arguments.nodes[0]!;
const { fnArg, arity } = lowerArrayCallback(lowerer, argNode, [valueT], receiverT);
const { fnArg, arity } = lowerArrayCallback(
lowerer,
argNode,
[valueT],
receiverT,
false,
method === "map" ? (unitOnlyArrayElement(lowerer, call) ?? undefined) : undefined,
);
const fnRet = fnArg.type.ret;
if (method === "map" && fnRet.kind === "void") {
// `void[]` has no element (see the plain-receiver map path).
@@ -1806,10 +1822,10 @@ export function lowerArrayFlatMapCall(
/** How each arm of a flatMap callback's union result contributes to the
* output: an array arm appends its elements, an array arm the checker
* types `never[]` (an empty literal such as `s ? s : []`) appends nothing,
* and any other arm appends itself. */
* a tuple arm appends its fields, and any other arm appends itself. */
interface FlatMapUnionPlan {
outElem: IrType;
arms: ("spread" | "empty" | "value")[];
arms: ("spread" | "empty" | "tuple" | "value")[];
}
/** The plan for a callback returning `T | U[]`: every non-array arm must
@@ -1827,15 +1843,18 @@ function flatMapUnionPlan(
const outElem = out.elem;
const arms = lowerer.unions.get(fnRet.unionId)?.arms;
if (!arms) return null;
// The `never[]` constituents of the callback's checked result.
// The constituents of the callback's checked result that can hold no
// element: a `never[]` arm, and the empty tuple a bare `[]` arm infers
// when the enclosing call has a contextual array type (`return x ? [a]
// : []` in a `f(): T[]`).
const emptyArms: IrType[] = [];
const signature = lowerer.checker.getCallSignatures(lowerer.typeOf(argNode))[0];
if (signature) {
const ret = lowerer.checker.getReturnTypeOfSignature(signature);
for (const part of ret.isUnionType() ? ts.constituentTypes(ret) : [ret]) {
if (!lowerer.checker.isArrayType(part)) continue;
const [element] = lowerer.checker.getTypeArguments(part as ts.TypeReference);
if (element === undefined || (element.flags & ts.TypeFlags.Never) === 0) continue;
if (!lowerer.checker.isArrayType(part) && !lowerer.checker.isTupleType(part)) continue;
const element = lowerer.checker.getTypeArguments(part as ts.TypeReference)[0];
if (element !== undefined && (element.flags & ts.TypeFlags.Never) === 0) continue;
const mapped = lowerer.mapTypeOf(part);
if (mapped) emptyArms.push(mapped);
}
@@ -1847,12 +1866,19 @@ function flatMapUnionPlan(
if (outElem.kind !== "union") return false;
return lowerer.armTag(outElem.unionId, t) >= 0;
};
const tupleConverts = (arm: IrType & { kind: "record" }): boolean =>
(lowerer.shapes.get(arm.shapeId)?.fields ?? []).every((f) => converts(f.type));
const plan: FlatMapUnionPlan["arms"] = [];
for (const arm of arms) {
if (arm.kind === "array" && emptyArms.some((e) => typeEquals(e, arm))) plan.push("empty");
// An array arm copies its elements as they are stored, a present
// undefined included, so its element type is the result's.
else if (arm.kind === "array" && typeEquals(arm.elem, outElem)) plan.push("spread");
// A contextual array type makes a multi-statement body's `[pair]`
// infer as a one-element TUPLE (`return x ? [pair] : []`), whose
// fields contribute in order.
else if (arm.kind === "record" && lowerer.shapes.get(arm.shapeId)?.tuple && tupleConverts(arm))
plan.push("tuple");
else if (arm.kind !== "array" && converts(arm)) plan.push("value");
else return null;
}
@@ -1908,6 +1934,32 @@ function flatMapUnionHelper(
loc,
};
if (plan.arms[tag] === "empty") return [];
if (plan.arms[tag] === "tuple") {
const tuple = arm as IrType & { kind: "record" };
// The fields are stored by name ("0", "1", "10", "2", …): the
// elements append in index order.
const elements = [...(lowerer.shapes.get(tuple.shapeId)?.fields ?? [])].sort(
(a, b) => Number(a.name) - Number(b.name),
);
return elements.map((field) => {
const read: IrExpr = {
kind: "recordGet",
obj: narrowed,
shapeId: tuple.shapeId,
field: field.name,
type: field.type,
loc,
};
return arrayValueStore(
lowerer,
outRef(),
outLength(),
lowerer.coerceToExpected(read, plan.outElem),
plan.outElem,
loc,
);
});
}
if (plan.arms[tag] === "value") {
// A unit arm's value is the unit itself, which has no narrowed
// representation.
@@ -2642,11 +2694,15 @@ export function lowerArraySortCall(
let fnArg =
undefinedArg ?? lowerOptionalElementComparator(lowerer, argNode, unwrappedArg, elem, arrT);
// Reusable JS comparators often accept checked values. Adapt the array's
// element ABI just as other array callbacks do before sorting.
// element ABI just as other array callbacks do before sorting. A
// comparator declaring MORE parameters than sort passes (`(a, b, loose?)`,
// the imported semver shape) receives the surplus ones as undefined — the
// completion an ordinary two-argument call performs — so it adapts to the
// (a, b) slot like any other wider signature.
if (
fnArg.type.kind === "func" &&
fnArg.type.params.length <= 2 &&
fnArg.type.params.every((param) => lowerer.coercibleValue(elem, param))
fnArg.type.params.slice(2).every((param) => omittableTrailingParam(lowerer, param)) &&
fnArg.type.params.slice(0, 2).every((param) => lowerer.coercibleValue(elem, param))
) {
// Arithmetic on inferred JS values retains a checked result because
// it can produce BigInt. The sort callback still requires a number.
@@ -2655,7 +2711,7 @@ export function lowerArraySortCall(
? F64
: fnArg.type.ret;
const expected = funcOf(
fnArg.type.params.map(() => elem),
fnArg.type.params.slice(0, 2).map(() => elem),
result,
);
if (!typeEquals(fnArg.type, expected)) fnArg = lowerer.coerceToExpected(fnArg, expected);

Some files were not shown because too many files have changed in this diff Show More