trending/scriptc
1
0
Fork 0
mirror of https://github.com/vercel-labs/scriptc.git synced 2026-10-31 07:23:06 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge main

Browse Source
This commit is contained in:
Chris Tate
2026-10-27 23:02:33 -05:00
parent 9e91b7d912 18a47fedd5
commit 82ba2ef6ed
260 changed files with 12599 additions and 548 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/actions/libnode-dev-pack/action.yml
-31
View File
@@ -1,31 +0,0 @@
name: libnode development pack (Linux x64, macOS)
description: >-
Provide a libnode pack for the Node island tests and export
SCRIPTC_LIBNODE: restored from the cache, or on a miss assembled from the
pinned, checksum-verified inputs in packages/libnode-common/dev-pack.json
for the runner's platform (a prebuilt libnode and Node's headers; the
bridge builds in seconds). Temporary until the NI-13 packs exist. Fails
the job if no pack results: the island tests must never skip in CI.
runs:
using: composite
steps:
- name: Name the runner's platform
id: platform
shell: bash
run: node -p '`name=${process.platform}-${process.arch}`' >> "$GITHUB_OUTPUT"
- uses: actions/cache@v4
id: libnode
with:
path: ${{ runner.temp }}/libnode-dev-pack
key: libnode-dev-pack-${{ steps.platform.outputs.name }}-${{ hashFiles('packages/libnode-common/dev-pack.json', 'packages/libnode-common/build-pack.mjs', 'packages/libnode-common/fetch-dev-pack.mjs', 'packages/libnode-common/bridge/**', 'packages/runtime/src/scr_node_bridge.h') }}
- name: Assemble the libnode pack
if: steps.libnode.outputs.cache-hit != 'true'
shell: bash
run: node packages/libnode-common/fetch-dev-pack.mjs --out "$RUNNER_TEMP/libnode-dev-pack"
- name: Export SCRIPTC_LIBNODE
shell: bash
run: |
case "$RUNNER_OS" in macOS) bridge=libscr_node_bridge.dylib ;; *) bridge=libscr_node_bridge.so ;; esac
test -f "$RUNNER_TEMP/libnode-dev-pack/manifest.json"
test -f "$RUNNER_TEMP/libnode-dev-pack/$bridge"
echo "SCRIPTC_LIBNODE=$RUNNER_TEMP/libnode-dev-pack" >> "$GITHUB_ENV"
.github/actions/libnode/action.yml
+64
View File
@@ -0,0 +1,64 @@
name: libnode pack (Linux x64, macOS)
description: >-
Provide the runner's libnode pack for the Node island tests and export
SCRIPTC_LIBNODE. Restores the libnode build that the libnode-packs
workflow cached and assembles @scriptc/libnode-<platform> from it (the
bridge compiles in seconds; Node is never built here). Until that cache
exists for this ref, it falls back to the pinned, checksum-verified
development pack (packages/libnode-common/dev-pack.json). Fails the job if
no pack results: the island tests must never skip in CI.
runs:
using: composite
steps:
- name: Name the runner's pack
id: target
shell: bash
run: |
case "$RUNNER_OS-$RUNNER_ARCH" in
Linux-X64) platform=linux-x64-gnu toolchain=manylinux_2_28-gcc-toolset-14 bridge=libscr_node_bridge.so ;;
macOS-ARM64) platform=darwin-arm64 toolchain=macos-15-xcode-clang bridge=libscr_node_bridge.dylib ;;
macOS-X64) platform=darwin-x64 toolchain=macos-15-xcode-clang bridge=libscr_node_bridge.dylib ;;
*) echo "::error::no libnode pack for $RUNNER_OS $RUNNER_ARCH"; exit 1 ;;
esac
printf 'platform=%s\ntoolchain=%s\nbridge=%s\n' "$platform" "$toolchain" "$bridge" >> "$GITHUB_OUTPUT"
- name: Compute the libnode cache key
id: key
shell: bash
env:
SCRIPTC_LIBNODE_TOOLCHAIN: ${{ steps.target.outputs.toolchain }}
run: node packages/libnode-common/scripts/cache-key.mjs --platform ${{ steps.target.outputs.platform }} >> "$GITHUB_OUTPUT"
- uses: actions/cache/restore@v4
id: libnode
with:
path: .libnode-cache/${{ steps.key.outputs.path }}
key: ${{ steps.key.outputs.key }}
- name: Assemble @scriptc/libnode-${{ steps.target.outputs.platform }}
if: steps.libnode.outputs.cache-hit == 'true'
shell: bash
env:
SCRIPTC_LIBNODE_TOOLCHAIN: ${{ steps.target.outputs.toolchain }}
SCRIPTC_LIBNODE_CACHE: ${{ github.workspace }}/.libnode-cache
run: |
node packages/libnode-${{ steps.target.outputs.platform }}/scripts/build.mjs --from-cache
node packages/libnode-common/scripts/verify.mjs packages/libnode-${{ steps.target.outputs.platform }}
echo "SCRIPTC_LIBNODE=$GITHUB_WORKSPACE/packages/libnode-${{ steps.target.outputs.platform }}" >> "$GITHUB_ENV"
- uses: actions/cache@v4
if: steps.libnode.outputs.cache-hit != 'true'
id: devpack
with:
path: ${{ runner.temp }}/libnode-dev-pack
key: libnode-dev-pack-${{ steps.target.outputs.platform }}-${{ hashFiles('packages/libnode-common/dev-pack.json', 'packages/libnode-common/build-pack.mjs', 'packages/libnode-common/fetch-dev-pack.mjs', 'packages/libnode-common/bridge/**', 'packages/runtime/src/scr_node_bridge.h') }}
- name: Assemble the development pack
if: steps.libnode.outputs.cache-hit != 'true'
shell: bash
run: |
echo "::notice::no cached libnode-packs build for ${{ steps.key.outputs.key }}; using the development pack"
if [ "${{ steps.devpack.outputs.cache-hit }}" != true ]; then
node packages/libnode-common/fetch-dev-pack.mjs --out "$RUNNER_TEMP/libnode-dev-pack"
fi
echo "SCRIPTC_LIBNODE=$RUNNER_TEMP/libnode-dev-pack" >> "$GITHUB_ENV"
- name: Check the pack
shell: bash
run: |
test -f "$SCRIPTC_LIBNODE/manifest.json"
test -f "$SCRIPTC_LIBNODE/${{ steps.target.outputs.bridge }}"
.github/workflows/ci.yml
+4 -4
View File
@@ -215,7 +215,7 @@ jobs:
# Node island tests (`// @requires-libnode` corpus programs and
# node-island.test.ts) load Node from a libnode pack, and fail rather
# than skip in this lane when none is configured.
- uses: ./.github/actions/libnode-dev-pack
- uses: ./.github/actions/libnode
# Ubuntu pipes core dumps to apport, and RLIMIT_CORE does not limit a
# piped dump. The kernel then streams every page of the runtime's
# reserved allocator address space through the pipe, so each expected
@@ -286,7 +286,7 @@ jobs:
- run: pnpm build
# Node island tests load Node from a libnode pack, and fail rather
# than skip on macOS CI when none is configured.
- uses: ./.github/actions/libnode-dev-pack
- uses: ./.github/actions/libnode
- name: Tests (${{ (github.event_name != 'schedule' && inputs.macos_suite != 'full') && 'darwin suite' || 'corpus and integration suites' }})
run: node scripts/ci-test.mjs
@@ -707,7 +707,7 @@ jobs:
- run: pnpm build
# Node island tests load Node from a libnode pack, and fail rather
# than skip on macOS CI when none is configured.
- uses: ./.github/actions/libnode-dev-pack
- uses: ./.github/actions/libnode
- name: Node island harness
if: matrix.shard == 1
run: pnpm test tests/harness/node-island packages/libnode-common
@@ -808,7 +808,7 @@ jobs:
- run: pnpm build
# Node island tests load Node from a libnode pack, and fail rather
# than skip on macOS CI when none is configured.
- uses: ./.github/actions/libnode-dev-pack
- uses: ./.github/actions/libnode
- name: Node islands (Intel)
run: >-
pnpm test tests/harness/node-island packages/libnode-common
.github/workflows/libnode-packs.yml
+145
View File
@@ -0,0 +1,145 @@
name: libnode packs
# Builds the @scriptc/libnode-<platform> packs: the pinned Node.js release
# as a shared library plus the scriptc bridge (packages/libnode-common).
# A Node build takes about half an hour on 8 cores, so this workflow is
# never part of the ordinary build or test path. It runs only when the pin,
# the bridge or the pack packages change, or by hand, and it caches each
# platform's libnode build by (Node release, configure flags, platform,
# toolchain): a bridge-only change restores libnode from the cache and
# rebuilds just the bridge in seconds.
#
# Linux packs build in the manylinux_2_28 images (AlmaLinux 8, glibc 2.28,
# gcc-toolset-14), the glibc floor of Node's official Linux builds, on
# native runners: Node disables its startup snapshot when cross-compiling.
# Every pack is smoke-tested by a C program that loads it the way the
# scriptc runtime does (packages/libnode-common/smoke/smoke.c).
on:
workflow_dispatch:
workflow_call:
push:
branches:
- main
paths:
- "packages/libnode-*/**"
- "packages/runtime/src/scr_node_bridge.h"
- ".github/workflows/libnode-packs.yml"
pull_request:
paths:
- "packages/libnode-*/**"
- "packages/runtime/src/scr_node_bridge.h"
- ".github/workflows/libnode-packs.yml"
concurrency:
# A fixed name: under workflow_call, github.workflow is the caller's, and
# sharing the caller's group would cancel the run as a deadlock.
group: libnode-packs-${{ github.ref }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
permissions:
contents: read
jobs:
build:
name: Build libnode pack (${{ matrix.platform }})
strategy:
fail-fast: false
matrix:
include:
- platform: linux-x64-gnu
runner: ubuntu-24.04
image: quay.io/pypa/manylinux_2_28_x86_64
toolchain: manylinux_2_28-gcc-toolset-14
- platform: linux-arm64-gnu
runner: ubuntu-24.04-arm
image: quay.io/pypa/manylinux_2_28_aarch64
toolchain: manylinux_2_28-gcc-toolset-14
- platform: darwin-arm64
runner: macos-15
toolchain: macos-15-xcode-clang
- platform: darwin-x64
runner: macos-15-intel
toolchain: macos-15-xcode-clang
runs-on: ${{ matrix.runner }}
# A cold Linux build is ~30 minutes on 8 cores; the 3- and 4-core macOS
# runners are expected to take well over an hour. A cache hit is minutes.
timeout-minutes: 240
env:
SCRIPTC_LIBNODE_TOOLCHAIN: ${{ matrix.toolchain }}
steps:
- uses: actions/checkout@v4
- uses: actions/setup-node@v4
with:
node-version: "24"
- name: Compute the libnode cache key
id: key
run: node packages/libnode-common/scripts/cache-key.mjs --platform ${{ matrix.platform }} >> "$GITHUB_OUTPUT"
- name: Restore the libnode build
uses: actions/cache@v4
with:
path: .libnode-cache/${{ steps.key.outputs.path }}
key: ${{ steps.key.outputs.key }}
- name: Build the pack (Linux, manylinux_2_28)
if: startsWith(matrix.platform, 'linux')
run: |
NODE_ROOT=$(dirname "$(dirname "$(command -v node)")")
docker run --rm --volume "$GITHUB_WORKSPACE:/work" --volume "$NODE_ROOT:/node:ro" \
--env SCRIPTC_LIBNODE_TOOLCHAIN --env SCRIPTC_LIBNODE_CACHE=/work/.libnode-cache \
--env CC=gcc --env CXX=g++ \
--env PATH=/opt/rh/gcc-toolset-14/root/usr/bin:/opt/python/cp312-cp312/bin:/node/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \
--workdir /work ${{ matrix.image }} sh -c '
set -eu
dnf install --assumeyes --quiet zstd
pip install --quiet ninja
node packages/libnode-${{ matrix.platform }}/scripts/build.mjs --compress
node packages/libnode-common/scripts/verify.mjs packages/libnode-${{ matrix.platform }}
# The glibc 2.28 floor: load the pack where it was built.
node packages/libnode-common/scripts/smoke.mjs --platform ${{ matrix.platform }}
chown -R "$(stat -c %u:%g /work)" /work/packages /work/.libnode-cache
'
- name: Build the pack (macOS)
if: startsWith(matrix.platform, 'darwin')
env:
CC: clang
CXX: clang++
SCRIPTC_LIBNODE_CACHE: ${{ github.workspace }}/.libnode-cache
run: |
brew install ninja zstd
node packages/libnode-${{ matrix.platform }}/scripts/build.mjs --compress
node packages/libnode-common/scripts/verify.mjs packages/libnode-${{ matrix.platform }}
- name: Smoke-test the pack on the runner
run: node packages/libnode-common/scripts/smoke.mjs --platform ${{ matrix.platform }}
- name: Report sizes
run: |
{
echo "### @scriptc/libnode-${{ matrix.platform }} (Node ${{ steps.key.outputs.node }})"
echo
echo '```'
ls -l packages/libnode-${{ matrix.platform }} .libnode-cache/compressed/${{ matrix.platform }}
cat .libnode-cache/${{ steps.key.outputs.path }}/build.json
echo '```'
} >> "$GITHUB_STEP_SUMMARY"
- name: Upload the pack
uses: actions/upload-artifact@v4
with:
name: libnode-${{ matrix.platform }}
path: |
packages/libnode-${{ matrix.platform }}
!packages/libnode-${{ matrix.platform }}/scripts
retention-days: 30
- name: Upload compressed libraries
uses: actions/upload-artifact@v4
with:
name: libnode-${{ matrix.platform }}-zst
path: .libnode-cache/compressed/${{ matrix.platform }}
retention-days: 30
.gitignore
+6
View File
@@ -33,6 +33,12 @@ output/
/packages/runtime-*/artifacts/
/packages/runtime-*/runtime-pack.json
/packages/runtime-*/.runtime-pack-*
# libnode packs: built by build:native, never committed
/packages/libnode-*/libnode.so.*
/packages/libnode-*/libnode.*.dylib
/packages/libnode-*/libscr_node_bridge.*
/packages/libnode-*/manifest.json
/packages/libnode-*/LICENSE
!tests/fixtures/npm/node_modules/*/dist/
!tests/fixtures/npm/node_modules/@*/*/dist/
!tests/fixtures/npm/workspace/*/dist/
RELEASING.md
+16
View File
@@ -25,3 +25,19 @@ Each published package must have a GitHub Actions trusted publisher for
`release.yml` and the `Release` environment. A missing configuration fails
before upload. Re-runs skip package versions already present on npm, so a
partially published release can be resumed safely.
## libnode packs
The `@scriptc/libnode-<platform>` packs (linux-x64-gnu, linux-arm64-gnu, darwin-arm64, darwin-x64) carry the Node.js shared library that programs with Node islands load in process, plus the scriptc bridge compiled against it. They version in lockstep with the other platform packages (`sync-versions.mjs` stamps them); the Node release they contain is pinned separately in `packages/libnode-common/libnode.json`, which must match `.node-version` and `internal/compatibility/node-v24.json`. Each pack's `manifest.json` records the Node version, `NODE_MODULE_VERSION`, the bridge ABI and the size and SHA-256 of both libraries.
A Node `--shared` build takes about half an hour on 8 cores, so packs never build in `pnpm -r build`, the test suites or the release job. `.github/workflows/libnode-packs.yml` builds them on matching hosts when the pin, the bridge (`packages/libnode-common/bridge/`, `packages/runtime/src/scr_node_bridge.h`) or the pack packages change, or on manual dispatch. Linux packs build in the `manylinux_2_28` images (glibc 2.28, gcc-toolset-14) on native x64 and arm64 runners; Node disables its startup snapshot when cross-compiling, so arm64 is never cross-built. Each platform's libnode build is cached by Node release, configure flags, platform and toolchain; a bridge-only change restores libnode from the cache and rebuilds only the bridge. Every pack is smoke-tested by `packages/libnode-common/smoke/smoke.c`, which loads it the way the runtime does, and uploaded as the `libnode-<platform>` artifact, with zstd copies of both libraries as `libnode-<platform>-zst`. The Linux x64 and macOS test lanes of `ci.yml` restore the cached build for their platform through `.github/actions/libnode`, assemble the pack (`build.mjs --from-cache`, which never builds Node) and set `SCRIPTC_LIBNODE`, so the Node island tests run; without a cached build they fall back to the pinned development pack (`packages/libnode-common/dev-pack.json`). `packages/libnode-common/build-pack.mjs` assembles a pack from any matching libnode and headers, the bridge-only rebuild path.
To build one pack by hand on its own platform (it needs Python 3.9+, Ninja, a C++20 compiler and zstd; on Linux, build in a manylinux_2_28 container for a distributable pack):
```bash
pnpm --filter @scriptc/libnode-linux-x64-gnu build:native
node packages/libnode-common/scripts/verify.mjs packages/libnode-linux-x64-gnu
node packages/libnode-common/scripts/smoke.mjs
```
The packs are `private` until the in-process Node island runtime ships. Publishing them then means removing `private`, configuring their npm trusted publishers, and having the release job download the `libnode-<platform>` artifacts of the workflow run for the release commit (or call the workflow, which restores every platform from its cache) and publish them with the other platform packages.
docs/content/docs/experimental.mdx
+2 -2
View File
@@ -389,9 +389,9 @@ scriptc requires:
- `T` to be a `ProgramLifetime` class, and the field to be an instance field of a `ProgramLifetime` class;
- the constructor of the class that declares the field to write it once, from one of its parameters. A subclass constructor passes one of its own parameters there through `super(...)`. Nothing else writes the field;
- the array passed to the constructor to be one that nothing else keeps: a new array (an array literal, a copy such as `slice()`, `[...items]` or `Array.from(items)`, or the result of a function that returns only such arrays or its own argument), a local array at its last use (calls that only read it or add to it may come first, and a loop may pass it and then assign a new array to the local), a parameter whose callers each pass such an array, an array that an InlineArray field already holds, or a `const` array that is never written, such as a shared empty array;
- the arrays fields hold never to be written, through a read of the field (which the `readonly` type allows only through a cast) or through any other reference. scriptc follows these arrays through locals, parameters, return values, fields, the elements of arrays, Maps and Sets, and closures, and reports one that reaches a write, a dynamic value, a call through a function value, or a library call other than printing and `JSON.stringify` without a replacer. The report names the field read the array may come from;
- the arrays fields hold never to be written, through a read of the field (which the `readonly` type allows only through a cast) or through any other reference. scriptc follows these arrays through locals, parameters, return values, fields, the elements of arrays, Maps and Sets, and closures, and reports one that reaches a write, a dynamic value (also inside the fields of an object that becomes one), a call through a function value (also inside an array, Map or Set passed there), or a library call other than printing and `JSON.stringify` without a replacer. The report names the field read the array may come from;
- the arrays fields hold, and their reads, not to be compared by identity: with `===` or `!==`, as an `indexOf` or `includes` argument, or as a Map key or Set element. Two reads of the field need not give the same array. Compare the elements instead, or keep a copy made with `slice()` where identity matters;
- instances of the class not to be constructed through a class value, and not to become dynamic values (for example through `JSON.stringify` or a conversion to `unknown`).
- instances of the class not to be constructed through a class value, and not to become dynamic values (for example through a conversion to `unknown`), directly or inside another value that can hold one: an instance of a class with such a field, a record, array, Map or Set, or a value typed as a base class. A dynamic view reads the field as a new array each time, so a class whose instances can become dynamic values cannot hold an InlineArray field. Printing, `JSON.stringify` without a replacer, `publish` and a worker's `workerData` only read or copy the value and are allowed. The report names the conversion.
scriptc reports a field that does not keep these rules (SC7009).
packages/cli/src/bootstrap.ts
+4 -1
View File
@@ -70,6 +70,9 @@ async function tryFastPath(): Promise<number | null> {
values["profile-generate"] !== undefined ||
values["profile-use"] !== undefined ||
values.sparse ||
// A --libnode=beside build places the pack and the island modules next
// to the executable, which a routed hit does not do.
values.libnode === "beside" ||
// The full compiler resolves `native` against the build host.
values["target-cpu"] !== undefined ||
(values["external-types"] ?? []).length > 0 ||
@@ -173,7 +176,7 @@ async function tryFastPath(): Promise<number | null> {
: startup.executableNativeEnvironmentFingerprint(),
nodeVersion: process.version,
// An entry of a program that calls into Node.js serves only the same pack.
libnodeIdentity: () => startup.libnodeCacheKey(values.libnode),
libnodeIdentity: () => startup.libnodeCacheKey(values.libnode, process.env, dirname(input)),
});
if (hit === null) return null;
if (!values["keep-llvm"]) await rm(hit.llvmPath, { force: true });
packages/cli/test/bootstrap.test.ts
+55
View File
@@ -123,6 +123,61 @@ test("bootstrap exact builds use the routed cache and source edits fall through"
}
}, 120_000);
test.skipIf(!persistentExecutable)(
"bootstrap leaves --libnode=beside builds to the full compiler",
async () => {
const dir = await mkdtemp(join(tmpdir(), "scriptc-bootstrap-beside-"));
const entry = join(dir, "main.ts");
const outPath = join(dir, "program");
const preload = join(dir, "reject-full-compiler.mjs");
const env = { ...process.env, SCRIPTC_CACHE_DIR: join(dir, "cache") };
delete env.SCRIPTC_NO_CACHE;
const build = (args: string[], rejectFullCompiler = false) =>
execFileAsync(
process.execPath,
[
...(rejectFullCompiler ? ["--import", preload] : []),
bootstrap,
"build",
entry,
"-o",
outPath,
...args,
],
{ env, maxBuffer: 4 * 1024 * 1024 },
);
try {
await Promise.all([
writeFile(entry, 'console.log("beside");\n'),
writeFile(
preload,
[
"import { registerHooks } from 'node:module';",
"registerHooks({ load(url, context, nextLoad) {",
" if (url.includes('/packages/compiler/dist/index.js')) throw new Error('compiler graph loaded');",
" return nextLoad(url, context);",
"}});",
"",
].join("\n"),
),
]);
for (const args of [[], ["--libnode=beside"]]) {
await build(args);
await build(args);
}
await expect(build([], true)).resolves.toBeDefined();
// A routed hit would skip placing the pack and the island modules.
await expect(build(["--libnode=beside"], true)).rejects.toMatchObject({
stderr: expect.stringContaining("compiler graph loaded"),
});
expect((await execFileAsync(outPath)).stdout).toBe("beside\n");
} finally {
await rm(dir, { recursive: true, force: true });
}
},
120_000,
);
test.skipIf(!runtimePackHost)(
"bootstrap cache hits reuse LLVM executables",
async () => {
packages/compiler/native/host.c
+14
View File
@@ -73,12 +73,14 @@ void scriptc_native_collect_cycles(void) {
int32_t scriptc_native_fork(void) { return -1; }
void scriptc_native_exit(int32_t code) { ExitProcess((UINT)code); }
int32_t scriptc_native_wait(int32_t pid) { (void)pid; return -1; }
int32_t scriptc_native_exited(void) { return -1; }
uint8_t scriptc_native_lock_open(const uint8_t *bytes, size_t size) { (void)bytes; (void)size; return 0; }
void scriptc_native_lock(void) {}
void scriptc_native_unlock(void) {}
#else
#include <errno.h>
#include <fcntl.h>
#include <signal.h>
#include <sys/file.h>
#include <sys/wait.h>
@@ -119,6 +121,18 @@ int32_t scriptc_native_wait(int32_t pid) {
return -1;
}
/* The pid of a child that has ended, waiting for one if none has, or -1
* when waitid fails. The child is left to scriptc_native_wait: a child
* another part of the process waits for is not reaped here. */
int32_t scriptc_native_exited(void) {
siginfo_t info;
memset(&info, 0, sizeof info);
while (waitid(P_ALL, 0, &info, WEXITED | WNOWAIT) < 0) {
if (errno != EINTR) return -1;
}
return info.si_pid > 0 ? (int32_t)info.si_pid : -1;
}
static int scriptc_native_lock_fd = -1;
uint8_t scriptc_native_lock_open(const uint8_t *bytes, size_t size) {
packages/compiler/native/host.ffi.json
+1
View File
@@ -6,6 +6,7 @@
{ "name": "compilerNativeFork", "symbol": "scriptc_native_fork", "params": [], "returns": "i32" },
{ "name": "compilerNativeExit", "symbol": "scriptc_native_exit", "params": ["i32"], "returns": "void" },
{ "name": "compilerNativeWait", "symbol": "scriptc_native_wait", "params": ["i32"], "returns": "i32" },
{ "name": "compilerNativeExited", "symbol": "scriptc_native_exited", "params": [], "returns": "i32" },
{ "name": "compilerNativeLockOpen", "symbol": "scriptc_native_lock_open", "params": ["string"], "returns": "bool" },
{ "name": "compilerNativeLock", "symbol": "scriptc_native_lock", "params": [], "returns": "void" },
{ "name": "compilerNativeUnlock", "symbol": "scriptc_native_unlock", "params": [], "returns": "void" },
packages/compiler/src/backend/executable-features.ts
+3 -1
View File
@@ -43,7 +43,9 @@ export function executableLinkFeatures(mod: IrModule, largePages = false): Nativ
...(features.dynBuffer ? { dynBuffer: true } : {}),
...(features.builtinValues ? { builtinValues: true } : {}),
...(features.computedRequire ? { computedRequire: true } : {}),
events: features.processEvents,
// Island code's signal and exit listeners join the events unit's
// registries (scope-B NI-3).
events: features.processEvents || features.nodeIsland === true,
emitter: features.emitter,
symbol: features.symbol,
bigint: features.bigint,
packages/compiler/src/backend/llvm/emit-module.ts
+68 -17
View File
@@ -115,24 +115,56 @@ export function workerMinFunctions(): number {
return Number.isFinite(value) && value > 0 ? value : 2000;
}
/** A function's estimated emission cost beyond its IR nodes, in nodes:
* fitted to the worker emission times of a large program's ranges. */
const FUNCTION_EMIT_COST = 22;
/** Where each of about `count` contiguous ranges of `functions` starts,
* followed by the end: ranges of about equal estimated emission cost, a
* function's node count plus FUNCTION_EMIT_COST. No range is empty. */
export function emitRangeStarts(
functions: readonly IrFunction[],
count: number,
nodeCount: (fn: IrFunction) => number | undefined,
): number[] {
const costs = functions.map((fn) => (nodeCount(fn) ?? 0) + FUNCTION_EMIT_COST);
const total = costs.reduce((sum, cost) => sum + cost, 0);
const starts = [0];
let sum = 0;
for (let i = 0; i < functions.length - 1 && starts.length < count; i++) {
sum += costs[i]!;
if (sum >= (total * starts.length) / count) starts.push(i + 1);
}
starts.push(functions.length);
return starts;
}
/** The function bodies emitted on forked worker processes, `workers` at a
* time: the function list splits into contiguous ranges, each emitted in
* its own process, which inherits this emitter as it is at the fork and
* records a journal (emit-journal.ts). Ranges merge in function order, so
* the text is the serial text. A range whose worker fails is emitted here
* when its turn comes. Null without fork primitives (Node, Windows, inside
* a worker) or when the module is small. */
* time: the function list splits into contiguous ranges of about equal
* estimated cost (emitRangeStarts), each emitted in its own process, which
* inherits this emitter as it is at the fork and records a journal
* (emit-journal.ts). A new range starts whenever a worker ends, and ranges
* merge in function order, so the text is the serial text. A range whose
* worker fails is emitted here when its turn comes. Null without fork
* primitives (Node, Windows, inside a worker) or when the module is small. */
function emitFunctionsOnWorkers(emitter: LlEmitter, workers: number): string[] | null {
const fork = emitter.targetOptions.forkPrimitives;
const functions = emitter.mod.functions;
if (fork === undefined || workers < 2) return null;
if (functions.length < workerMinFunctions()) return null;
// More ranges than workers: a range's merge overlaps the later ranges'
// emission, and a slow range holds back less.
const count = Math.min(functions.length, workers * 3);
const size = Math.ceil(functions.length / count);
// emission, and a slow range holds back less. Ranges of equal cost finish
// at about the same time.
const starts = emitRangeStarts(functions, Math.min(functions.length, workers * 3), (fn) =>
emitter.nodeIndex.nodeCount(fn),
);
const count = starts.length - 1;
const rangeFunctions = (range: number): readonly IrFunction[] =>
functions.slice(starts[range], starts[range + 1]);
const dir = mkdtempSync(join(tmpdir(), "scriptc-emit-"));
const pids: number[] = [];
// The workers not yet reaped, by pid, with their ranges.
const running = new Map<number, number>();
const start = (range: number): void => {
const pid = fork.fork();
if (pid === 0) {
@@ -141,7 +173,7 @@ function emitFunctionsOnWorkers(emitter: LlEmitter, workers: number): string[] |
emitter.journal = new EmitJournal();
emitter.debug?.record();
const texts: string[] = [];
for (const fn of functions.slice(range * size, (range + 1) * size)) {
for (const fn of rangeFunctions(range)) {
texts.push(emitter.emitFunction(fn));
relieveHeapPressure();
}
@@ -155,17 +187,37 @@ function emitFunctionsOnWorkers(emitter: LlEmitter, workers: number): string[] |
fork.exit(code);
}
pids.push(pid);
if (pid > 0) running.set(pid, range);
};
const fnDefs: string[] = [];
// Ended ranges by index, with what their workers wrote (null when a worker
// failed), waiting to merge in function order.
const ended = new Map<number, { result: ChunkResult; texts: string[] } | null>();
// Reaps a worker and starts the next range in its place. A worker's journal
// records requests, not table contents, so its range's text doesn't depend
// on what merged before it forked.
const reap = (pid: number, range: number): void => {
const status = pid > 0 ? fork.wait(pid) : -1;
running.delete(pid);
if (pids.length < count) start(pids.length);
ended.set(range, status === 0 ? readRange(dir, range) : null);
};
try {
for (let range = 0; range < Math.min(workers, count); range++) start(range);
for (let range = 0; range < count; range++) {
const pid = pids[range]!;
const status = pid > 0 ? fork.wait(pid) : -1;
// The next worker starts from this emitter as merged so far, which
// the journal allows: it records requests, not table contents.
const merged = status === 0 ? readRange(dir, range) : null;
const slice = functions.slice(range * size, (range + 1) * size);
// Until this range's worker ends, reap whichever worker ends first, so
// that `workers` keep running while ranges merge in order. A child
// that isn't a worker (or no `exited`) waits for this range's worker.
while (!ended.has(range)) {
const pid = pids[range]!;
const first = pid > 0 && fork.exited !== undefined ? fork.exited() : pid;
const other = running.get(first);
if (first !== pid && other !== undefined) reap(first, other);
else reap(pid, range);
}
const merged = ended.get(range)!;
ended.delete(range);
const slice = rangeFunctions(range);
if (merged !== null && merged.texts.length === slice.length) {
const rename = emitter.mergeChunk(merged.result);
slice.forEach((fn, index) => {
@@ -180,7 +232,6 @@ function emitFunctionsOnWorkers(emitter: LlEmitter, workers: number): string[] |
fnDefs.push(def);
relieveHeapPressure();
}
if (pids.length < count) start(pids.length);
}
} finally {
rmSync(dir, { recursive: true, force: true });
packages/compiler/src/backend/llvm/emitter.ts
+4
View File
@@ -516,6 +516,10 @@ export interface ForkPrimitives {
exit(code: number): void;
/** The worker's exit status, 128 plus a signal number, or -1. */
wait(pid: number): number;
/** The pid of a child process that has ended, waiting for one if none
* has, or -1, without reaping it. Without it, workers are waited for in
* order. */
exited?(): number;
}
export interface LlvmTargetOptions {
packages/compiler/src/backend/llvm/lib-shared.ts
+22
View File
@@ -1045,6 +1045,17 @@ export const LIB_FN_SYMS: Record<string, string> = {
"json.stringifyValue": "scr_json_stringify_value",
"nodeIsland.callSync": "scr_node_island_call_sync",
"nodeIsland.call": "scr_node_island_call",
"nodeIsland.require": "scr_ni_require",
"nodeIsland.import": "scr_ni_import",
"nodeIsland.get": "scr_node_island_get",
"nodeIsland.getKey": "scr_ni_get",
"nodeIsland.set": "scr_node_island_set",
"nodeIsland.setKey": "scr_node_island_set_key",
"nodeIsland.invoke": "scr_node_island_invoke",
"nodeIsland.apply": "scr_node_island_apply",
"nodeIsland.applyTo": "scr_node_island_apply_to",
"nodeIsland.construct": "scr_node_island_construct",
"nodeIsland.instanceof": "scr_node_island_instanceof",
"dyn.keySet": "scr_dyn_key_set",
"dyn.keySetComputed": "scr_dyn_key_set_computed",
"dyn.keyDelete": "scr_dyn_key_delete",
@@ -2022,4 +2033,15 @@ export const USES_TIMERS_LIB_FNS = new Set<string>([
// drives (scr_node_island.c) before the program exits.
"nodeIsland.call",
"nodeIsland.callSync",
"nodeIsland.require",
"nodeIsland.import",
"nodeIsland.get",
"nodeIsland.getKey",
"nodeIsland.set",
"nodeIsland.setKey",
"nodeIsland.invoke",
"nodeIsland.apply",
"nodeIsland.applyTo",
"nodeIsland.construct",
"nodeIsland.instanceof",
]);
packages/compiler/src/backend/native/executable.ts
+4 -2
View File
@@ -209,7 +209,9 @@ const RUNTIME_UNITS: readonly RuntimeUnit[] = [
u.opts.workers ||
u.opts.dc ||
u.opts.fileHandle ||
u.nativeFetch,
u.nativeFetch ||
// Island promises cross as dyn promise boxes (scr_node_island_handle.c).
u.opts.nodeIsland === true,
"scr_async_dyn.c",
),
// The performance timeline (every perf_hooks user-timing call selects
@@ -369,7 +371,7 @@ const RUNTIME_UNITS: readonly RuntimeUnit[] = [
unitArgs((u) => (u.tlsCa && targetPlatform(u.driver) === "win32" ? ["-lcrypt32"] : [])),
// Fetch uses the native network, TLS, and HTTP runtime.
unit((u) => u.nativeFetch, "scr_fetch.c"),
unit((u) => u.opts.nodeIsland === true, "scr_node_island.c"),
unit((u) => u.opts.nodeIsland === true, "scr_node_island.c", "scr_node_island_handle.c"),
// Readable.fromWeb reads a web ReadableStream through its reader.
unit((u) => u.nativeFetch && u.opts.stream, "scr_stream_from_web.c"),
];
packages/compiler/src/cli/command.ts
+1
View File
@@ -665,6 +665,7 @@ async function main(args: string[], host: CliHost): Promise<number> {
? {}
: { profileUse: resolve(values["profile-use"]) }),
...(npmStatic !== undefined ? { npmStatic } : {}),
...(nodeIslands !== undefined ? { nodeIslands } : {}),
...(values["shape-overlay"] === false ? { shapeOverlay: false } : {}),
...(ffiProfilePath !== undefined ? { ffiProfilePath } : {}),
...(printNativeLinkInfo ? { nativeLinkInfo: true } : {}),
packages/compiler/src/compile-types.ts
+3
View File
@@ -11,6 +11,9 @@ import type { NodeIslandOptions } from "./node-islands/policy.js";
export type CompileOutputKind = "ir" | "llvm" | "asm" | "obj" | "exe";
export interface CompileBaseOptions {
/** The Node island policy's command-line half (`--node-islands`,
* `--node-island`, `--native`): which npm packages run in Node islands. */
nodeIslands?: NodeIslandOptions;
/** Primary artifact path. The CLI supplies the output-kind default. */
outPath: string;
/** Where generated intermediates and compatibility side artifacts land. */
packages/compiler/src/contracts/check.ts
+2 -1
View File
@@ -373,10 +373,11 @@ class ContractChecker {
);
for (const r of plan.rejected) {
const note = notes.get(`${r.declaringClass}\0${r.field}`)!;
const at = r.loc === undefined ? "" : ` (at ${this.where(r.loc)})`;
this.report(
"SC7009",
note.loc,
`'${note.name}' cannot keep its InlineArray contract: ${r.reason}`,
`'${note.name}' cannot keep its InlineArray contract: ${r.reason}${at}`,
"write the field once in the constructor from one of its parameters (subclasses pass theirs through super(...)), never again, and keep the class's instances out of class values and dynamic values",
);
}
packages/compiler/src/diagnostics/codes.ts
+2 -2
View File
@@ -108,11 +108,13 @@ export const DIAGNOSTIC_CODES = {
SC7007: "Acyclic type can lie on a reference cycle",
SC7008: "ProgramLifetime class extends another class",
SC7009: "InlineArray field contract not kept",
SC8001: "package or module runs in a Node island (note)",
SC8002: "Node islands off for a package or module outside the native subset",
SC8003: "package marked native is outside the native subset",
SC8004: "unknown scriptc.packages or scriptc.nodeIslands configuration",
SC8005: "no usable libnode pack for a program that calls into Node.js",
SC8006: "Node island modules the build cannot collect",
SC8010: "value crossing a Node island boundary in a form without an exact copy",
SC8030: "type patch: no range covers the installed copy",
SC8031: "type patch: selector matches nothing",
SC8032: "type patch: declaration already has the type",
@@ -150,8 +152,6 @@ export const RETIRED_DIAGNOSTIC_CODES = {
* that work lands it moves its codes into DIAGNOSTIC_CODES; when it is
* abandoned, delete them here. */
export const RESERVED_DIAGNOSTIC_CODES = {
SC8001: "Node islands (ni/* branches)",
SC8010: "Node islands (ni/* branches)",
SC8011: "Node islands (ni/* branches)",
SC8012: "Node islands (ni/* branches)",
SC8020: "Node islands (ni/* branches)",
packages/compiler/src/diagnostics/diagnostic.ts
+33
View File
@@ -116,6 +116,11 @@ export interface ScrDiagnostic {
* alone does not say it (a computed require(), a native addon, `with`).
* Report-only: never part of the rendered message. */
reason?: string;
/** Runtime fences only: the fenced site is in a module's top-level code
* (outside every function body), so it runs, and throws, when the module
* evaluates. Island selection treats such a fence like one that needs the
* runtime (scope-B NI-4). */
moduleInit?: true;
/** A runtime fence in a module's top-level code, outside every function
* and class member body: it throws when the module initializes, which is
* program startup for a module the entry imports. */
@@ -357,6 +362,34 @@ export function nodeIslandsOffModuleDiag(
};
}
/** SC8001 — a package runs in a Node island (a note, once per island in
* the build output). */
export function nodeIslandNoteDiag(
pkg: string,
version: string | null,
klass: string,
reason: string,
loc: SrcLoc,
): ScrDiagnostic {
return {
code: "SC8001",
message: `package '${version === null ? pkg : `${pkg}@${version}`}' runs in a Node island: ${klass}: ${reason}`,
loc,
hint: "it runs in Node.js loaded into the program on first use; --node-islands=off makes this an error",
};
}
/** SC8010 — a value crosses an island boundary in a way NI-4 cannot keep
* exact yet (callbacks, native objects, remote promises). */
export function nodeIslandBoundaryDiag(what: string, loc: SrcLoc): ScrDiagnostic {
return {
code: "SC8010",
message: `${what} cannot cross into or out of a Node island yet`,
loc,
hint: "pass primitives, strings, island objects or fresh object and array literals of those; callbacks, native objects and promises cross from NI-5 on",
};
}
/** SC8030–SC8033 — a type patch (`scriptc.typePatches`) that did not
* apply as written: no range covers the installed copy (SC8030), its
* selector matches nothing (SC8031), the declaration already has the type
packages/compiler/src/executable/executable-cache.ts
+4
View File
@@ -185,6 +185,8 @@ export interface EarlyExecutableCacheOptions {
/** Omitted unless SCRIPTC_HEAP_CENSUS is set (backend/llvm/heap-census.ts). */
heapCensus?: true;
npmStatic: readonly string[] | "auto" | null;
/** Omitted for the default Node island policy; else nodeIslandOptionsKey. */
nodeIslands?: string;
/** Omitted unless shape inference's overlay compiles the packages
* (frontend/shape-inference). */
shapeOverlay?: true;
@@ -341,6 +343,7 @@ function cacheKey(options: EarlyExecutableCacheOptions): string {
...(options.functionLayout !== undefined ? [`layout:${options.functionLayout}`] : []),
...(options.checker !== undefined ? [`checker:${options.checker}`] : []),
...(options.youngNursery ? ["young-nursery"] : []),
...(options.nodeIslands !== undefined ? [`node-islands:${options.nodeIslands}`] : []),
...(options.heapCensus ? ["heap-census"] : []),
options.npmStatic === null
? "<npm-static-off>"
@@ -389,6 +392,7 @@ function routeKey(options: Omit<EarlyExecutableRouteOptions, "nativeEnvironment"
if (options.functionLayout !== undefined) hash.update(`layout:${options.functionLayout}\0`);
if (options.checker !== undefined) hash.update(`checker:${options.checker}\0`);
if (options.youngNursery) hash.update("young-nursery\0");
if (options.nodeIslands !== undefined) hash.update(`node-islands:${options.nodeIslands}\0`);
if (options.heapCensus) hash.update("heap-census\0");
hash
.update(
packages/compiler/src/executable/prepare.ts
+348 -7
View File
@@ -3,6 +3,7 @@ import type { FfiProfile } from "../ffi/ffi-manifest.js";
import { frontendDisposed } from "../frontend/dispose-hook.js";
import {
checkerPanicDiag,
nodeIslandNoteDiag,
npmStaticRefusalDiag,
iceDiag,
isCheckerPanic,
@@ -19,6 +20,16 @@ import {
import type { LowerResult, LowerStats } from "../frontend/lowering/lowerer.js";
import type { FrontendFactory } from "../frontend/pipeline.js";
import type { CompilationTiming } from "../timing.js";
import { readProjectConfig } from "../project-config.js";
import { nodeIslandPolicy } from "../node-islands/policy.js";
import { buildNodeIslandReport } from "../node-islands/report.js";
import type { IslandPackage, NodeIslandSet } from "../node-islands/boundary.js";
import { classifyDiagnostic, isIslandReasonCode } from "../node-islands/classify.js";
import { packageDirOf, packageVersionOf } from "../node-islands/origin.js";
import { describe } from "../node-islands/summary.js";
import { resolveBareModule } from "../frontend/resolve.js";
import { isEsmFile } from "../frontend/lowering/lower-node-island.js";
import type { PackageEdge } from "../node-islands/scan.js";
export interface PreparedExecutableModule {
ok: true;
@@ -38,11 +49,30 @@ export function prepareExecutableModule(
buildPlatform: string,
createFrontend: FrontendFactory,
timing: CompilationTiming = () => {},
lowerIslands: ReadonlyMap<string, IslandPackage> = new Map(),
retryFences = true,
): PreparedExecutableModule | CompileFailure {
const fe = createFrontend(entryPath, opts.npmStatic);
// The island decision reads the detector's scan unless islands are off.
const islandScan =
nodeIslandPolicy(readProjectConfig(entryPath), opts.nodeIslands).mode !== "off";
let fe = createFrontend(entryPath, opts.npmStatic, undefined, undefined, { islandScan });
timing("frontend-load");
// Node islands (scope-B NI-4): packages refused natively run in Node,
// unless the policy says otherwise. The frontend reloads without them,
// so their declarations type the program.
const islandDecision = decideNodeIslands(entryPath, opts, fe, lowerIslands);
if (islandDecision !== null) {
fe.dispose();
fe = createFrontend(entryPath, opts.npmStatic, undefined, undefined, {
islandPackages: new Set(islandDecision.packages.keys()),
islandScan,
});
timing("frontend-island-reload");
}
let lowered: LowerResult;
let fenceEdges: readonly PackageEdge[] | undefined;
let sourceTexts: Map<string, string>;
let disposed = false;
// The frontend (and its tsgo server) is released as soon as lowering
// ends — clang and the link never hold it open.
try {
@@ -65,10 +95,32 @@ export function prepareExecutableModule(
),
);
}
if (fe.preflight.length > 0) return fail(fe.preflight);
if (fe.preflight.length > 0) {
// An import of a natively compiled package that preflight refuses
// in the importer (a CommonJS namespace import) is that package's
// refusal: it runs in a Node island instead.
const grown = preflightIslands(entryPath, opts, fe, lowerIslands);
if (grown !== null) {
fe.dispose();
frontendDisposed();
disposed = true;
return prepareExecutableModule(
entryPath,
opts,
ffi,
buildPlatform,
createFrontend,
timing,
grown,
retryFences,
);
}
return fail(fe.preflight);
}
try {
lowered = fe.lower({
...(islandDecision !== null ? { nodeIslands: islandDecision } : {}),
targetPlatform: buildPlatform,
...(ffi !== null ? { ffiImports: ffi.functions } : {}),
...(opts.tier !== undefined ? { tierMode: opts.tier } : {}),
@@ -83,7 +135,35 @@ export function prepareExecutableModule(
checkerPanicDiag(e.message.split("\n", 1)[0]!, { file: entryPath, start: 0, end: 0 }),
]);
}
if (lowered.module === null) return fail(lowered.diagnostics);
if (lowered.module === null) {
// Packages whose own modules refused to lower run in Node islands
// too (the authoritative lowering, DESIGN.md §1.4): build again with
// them as islands.
const grown = loweringIslands(
entryPath,
opts,
fe,
lowered.diagnostics,
lowerIslands,
fe.nodeIslandScan?.({ userJs: false }).edges,
);
if (grown !== null) {
fe.dispose();
frontendDisposed();
disposed = true;
return prepareExecutableModule(
entryPath,
opts,
ffi,
buildPlatform,
createFrontend,
timing,
grown,
retryFences,
);
}
return fail(lowered.diagnostics);
}
const validation = validateModule(lowered.module);
timing("ir-validate");
@@ -104,6 +184,10 @@ export function prepareExecutableModule(
return fail([targetRefusalDiag("wasm32-wasi", unavailable.surface, unavailable.loc)]);
}
}
// The package import edges island selection reads after lowering, taken
// while the frontend is open.
if (retryFences && lowered.runtimeFences.some(sureFence))
fenceEdges = fe.nodeIslandScan?.({ userJs: false }).edges;
sourceTexts = fe.sourceTexts();
if (lowered.contracts) {
const checked = checkContracts(lowered.module, lowered.contracts, sourceTexts);
@@ -112,20 +196,277 @@ export function prepareExecutableModule(
if (checked.contracts) lowered.module.contracts = checked.contracts;
}
} finally {
fe.dispose();
frontendDisposed();
timing("frontend-dispose");
if (!disposed) {
fe.dispose();
frontendDisposed();
timing("frontend-dispose");
}
}
return {
const prepared: PreparedExecutableModule = {
ok: true,
mod: lowered.module!,
sourceTexts,
stats: lowered.stats,
warnings: [
...(islandDecision?.notes ?? []),
...(fe.warnings ?? []),
...(lowered.divergences ?? []),
...(lowered.performance ?? []),
],
};
// Runtime fences in a compiled package that surely fire: fences that need
// the JavaScript runtime itself (R1: code from strings, computed
// import()/require(), addons, loader hooks), and fences in top-level
// module code, which throw when the module evaluates. Build again with
// those packages as islands, and keep this build when that one cannot
// compile. A gap fence inside a function body may never run, so it stays
// a deferred refusal.
const runtimeFences = lowered.runtimeFences.filter(sureFence);
if (retryFences && runtimeFences.length > 0) {
const grown = loweringIslands(entryPath, opts, fe, runtimeFences, lowerIslands, fenceEdges);
if (grown !== null) {
const islanded = prepareExecutableModule(
entryPath,
opts,
ffi,
buildPlatform,
createFrontend,
timing,
grown,
false,
);
if (islanded.ok) return islanded;
}
}
return prepared;
}
/** A run-time fence that surely fires: it needs the JavaScript runtime
* itself, or it is in top-level module code. */
function sureFence(d: ScrDiagnostic): boolean {
return d.moduleInit === true || classifyDiagnostic(d, "fence").class === "runtime";
}
/** The packages a build runs in Node islands, or null for none: under
* `--node-islands=auto` (the default) every npm package the native
* pipeline refused, plus packages marked `node`; under `explicit` only the
* marked ones; under `off`, none (refusals stay SC2013). Each island is
* reported once as an SC8001 note. */
function decideNodeIslands(
entryPath: string,
opts: CompileRequestOptions,
fe: ReturnType<FrontendFactory>,
lowerIslands: ReadonlyMap<string, IslandPackage>,
): (NodeIslandSet & { notes: ScrDiagnostic[] }) | null {
const policy = nodeIslandPolicy(readProjectConfig(entryPath), opts.nodeIslands);
if (policy.mode === "off" || fe.nodeIslandScan === undefined) return null;
// A package marked native stays refused: the build reports it rather
// than running it in Node.
const refused = new Set(
policy.mode === "auto"
? fe.npmStatic
.filter((e) => e.status === "refused" && !policy.forcedNative.has(e.package))
.map((e) => e.package)
: [],
);
for (const name of lowerIslands.keys()) refused.add(name);
if (refused.size === 0 && policy.forcedNode.size === 0) return null;
const report = buildNodeIslandReport({
entry: entryPath,
scan: fe.nodeIslandScan({ userJs: false }),
npm: fe.npmStatic,
diagnostics: [],
fences: [],
policy,
});
const packages = new Map<string, IslandPackage>();
const notes: ScrDiagnostic[] = [];
const seen = new Set<string>();
for (const island of report.islands) {
if (island.path === null) continue;
seen.add(island.package);
if (!refused.has(island.package) && !policy.forcedNode.has(island.package)) continue;
packages.set(island.package, {
name: island.package,
version: island.version,
dir: island.path,
class: island.class,
reason: island.reason,
});
notes.push(
nodeIslandNoteDiag(
island.package,
island.version,
island.class,
island.reason,
island.importedFrom[0] ??
fe.npmImportSites.get(island.package) ?? { file: entryPath, start: 0, end: 0 },
),
);
}
// Refused packages the report has no installed directory for (refused
// before they joined the program): their directory from the runtime
// entry Node resolves.
for (const name of refused) {
if (packages.has(name) || lowerIslands.has(name)) continue;
const site = fe.npmImportSites.get(name) ?? { file: entryPath, start: 0, end: 0 };
const js =
resolveBareModule(site.file, name, "js-only") ??
resolveBareModule(site.file, name, "js-only", "require");
const at = js === null ? null : packageDirOf(js.typesFile);
if (at === null) continue;
const status = fe.npmStatic.find((e) => e.package === name);
packages.set(name, {
name,
version: packageVersionOf(at.dir),
dir: at.dir,
class: "packaging",
reason: status?.detail ?? "its static compilation was refused",
});
notes.push(
nodeIslandNoteDiag(name, packageVersionOf(at.dir), "packaging", status?.detail ?? "", site),
);
}
// Packages islanded by their lowering refusals keep their first reason
// (the reload no longer compiles them, so the scan has no reason).
for (const [name, island] of lowerIslands) {
if (seen.has(name) && packages.has(name)) {
const kept = packages.get(name)!;
if (kept.reason === "" || kept.class === "") packages.set(name, island);
continue;
}
packages.set(name, island);
notes.push(
nodeIslandNoteDiag(
island.name,
island.version,
island.class,
island.reason,
fe.npmImportSites.get(name) ?? { file: entryPath, start: 0, end: 0 },
),
);
}
return packages.size === 0 ? null : { packages, notes };
}
/** The island set grown by the npm packages whose own modules refused to
* lower (`--node-islands=auto` only), or null when there are none. */
function loweringIslands(
entryPath: string,
opts: CompileRequestOptions,
fe: ReturnType<FrontendFactory>,
diagnostics: readonly ScrDiagnostic[],
known: ReadonlyMap<string, IslandPackage>,
edges: readonly PackageEdge[] | undefined,
): Map<string, IslandPackage> | null {
const policy = nodeIslandPolicy(readProjectConfig(entryPath), opts.nodeIslands);
if (policy.mode !== "auto" || known.size > 16) return null;
const compiled = new Set(fe.npmStatic.filter((e) => e.status === "static").map((e) => e.package));
const grown = new Map(known);
for (const d of diagnostics) {
// A use of a package outside the compiled module graph (an import()
// the frontend did not compile): that package runs in an island.
const named = d.code === "SC2013" ? /npm package '([^']+)'/.exec(d.message) : null;
if (named !== null) {
const name = named[1]!;
if (grown.has(name) || policy.forcedNative.has(name) || policy.unloaded.has(name)) continue;
const js =
resolveBareModule(d.loc.file, name, "js-only") ??
resolveBareModule(d.loc.file, name, "js-only", "require");
const at = js === null ? null : packageDirOf(js.typesFile);
if (at === null || at.name !== name) continue;
grown.set(name, {
name,
version: packageVersionOf(at.dir),
dir: at.dir,
class: "packaging",
reason: `${d.code}: ${d.message}`,
});
continue;
}
if (!isIslandReasonCode(d.code)) continue;
const at = packageDirOf(d.loc.file);
if (at === null || !compiled.has(at.name) || grown.has(at.name)) continue;
if (policy.forcedNative.has(at.name) || policy.unloaded.has(at.name)) continue;
const reason = classifyDiagnostic(d, "refusal");
grown.set(at.name, {
name: at.name,
version: packageVersionOf(at.dir),
dir: at.dir,
class: reason.class,
reason: describe(reason),
});
}
// A native ES module that imports an island CommonJS module would read
// its bindings from the module's exports object, where Node gives it the
// namespace its CommonJS facade builds (the lexed names, module.exports
// as the default): the importing package runs in the island too.
for (let added = edges !== undefined; added;) {
added = false;
for (const edge of edges!) {
const name = edge.to;
if (!grown.has(name) || edge.fromDir === null || edge.shim) continue;
const at = packageDirOf(edge.fromFile);
if (at === null || grown.has(at.name) || !compiled.has(at.name)) continue;
if (policy.forcedNative.has(at.name) || policy.unloaded.has(at.name)) continue;
if (!isEsmFile(edge.fromFile)) continue;
const target =
resolveBareModule(edge.fromFile, name, "js-only") ??
resolveBareModule(edge.fromFile, name, "js-only", "require");
if (target === null || isEsmFile(target.typesFile)) continue;
grown.set(at.name, {
name: at.name,
version: packageVersionOf(at.dir),
dir: at.dir,
class: "packaging",
reason: `an ES module of the package imports the CommonJS island package '${name}'`,
});
added = true;
}
}
return grown.size === known.size ? null : grown;
}
/** Preflight refusals at import statements of natively compiled npm
* packages (`--node-islands=auto` only): those packages become islands. */
function preflightIslands(
entryPath: string,
opts: CompileRequestOptions,
fe: ReturnType<FrontendFactory>,
known: ReadonlyMap<string, IslandPackage>,
): Map<string, IslandPackage> | null {
const policy = nodeIslandPolicy(readProjectConfig(entryPath), opts.nodeIslands);
if (policy.mode !== "auto" || known.size > 16) return null;
const compiled = new Set(fe.npmStatic.filter((e) => e.status === "static").map((e) => e.package));
if (compiled.size === 0) return null;
const texts = fe.sourceTexts();
const grown = new Map(known);
for (const d of fe.preflight) {
if (d.code !== "SC1013") continue;
const text = texts.get(d.loc.file);
if (text === undefined) continue;
// The import statement around the diagnostic: its module specifier.
const start = text.lastIndexOf("import", d.loc.start);
const end = text.indexOf(";", d.loc.end);
const stmt = text.slice(start < 0 ? d.loc.start : start, end < 0 ? text.length : end);
const m = /\bfrom\s*["']([^"']+)["']/.exec(stmt);
if (m === null) continue;
const spec = m[1]!;
const name = spec.startsWith("@") ? spec.split("/").slice(0, 2).join("/") : spec.split("/")[0]!;
if (!compiled.has(name) || grown.has(name)) continue;
if (policy.forcedNative.has(name) || policy.unloaded.has(name)) continue;
const site = fe.npmImportSites.get(name);
const jsEntry = site === undefined ? null : resolveBareModule(site.file, spec, "js-only");
const at = jsEntry === null ? null : packageDirOf(jsEntry.typesFile);
if (at === null) continue;
grown.set(name, {
name,
version: packageVersionOf(at.dir),
dir: at.dir,
class: "gap",
reason: `${d.code}: ${d.message}`,
});
}
return grown.size === known.size ? null : grown;
}
packages/compiler/src/frontend/lowering/builtins/module-resolution.ts
+7 -2
View File
@@ -194,9 +194,13 @@ function createRequireFence(
surface: string,
node: ts.Node,
hint: string,
reason?: string,
): IrStmt & { kind: "runtimeFence" } {
const loc = locOf(node);
const d = noLoweringDiag(surface, loc, hint);
// The Node island reason the refusal would carry (a computed require is
// dynamic module loading), so island selection classifies the fence.
if (reason !== undefined) d.reason = reason;
lowerer.runtimeFences.push(d);
const line = ts.getLineAndCharacterOfPosition(ts.getSourceFileOfNode(node), loc.start).line + 1;
return {
@@ -299,6 +303,7 @@ function deferredCreateRequireThrow(
surface: string,
hint: string,
loc: SrcLoc,
reason?: string,
): IrExpr {
const stmts: IrStmt[] = call.arguments.nodes.map((arg) => ({
kind: "exprStmt",
@@ -310,7 +315,7 @@ function deferredCreateRequireThrow(
slot.mutable = true;
stmts.push(
{ kind: "varDecl", localId: slot.id, init: null, loc },
createRequireFence(lowerer, surface, call, hint),
createRequireFence(lowerer, surface, call, hint, reason),
);
return {
kind: "seqExpr",
@@ -339,7 +344,7 @@ export function lowerCreateRequireCall(
// Spread arguments have no run-time resolution: they run, then the
// call throws the fence.
if (args.some((a) => ts.isSpreadElement(a)))
return deferredCreateRequireThrow(lowerer, call, surface, hint, loc);
return deferredCreateRequireThrow(lowerer, call, surface, hint, loc, "dynamic-require");
// Node evaluates every argument, then requires the first.
const stmts: IrStmt[] = [];
let id = args.length > 0 ? lowerer.lowerExprExpecting(args[0]!, DYN) : dynUndefinedExpr(loc);
packages/compiler/src/frontend/lowering/lower-exprs.ts
+8
View File
@@ -12,6 +12,7 @@ import {
lowerNewTargetName,
} from "./new-target.js";
import { lowerWebAssemblyValue, refuseWebAssemblyUse } from "./builtins/webassembly.js";
import { lowerIslandExpr } from "./lower-node-island.js";
import { lowerAwaitedUniformPromiseAll } from "./builtins/promises.js";
import { dynUndefinedExpr, nodeThrowExpr, numLit, strLit, varRef } from "../../ir/build.js";
import { InternalCompilerError } from "../../errors.js";
@@ -289,6 +290,13 @@ function lowerExprInner(lowerer: Lowerer, expr: ts.Expression): IrExpr {
const chainRecv = lowerer.chainRecvByNode.get(expr);
if (chainRecv) return { ...chainRecv, loc };
// Node islands (scope-B NI-4): member operations on island values and
// island import bindings. Programs without islands skip it.
if (lowerer.nodeIslands !== null) {
const island = lowerIslandExpr(lowerer, expr);
if (island !== null) return island;
}
// --external-types supplies CHECKER truth only. A value rooted in one
// of those declarations has no runtime implementation in scriptc, so
// every use fences at its expression instead of accidentally lowering
packages/compiler/src/frontend/lowering/lower-modules.ts
+46 -3
View File
@@ -1,6 +1,7 @@
import { computedGlobalReadOf, globalValuesInstall } from "./global-values.js";
import type { TypeFailureSlot } from "../../diagnostics/diagnostic.js";
import { THREADS_MODULE } from "../threads-import.js";
import { collectIslandImport } from "./lower-node-island.js";
import { NODE_API_MODULE } from "../node-api-import.js";
import { positionTag } from "./structural-positions.js";
import { EXPERIMENTAL_MODULE } from "../experimental-import.js";
@@ -34,11 +35,13 @@ import {
locOf,
makeCycleAdmission,
orderedImportsOf,
npmStaticDepSf7,
requireSpecOf,
resolveImport,
resolveNpmImport,
} from "../program.js";
import type { CycleEdge } from "../program.js";
import type { CycleEdge, StartupCrash } from "../program.js";
import { isUnloadedPackage } from "../package-shims.js";
import {
markCycleEarlyBindings,
withClassInitFlag,
@@ -175,6 +178,8 @@ export interface FileParts {
export interface AppendGraph {
edges?: ReadonlyMap<ts.SourceFile, readonly CycleEdge[]> | undefined;
inlineCycleCalls?: Set<ts.Node>;
/** LoadResult.dynamicImportRejections: import() roots that never load. */
rejected?: ReadonlyMap<ts.SourceFile, StartupCrash> | undefined;
}
/** An appended module's edges: its import and export declarations, then
@@ -440,7 +445,8 @@ export function appendDynamicImportModules(
if (scan.length === 0) break;
for (const sf of scan) {
scanned.add(sf);
for (const dep of dynamicProgramImportsOf(program, sf)) visit(dep);
for (const dep of dynamicProgramImportsOf(program, sf))
if (graph?.rejected?.has(dep) !== true) visit(dep);
}
}
if (added.length > 0) order.splice(order.length - 1, 0, ...added);
@@ -648,7 +654,13 @@ export function collectNpmImports(lowerer: Lowerer, parts: FileParts[]): void {
// graph: resolveImport follows its node_modules link to the loaded
// TypeScript entry. Its internal `.js` spellings have already been
// resolved to `.ts` by that graph.
if (!isRelativeSpecifier(spec) && resolveImport(lowerer.program, fp.sf, spec) !== null) {
const projectDep = isRelativeSpecifier(spec)
? null
: resolveImport(lowerer.program, fp.sf, spec);
if (projectDep !== null) {
// A Node island package that tsconfig `paths` maps to its installed
// directory resolves to its declarations: it loads in Node.
if (projectDep.isDeclarationFile) collectIslandImport(lowerer, fp.sf, stmt, spec);
continue;
}
const npm = resolveNpmImport(fp.sf.fileName, spec);
@@ -700,6 +712,9 @@ export function collectNpmImports(lowerer: Lowerer, parts: FileParts[]): void {
// it, %main throws before any init runs, and the bindings never
if (npm !== null && probeNodeImportRefusal(fp.sf.fileName, spec) !== null) continue;
// A Node island package (scope-B NI-4): loaded in Node at this
// module's position, its bindings are island values.
if (collectIslandImport(lowerer, fp.sf, stmt, spec)) continue;
{
lowerer.pushDiag(unsupportedPackageImportDiag(npm?.packageName ?? relPkg!, locOf(stmt)));
continue;
@@ -722,6 +737,27 @@ export function collectNpmImports(lowerer: Lowerer, parts: FileParts[]): void {
* specifiers are skipped — the lowering owns that fence (the module
* graph is a build-time artifact; there is nothing to embed for a
* runtime-computed name). */
/** The refusal a literal import() rejects with, or null when it loads:
* Node's ERR_MODULE_NOT_FOUND for a package the project marks never loaded
* (package-shims.ts), or the first refusal in the static graph of a module
* only import() loads (LoadResult.dynamicImportRejections). */
function dynamicImportRejectionOf(
lowerer: Lowerer,
sf: ts.SourceFile,
spec: string,
call: ts.CallExpression,
): StartupCrash | null {
if (isUnloadedPackage(spec)) {
const refusal = probeNodeImportRefusal(sf.fileName, spec);
if (refusal === null) return null;
return { message: refusal.message, className: "%Error", code: refusal.code, loc: locOf(call) };
}
if (lowerer.dynamicImportRejections.size === 0) return null;
const root =
resolveImport(lowerer.program, sf, spec) ?? npmStaticDepSf7(lowerer.program, sf, spec);
return root === null ? null : (lowerer.dynamicImportRejections.get(root) ?? null);
}
function collectDynamicImports(lowerer: Lowerer, sf: ts.SourceFile): void {
ts.walkPreorder(sf, (node) => {
if (
@@ -742,6 +778,13 @@ function collectDynamicImports(lowerer: Lowerer, sf: ts.SourceFile): void {
// module has no runtime namespace object — the per-site fence
// says so. Only non-declaration source files count; a sibling
// .d.mts typing shipped JS is the EMBED case, not this one.
// A package the project marks never loaded, or a module whose
// static graph Node's resolution refuses: the import() rejects.
const rejection = dynamicImportRejectionOf(lowerer, sf, spec, node);
if (rejection !== null) {
lowerer.dynImports.set(mapKey, { kind: "rejected", crash: rejection });
return undefined;
}
const modSym = lowerer.checker.getSymbolAtLocation(node.arguments.nodes[0]);
const ownModule =
modSym !== undefined &&
packages/compiler/src/frontend/lowering/lower-namespaces.ts
+4
View File
@@ -41,6 +41,7 @@ import * as ts from "../ts7/adapter.js";
import { isPackageShimFile, isShimIntrinsicName, packageShimFor } from "../package-shims.js";
import type { Lowerer } from "./lowerer.js";
import { boundIdentifiersOf } from "./lowerer.js";
import { isIslandImportBinding } from "./lower-node-island.js";
import type { FileParts } from "./lower-modules.js";
import { isCjsJsFile, isJsSourceFile, locOf, resolveImport } from "../program.js";
import { F64, type IrExpr, type IrStmt, type IrType, STRING } from "../../ir/ir.js";
@@ -686,6 +687,9 @@ export function ambientUndefinedVarSymbolOf(
lowerer: Lowerer,
ident: ts.Identifier,
): ts.Symbol | null {
// An island package's `declare const` is defined by its JavaScript, which
// runs in the island.
if (isIslandImportBinding(lowerer, ident)) return null;
const sym = lowerer.resolveValueSymbol(ident);
const decl = sym ? sym.declarations[0] : undefined;
if (
packages/compiler/src/frontend/lowering/lower-node-island.ts
+789 -4
View File
@@ -16,14 +16,35 @@
* checked against the declared result type, like `JSON.parse(s) as T`; a
* thrown Error crosses as its nearest intrinsic class with its message and
* string `code`. */
import { realpathSync } from "node:fs";
import { isAbsolute } from "node:path";
import { readFileSync, realpathSync } from "node:fs";
import { resolveFrom } from "../../node-islands/resolve.js";
import { dirname, isAbsolute, join } from "node:path";
import * as ts from "../ts7/adapter.js";
import { locOf } from "../program.js";
import type { NodeApiMember } from "../node-api-import.js";
import { PoisonError, type Lowerer } from "./lowerer.js";
import { nodeIslandModuleDiag } from "../../diagnostics/diagnostic.js";
import { DYN, STRING, type IrExpr, type IrType } from "../../ir/ir.js";
import {
BOOL,
DYN,
FOREIGN,
STRING,
type IrExpr,
type IrGlobal,
type IrStmt,
type IrType,
type SrcLoc,
} from "../../ir/ir.js";
import { varRef } from "../../ir/build.js";
import { resolveBareModule } from "../resolve.js";
import { nodeIslandBoundaryDiag, nodeIslandModuleDiag } from "../../diagnostics/diagnostic.js";
import {
crossingOf,
isForeignType,
islandPackageOfFile,
type NodeIslandSet,
} from "../../node-islands/boundary.js";
import { stripTypeWrappers } from "./typed-arguments.js";
import { thenReader } from "./checked-promise-all.js";
const JSON_HINT =
"only JSON data (numbers, strings, booleans, null, arrays, plain objects of those, and unknown) crosses into Node.js in this version";
@@ -88,6 +109,7 @@ export function lowerNodeApiCall(
const loc = locOf(call);
const surface = `${member} (@scriptc/node)`;
const argNodes = call.arguments.nodes;
if (member === "requireModule") return lowerRequireModule(lowerer, call, surface);
if (call.questionDotToken || argNodes.length < 2 || argNodes.some(ts.isSpreadElement)) {
lowerer.noLowering(
`${surface} without a specifier and an export name, or with spread arguments`,
@@ -155,3 +177,766 @@ export function lowerNodeApiCall(
// `JSON.parse(s) as T`: types don't lie, a mismatch throws.
return lowerer.coerceToExpected(raw, declared);
}
/** requireModule(specifier): Node's require() from the calling module, run
* in the island; the exports cross by reference (scope-B NI-4: an island
* handle the program uses as an `any` value). The specifier must be a
* string literal: it resolves at build time exactly as require.resolve
* resolves it from the caller, and the island loads that file, shipped
* with the program (islandLoadPath). */
function lowerRequireModule(lowerer: Lowerer, call: ts.CallExpression, surface: string): IrExpr {
const loc = locOf(call);
const argNodes = call.arguments.nodes;
const arg = argNodes[0];
if (
call.questionDotToken ||
argNodes.length !== 1 ||
arg === undefined ||
!ts.isStringLiteral(arg)
) {
lowerer.noLowering(
`${surface} without exactly one string-literal specifier`,
call,
"pass the module specifier as a string literal",
);
}
const from = callerPath(call);
let path: string;
try {
path = resolveFrom(from, arg.text);
} catch {
lowerer.noLowering(
`${surface} of '${arg.text}', which does not resolve from ${from}`,
call,
"install the module or fix the specifier",
);
}
lowerer.usesNodeIsland = true;
const raw: IrExpr = {
kind: "libCall",
fn: "nodeIsland.require",
args: [islandLoadPath(lowerer, path, loc)],
type: DYN,
loc,
};
const declared = lowerer.mapTypeOf(lowerer.typeOf(call));
if (declared === null || declared === undefined || declared.kind === "dyn") return raw;
return lowerer.coerceToExpected(raw, declared);
}
/* ── compiler-managed Node islands (scope-B NI-4) ─────────────────────────
*
* A package the build runs in a Node island is never compiled. Its static
* imports load the island module at the importing module's position
* (eager evaluation: `nodeIsland.require` in the module's %init prelude,
* before the body), and their bindings read island values: a named import
* of an ES module reads the live namespace member at each use; a CommonJS
* module's named imports are read once after it evaluates, as Node's
* namespace for CommonJS snapshots them; a default import of CommonJS is
* module.exports. `require()` of an island package anywhere and `import()`
* load the same way at the call.
*
* Island values are typed by the package's declarations: values of
* island-declared object types are island handles (FOREIGN, see
* node-islands/boundary.ts), and member reads, writes, calls and `new` on
* them become bridge calls whose results meet the declared type through
* the checked dyn conversion (types don't lie: a mismatch throws). */
/** One island module load: the module global that holds what the import
* form sees (module.exports for CommonJS, the namespace for ESM). */
export interface IslandModuleBinding {
global: IrGlobal;
/** The JavaScript file Node loads. */
path: string;
esm: boolean;
/** CommonJS named imports, snapshotted after evaluation: name -> global. */
snapshots: Map<string, IrGlobal>;
}
function boundaryFail(lowerer: Lowerer, what: string, node: ts.Node): never {
lowerer.pushDiag(nodeIslandBoundaryDiag(what, locOf(node)));
throw new PoisonError();
}
/** Whether the JavaScript file is an ES module by Node's rules. */
export function isEsmFile(file: string): boolean {
if (file.endsWith(".mjs")) return true;
if (file.endsWith(".cjs")) return false;
for (let dir = dirname(file); ;) {
try {
const pkg = JSON.parse(readFileSync(join(dir, "package.json"), "utf8")) as {
type?: unknown;
};
return pkg.type === "module";
} catch {
/* no package.json here */
}
const parent = dirname(dir);
if (parent === dir) return false;
dir = parent;
}
}
/** The island module file `spec` names from `fromFile`, or null when the
* specifier does not resolve into an island package. */
export function islandModuleFile(
islands: NodeIslandSet,
fromFile: string,
spec: string,
kind: "import" | "require",
): string | null {
const js = resolveBareModule(fromFile, spec, "js-only", kind);
if (js === null) return null;
if (islandPackageOfFile(islands, js.typesFile) === null) return null;
try {
return realpathSync(js.typesFile);
} catch {
return js.typesFile;
}
}
/** The module global of the island module at `path`, created on first use. */
function islandModuleGlobal(lowerer: Lowerer, path: string): IslandModuleBinding {
let binding = lowerer.islandModules.get(path);
if (binding === undefined) {
const global: IrGlobal = {
id: `%g.island.${lowerer.globalsList.length}`,
name: path.split("/").at(-1) ?? "island",
type: DYN,
mutable: true,
};
lowerer.globalsList.push(global);
binding = { global, path, esm: isEsmFile(path), snapshots: new Map() };
lowerer.islandModules.set(path, binding);
}
return binding;
}
/** The path literal of a load of the file at `path`. The build ships the
* file with the program (node-islands/modules.ts): the literal names it by
* its path in the shipped tree once lowering ends. A built-in module keeps
* its name. */
function islandLoadPath(lowerer: Lowerer, path: string, loc: SrcLoc): IrExpr {
const literal: IrExpr & { kind: "strLit" } = { kind: "strLit", value: path, type: STRING, loc };
if (!isAbsolute(path)) return literal;
let real = path;
try {
real = realpathSync(path);
} catch {
/* the resolved path */
}
const problem = lowerer.nodeIslandModules.addLoaded(real);
if (problem !== null) {
lowerer.pushDiag(nodeIslandModuleDiag(problem, loc));
throw new PoisonError();
}
lowerer.nodeIslandLoads.push({ literal, path: real });
return literal;
}
function loadIslandModule(lowerer: Lowerer, path: string, loc: SrcLoc): IrExpr {
return {
kind: "libCall",
fn: "nodeIsland.require",
args: [islandLoadPath(lowerer, path, loc)],
type: DYN,
loc,
};
}
/** A static import (or re-export) of an island package: loads the module
* in the importer's %init prelude. Returns false when `spec` is not an
* island module. */
export function collectIslandImport(
lowerer: Lowerer,
sf: ts.SourceFile,
stmt: ts.Statement,
spec: string,
): boolean {
const islands = lowerer.nodeIslands;
if (islands === null) return false;
const path = islandModuleFile(islands, sf.fileName, spec, "import");
if (path === null) return false;
if (ts.isExportDeclaration(stmt)) {
lowerer.pushDiag(
nodeIslandBoundaryDiag(`re-exporting the Node island module '${spec}'`, locOf(stmt)),
);
return true;
}
lowerer.usesNodeIsland = true;
const loc = locOf(stmt);
const binding = islandModuleGlobal(lowerer, path);
const actions = lowerer.jsonInitActions.get(sf) ?? [];
lowerer.jsonInitActions.set(sf, actions);
actions.push({
kind: "assign",
localId: binding.global.id,
value: loadIslandModule(lowerer, path, loc),
loc,
});
// CommonJS named imports: Node's namespace snapshots them once the
// module evaluated.
const clause = ts.isImportDeclaration(stmt) ? stmt.importClause : undefined;
const named = clause?.namedBindings;
if (!binding.esm && named !== undefined && ts.isNamedImports(named)) {
for (const el of named.elements.nodes) {
if (el.isTypeOnly) continue;
const name = el.propertyName?.text ?? el.name.text;
if (name === "default") continue;
let g = binding.snapshots.get(name);
if (g === undefined) {
g = {
id: `%g.island.${lowerer.globalsList.length}`,
name,
type: DYN,
mutable: true,
};
lowerer.globalsList.push(g);
binding.snapshots.set(name, g);
}
actions.push({
kind: "assign",
localId: g.id,
value: islandGet(varRef(binding.global.id, DYN, loc), name, loc),
loc,
});
}
}
return true;
}
function islandGet(target: IrExpr, key: string, loc: SrcLoc): IrExpr {
return {
kind: "libCall",
fn: "nodeIsland.get",
args: [target, { kind: "strLit", value: key, type: STRING, loc }],
type: DYN,
loc,
};
}
type IslandBinding =
| { kind: "namespace"; module: IslandModuleBinding }
| { kind: "default"; module: IslandModuleBinding }
| { kind: "named"; module: IslandModuleBinding; name: string };
function moduleSpecifierOf(decl: ts.Node | undefined): string | null {
return ts.isImportDeclaration(decl) && ts.isStringLiteral(decl.moduleSpecifier)
? decl.moduleSpecifier.text
: null;
}
/** What an identifier binds from an island module, or null. */
/** Whether `ident` names an import of a Node island module: its value
* comes from the island, whatever the package's declarations say. */
export function isIslandImportBinding(lowerer: Lowerer, ident: ts.Identifier): boolean {
return islandBindingOf(lowerer, ident) !== null;
}
function islandBindingOf(lowerer: Lowerer, ident: ts.Identifier): IslandBinding | null {
const islands = lowerer.nodeIslands;
if (islands === null) return null;
const symbol = lowerer.checker.getSymbolAtLocation(ident);
const decl = symbol ? symbol.declarations[0] : undefined;
if (decl === undefined) return null;
let importDecl: ts.Node | undefined;
let binding: "namespace" | "default" | { name: string };
if (ts.isNamespaceImport(decl)) {
importDecl = decl.parent?.parent;
binding = "namespace";
} else if (ts.isImportClause(decl)) {
importDecl = decl.parent;
binding = "default";
} else if (ts.isImportSpecifier(decl)) {
importDecl = decl.parent?.parent?.parent;
const name = decl.propertyName?.text ?? decl.name.text;
binding = name === "default" ? "default" : { name };
} else {
return null;
}
const spec = moduleSpecifierOf(importDecl);
if (spec === null || importDecl === undefined) return null;
const path = islandModuleFile(
islands,
ts.getSourceFileOfNode(importDecl).fileName,
spec,
"import",
);
if (path === null) return null;
const module = lowerer.islandModules.get(path);
if (module === undefined) return null;
if (binding === "namespace") return { kind: "namespace", module };
if (binding === "default") return { kind: "default", module };
return { kind: "named", module, name: binding.name };
}
/** The raw (dyn) value an island import binding reads. */
function islandBindingValue(
lowerer: Lowerer,
b: IslandBinding,
node: ts.Node,
loc: SrcLoc,
): IrExpr {
const ns = varRef(b.module.global.id, DYN, loc);
switch (b.kind) {
case "namespace":
if (!b.module.esm)
boundaryFail(lowerer, "the namespace object of a CommonJS island module as a value", node);
return ns;
case "default":
return b.module.esm ? islandGet(ns, "default", loc) : ns;
case "named": {
if (b.module.esm) return islandGet(ns, b.name, loc);
const g = b.module.snapshots.get(b.name);
return g === undefined ? islandGet(ns, b.name, loc) : varRef(g.id, DYN, loc);
}
}
}
/** An island value (dyn) at the declared type of `node`: island handles
* stay dyn (FOREIGN), primitives and strings meet their declared type
* through the checked conversion. */
function fromIsland(lowerer: Lowerer, raw: IrExpr, node: ts.Expression): IrExpr {
const tsType = lowerer.typeOf(node);
const islands = lowerer.nodeIslands!;
if (isForeignType(islands, tsType, lowerer.checker)) return raw;
if ((tsType.flags & (ts.TypeFlags.Any | ts.TypeFlags.Unknown)) !== 0) return raw;
if ((tsType.flags & (ts.TypeFlags.Void | ts.TypeFlags.Undefined)) !== 0 && isDiscarded(node))
return raw;
const mapped = lowerer.mapTypeOf(tsType);
if (mapped === null)
boundaryFail(
lowerer,
`an island value of type '${lowerer.checker.typeToString(tsType)}'`,
node,
);
if (mapped.kind === "dyn") return raw;
if (mapped.kind === "void") return raw;
// An island promise is the same promise on both sides (the runtime keeps
// one ScrPromise per V8 promise): PromiseResolve of the dyn value, whose
// settlement value meets the declared type when it is awaited.
if (
mapped.kind === "promise" &&
(mapped.inner.kind === "dyn" || copiedType(lowerer, mapped.inner))
) {
const promise: IrExpr = {
kind: "libCall",
fn: "dyn.promiseResolve",
args: [raw, thenReader(lowerer, raw.loc)],
type: { kind: "promise", inner: DYN },
loc: raw.loc,
};
return mapped.inner.kind === "dyn" ? promise : lowerer.coerceToExpected(promise, mapped);
}
if (!copiedType(lowerer, mapped))
boundaryFail(
lowerer,
`an island value of type '${lowerer.checker.typeToString(tsType)}' (only primitives and strings are copied; objects of types the island does not declare stay handles from NI-5 on)`,
node,
);
return lowerer.coerceToExpected(raw, mapped);
}
/** Whether a value of IR type `t` is copied at the boundary: primitives,
* strings, null/undefined, and unions of those. */
function copiedType(lowerer: Lowerer, t: IrType): boolean {
switch (t.kind) {
case "f64":
case "string":
case "bool":
case "bigint":
case "nullT":
case "undefinedT":
return true;
case "union": {
const def = lowerer.unions.get(t.unionId);
return def !== undefined && def.arms.every((a) => copiedType(lowerer, a));
}
default:
return false;
}
}
function isDiscarded(node: ts.Node): boolean {
let n: ts.Node = node;
while (n.parent && ts.isParenthesizedExpression(n.parent)) n = n.parent;
return (
n.parent !== undefined && (ts.isExpressionStatement(n.parent) || ts.isVoidExpression(n.parent))
);
}
/** Whether `node` evaluates to an island handle (by its declared type) or
* is an island namespace binding. */
function isForeignExpr(lowerer: Lowerer, node: ts.Expression): boolean {
const islands = lowerer.nodeIslands;
if (islands === null) return false;
const inner = stripTypeWrappers(node);
if (ts.isIdentifier(inner) && islandBindingOf(lowerer, inner) !== null) return true;
return isForeignType(islands, lowerer.typeOf(inner), lowerer.checker);
}
/** A value crossing INTO the island (an argument or an assigned value), as
* dyn. */
function toIsland(lowerer: Lowerer, node: ts.Expression): IrExpr {
const islands = lowerer.nodeIslands!;
const inner = stripTypeWrappers(node);
if (ts.isObjectLiteralExpression(inner) || ts.isArrayLiteralExpression(inner)) {
return freshLiteral(lowerer, inner);
}
const crossing = crossingOf(islands, lowerer.typeOf(node), lowerer.checker);
switch (crossing) {
case "copy":
case "handle":
return lowerer.coerceToExpected(lowerer.lowerExpr(node), DYN);
case "dyn":
case "refused":
return boundaryFail(
lowerer,
`a '${lowerer.checker.typeToString(lowerer.typeOf(node))}' value (a native object would cross as a copy and lose its identity)`,
node,
);
}
}
/** A fresh object or array literal crosses as a copy (a plain dyn object
* or array the runtime copies into V8): nothing native can alias it. Its
* members cross by the same rules (toIsland). */
function freshLiteral(
lowerer: Lowerer,
node: ts.ObjectLiteralExpression | ts.ArrayLiteralExpression,
): IrExpr {
const loc = locOf(node);
if (ts.isArrayLiteralExpression(node)) {
const elems = node.elements.nodes.map((el) => {
if (ts.isSpreadElement(el) || ts.isOmittedExpression(el))
boundaryFail(lowerer, "a spread or hole in an array literal", el);
return toIsland(lowerer, el);
});
return { kind: "dynArrLit", elems, type: DYN, loc };
}
const fields = node.properties.nodes.map((p) => {
let key: string;
let value: IrExpr;
if (ts.isPropertyAssignment(p) && (ts.isIdentifier(p.name) || ts.isStringLiteral(p.name))) {
key = p.name.text;
value = toIsland(lowerer, p.initializer);
} else if (ts.isShorthandPropertyAssignment(p)) {
const name = p.name as ts.Identifier;
key = name.text;
value = toIsland(lowerer, name);
} else {
return boundaryFail(
lowerer,
"a computed key, method, accessor or spread in an object literal",
p,
);
}
return { key: { kind: "strLit", value: key, type: STRING, loc } as IrExpr, value };
});
return { kind: "dynObjLit", fields, type: DYN, loc };
}
/** Whether evaluating an argument runs no code that could change the
* receiver: literals, names, `this`, functions it creates, object and
* array literals of those, and operators over those. A unary operator
* (`+`, `-`, `~`) converts its operand with ToNumber, which calls a
* user-defined `valueOf`: only a numeric literal operand is inert. */
function inertArgument(node: ts.Expression): boolean {
const e = stripTypeWrappers(node);
if (
ts.isIdentifier(e) ||
ts.isLiteralExpression(e) ||
ts.isNoSubstitutionTemplateLiteral(e) ||
ts.isArrowFunction(e) ||
ts.isFunctionExpression(e) ||
e.kind === ts.SyntaxKind.ThisKeyword ||
e.kind === ts.SyntaxKind.TrueKeyword ||
e.kind === ts.SyntaxKind.FalseKeyword ||
e.kind === ts.SyntaxKind.NullKeyword
)
return true;
if (ts.isParenthesizedExpression(e)) return inertArgument(e.expression);
if (ts.isArrayLiteralExpression(e))
return e.elements.nodes.every((el) => !ts.isSpreadElement(el) && inertArgument(el));
if (ts.isObjectLiteralExpression(e))
return e.properties.nodes.every(
(p) =>
ts.isShorthandPropertyAssignment(p) ||
ts.isMethodDeclaration(p) ||
(ts.isPropertyAssignment(p) &&
!ts.isComputedPropertyName(p.name) &&
inertArgument(p.initializer)),
);
if (ts.isPrefixUnaryExpression(e)) {
// `!x` runs no conversion on its operand. `+`, `-` and `~` convert it
// with ToNumber, which calls a user-defined `valueOf` on an object:
// that can replace the method being called, so only a numeric literal
// is inert.
if (e.operator === ts.SyntaxKind.ExclamationToken) return inertArgument(e.operand);
return e.operand.kind === ts.SyntaxKind.NumericLiteral;
}
return false;
}
function argsOf(lowerer: Lowerer, args: readonly ts.Expression[], loc: SrcLoc): IrExpr {
for (const a of args) if (ts.isSpreadElement(a)) boundaryFail(lowerer, "spread arguments", a);
return { kind: "dynArrLit", elems: args.map((a) => toIsland(lowerer, a)), type: DYN, loc };
}
/** Expressions on island values (the lowerExpr hook; null: not an island
* expression, the ordinary lowering applies). */
export function lowerIslandExpr(lowerer: Lowerer, expr: ts.Expression): IrExpr | null {
if (lowerer.nodeIslands === null) return null;
const loc = locOf(expr);
if (ts.isIdentifier(expr)) {
const b = islandBindingOf(lowerer, expr);
if (b === null) return null;
lowerer.usesNodeIsland = true;
return fromIsland(lowerer, islandBindingValue(lowerer, b, expr, loc), expr);
}
if (ts.isPropertyAccessExpression(expr) && ts.isIdentifier(expr.name)) {
if (!isForeignExpr(lowerer, expr.expression)) return null;
if (expr.questionDotToken)
boundaryFail(lowerer, "an optional member read on an island value", expr);
lowerer.usesNodeIsland = true;
const target = islandReceiver(lowerer, expr.expression, expr.name.text, loc);
if (target.direct !== undefined) return fromIsland(lowerer, target.direct, expr);
return fromIsland(lowerer, islandGet(target.value, expr.name.text, loc), expr);
}
if (ts.isElementAccessExpression(expr)) {
if (!isForeignExpr(lowerer, expr.expression)) return null;
if (expr.questionDotToken)
boundaryFail(lowerer, "an optional member read on an island value", expr);
lowerer.usesNodeIsland = true;
const target = lowerer.lowerExpr(expr.expression);
const key = toIsland(lowerer, expr.argumentExpression);
return fromIsland(
lowerer,
{ kind: "libCall", fn: "nodeIsland.getKey", args: [target, key], type: DYN, loc },
expr,
);
}
if (ts.isCallExpression(expr)) {
const dynamicLoad = lowerIslandLoadCall(lowerer, expr, loc);
if (dynamicLoad !== undefined) return dynamicLoad;
const callee = stripTypeWrappers(expr.expression);
if (expr.questionDotToken) return null;
const args = expr.arguments.nodes;
if (ts.isPropertyAccessExpression(callee) && ts.isIdentifier(callee.name)) {
if (!isForeignExpr(lowerer, callee.expression)) return null;
if (callee.questionDotToken)
boundaryFail(lowerer, "an optional call on an island value", expr);
lowerer.usesNodeIsland = true;
const target = islandReceiver(lowerer, callee.expression, callee.name.text, loc);
if (target.direct !== undefined) {
return fromIsland(
lowerer,
{
kind: "libCall",
fn: "nodeIsland.apply",
args: [target.direct, argsOf(lowerer, args, loc)],
type: DYN,
loc,
},
expr,
);
}
const name: IrExpr = { kind: "strLit", value: callee.name.text, type: STRING, loc };
if (args.every((a) => inertArgument(a))) {
return fromIsland(
lowerer,
{
kind: "libCall",
fn: "nodeIsland.invoke",
args: [target.value, name, argsOf(lowerer, args, loc)],
type: DYN,
loc,
},
expr,
);
}
// An argument that runs code could replace the method: JavaScript
// reads the method first, then evaluates the arguments.
const stmts: IrStmt[] = [];
const receiver = lowerer.declareHiddenLocal("%islandRecv", target.value.type);
stmts.push({ kind: "varDecl", localId: receiver.id, init: target.value, loc });
const recv = varRef(receiver.id, target.value.type, loc);
const method = lowerer.declareHiddenLocal("%islandMethod", DYN);
stmts.push({
kind: "varDecl",
localId: method.id,
init: { kind: "libCall", fn: "nodeIsland.get", args: [recv, name], type: DYN, loc },
loc,
});
const call: IrExpr = {
kind: "libCall",
fn: "nodeIsland.applyTo",
args: [varRef(method.id, DYN, loc), recv, argsOf(lowerer, args, loc)],
type: DYN,
loc,
};
const result = fromIsland(lowerer, call, expr);
return { kind: "seqExpr", stmts, result, type: result.type, loc };
}
if (!isForeignExpr(lowerer, callee)) return null;
lowerer.usesNodeIsland = true;
const fn = lowerer.lowerExpr(callee);
return fromIsland(
lowerer,
{
kind: "libCall",
fn: "nodeIsland.apply",
args: [fn, argsOf(lowerer, args, loc)],
type: DYN,
loc,
},
expr,
);
}
if (ts.isNewExpression(expr)) {
if (!isForeignExpr(lowerer, expr.expression)) return null;
lowerer.usesNodeIsland = true;
const ctor = lowerer.lowerExpr(expr.expression);
return fromIsland(
lowerer,
{
kind: "libCall",
fn: "nodeIsland.construct",
args: [ctor, argsOf(lowerer, expr.arguments?.nodes ?? [], loc)],
type: DYN,
loc,
},
expr,
);
}
if (ts.isBinaryExpression(expr)) {
const op = expr.operatorToken.kind;
if (op === ts.SyntaxKind.InstanceOfKeyword && isForeignExpr(lowerer, expr.right)) {
lowerer.usesNodeIsland = true;
const value = lowerer.coerceToExpected(lowerer.lowerExpr(expr.left), DYN);
const ctor = lowerer.lowerExpr(expr.right);
return { kind: "libCall", fn: "nodeIsland.instanceof", args: [value, ctor], type: BOOL, loc };
}
if (op === ts.SyntaxKind.EqualsToken) {
const left = stripTypeWrappers(expr.left);
if (
(ts.isPropertyAccessExpression(left) && ts.isIdentifier(left.name)) ||
ts.isElementAccessExpression(left)
) {
if (!isForeignExpr(lowerer, left.expression)) return null;
lowerer.usesNodeIsland = true;
// JavaScript's order: the receiver, then the key, then the value.
const stmts: IrStmt[] = [];
const hold = (name: string, init: IrExpr, type: IrType): IrExpr => {
const local = lowerer.declareHiddenLocal(name, type);
stmts.push({ kind: "varDecl", localId: local.id, init, loc });
return varRef(local.id, type, loc);
};
const target = lowerer.lowerExpr(left.expression);
const receiver = hold("%islandSetTarget", target, target.type);
const key = ts.isPropertyAccessExpression(left)
? null
: hold("%islandSetKey", toIsland(lowerer, left.argumentExpression), DYN);
const v = hold("%islandSet", toIsland(lowerer, expr.right), DYN);
stmts.push({
kind: "exprStmt",
expr:
key === null
? {
kind: "libCall",
fn: "nodeIsland.set",
args: [
receiver,
{
kind: "strLit",
value: (left as ts.PropertyAccessExpression).name.text,
type: STRING,
loc,
},
v,
],
type: { kind: "void" },
loc,
}
: {
kind: "libCall",
fn: "nodeIsland.setKey",
args: [receiver, key, v],
type: { kind: "void" },
loc,
},
loc,
});
const result = fromIsland(lowerer, v, expr);
return { kind: "seqExpr", stmts, result, type: result.type, loc };
}
return null;
}
const left = stripTypeWrappers(expr.left);
if (
op >= ts.SyntaxKind.FirstCompoundAssignment &&
op <= ts.SyntaxKind.LastCompoundAssignment &&
(ts.isPropertyAccessExpression(left) || ts.isElementAccessExpression(left)) &&
isForeignExpr(lowerer, left.expression)
) {
boundaryFail(lowerer, "a compound assignment to an island object's member", expr);
}
}
return null;
}
/** The receiver of a member operation on an island value. A CommonJS
* namespace binding's members are module.exports' (and `default` is
* module.exports itself), answered directly. */
function islandReceiver(
lowerer: Lowerer,
receiver: ts.Expression,
member: string,
loc: SrcLoc,
): { value: IrExpr; direct?: IrExpr } {
const inner = stripTypeWrappers(receiver);
if (ts.isIdentifier(inner)) {
const b = islandBindingOf(lowerer, inner);
if (b !== null && b.kind === "namespace" && !b.module.esm) {
const exports = varRef(b.module.global.id, DYN, loc);
if (member === "default") return { value: exports, direct: exports };
return { value: exports };
}
}
return { value: lowerer.coerceToExpected(lowerer.lowerExpr(receiver), FOREIGN) };
}
/** `require("pkg")` (the global require, anywhere) and `import("pkg")` of
* an island module: load it at the call. Undefined: not such a call. */
function lowerIslandLoadCall(
lowerer: Lowerer,
expr: ts.CallExpression,
loc: SrcLoc,
): IrExpr | undefined {
const islands = lowerer.nodeIslands!;
const arg = expr.arguments.nodes[0];
if (expr.arguments.nodes.length !== 1 || arg === undefined || !ts.isStringLiteralLike(arg))
return undefined;
const isImport = expr.expression.kind === ts.SyntaxKind.ImportKeyword;
const isRequire =
ts.isIdentifier(expr.expression) &&
expr.expression.text === "require" &&
!lowerer.checker
.getSymbolAtLocation(expr.expression)
?.declarations.some((d) => !ts.getSourceFileOfNode(d).isDeclarationFile);
if (!isImport && !isRequire) return undefined;
const file = ts.getSourceFileOfNode(expr).fileName;
const path = islandModuleFile(islands, file, arg.text, isImport ? "import" : "require");
if (path === null) return undefined;
lowerer.usesNodeIsland = true;
if (isImport) {
return {
kind: "libCall",
fn: "nodeIsland.import",
args: [islandLoadPath(lowerer, path, loc)],
type: { kind: "promise", inner: DYN },
loc,
};
}
return fromIsland(lowerer, loadIslandModule(lowerer, path, loc), expr);
}
packages/compiler/src/frontend/lowering/lower-stdlib.ts
+65
View File
@@ -38,6 +38,7 @@ import {
isJsSourceFile,
locOf,
npmPackageNameOf,
type StartupCrash,
} from "../program.js";
import { foldedStringKeyOf, lowerDynObjectLiteral } from "./expressions/object-literals.js";
import { noteEventListeners } from "./lower-event-target.js";
@@ -2243,6 +2244,7 @@ export function lowerDynamicImportCall(lowerer: Lowerer, call: ts.CallExpression
}
const res = lowerer.dynImports.get(`${ts.getSourceFileOfNode(call).fileName}\u0000${arg.text}`);
if (res === undefined) lowerer.refuseUnsupportedApi("'import()'", call);
if (res.kind === "rejected") return lowerRejectedImport(lowerer, call, res.crash);
if (res.kind === "program-module") {
return lowerOwnStaticModuleImport(lowerer, call, arg);
}
@@ -2252,6 +2254,69 @@ export function lowerDynamicImportCall(lowerer: Lowerer, call: ts.CallExpression
});
}
/** An import() Node rejects before any module of its graph evaluates
* (a package marked never loaded, a refused edge in the module's static
* graph): the promise rejects on a microtask with Node's error, and the
* module never joins the compiled graph. Its namespace is never produced,
* so the result is typed dynamically. */
function lowerRejectedImport(
lowerer: Lowerer,
call: ts.CallExpression,
crash: StartupCrash,
): IrExpr {
const loc = locOf(call);
const resultT: IrType & { kind: "promise" } = { kind: "promise", inner: DYN };
const fnName = lowerer.closureName(loc, "_rejectedImport");
const fnType: IrType & { kind: "func" } = { kind: "func", params: [], ret: resultT };
const fnCtx = newFnCtx(true, null, fnType, DYN);
fnCtx.isAsync = true;
lowerer.fnStack.push(fnCtx);
try {
const hop: IrExpr = {
kind: "awaitExpr",
value: {
kind: "intrinsic",
name: "promise.resolve",
args: [],
type: { kind: "promise", inner: VOID },
loc,
},
type: VOID,
loc,
};
const errorKind = crash.className === "%TypeError" ? 1 : 0;
const body: IrStmt[] = [
{ kind: "exprStmt", expr: hop, loc },
{
kind: "exprStmt",
expr: nodeThrowExpr(errorKind, crash.code ?? "", crash.message, VOID, loc),
loc,
},
{ kind: "return", value: dynUndefinedExpr(loc), loc },
];
const ctx = lowerer.ctx;
lowerer.liftedFns.push({
name: fnName,
params: [],
returnType: DYN,
locals: ctx.locals,
captures: ctx.captures ?? [],
body,
async: true,
loc,
});
} finally {
lowerer.fnStack.pop();
}
return {
kind: "callValue",
callee: { kind: "closure", fnName, captures: [], type: fnType, loc },
args: [],
type: resultT,
loc,
};
}
/** The checker-resolved program source behind one literal import(). */
function programImportTarget(lowerer: Lowerer, arg: ts.StringLiteralLike): ts.SourceFile | null {
const modSym = lowerer.checker.getSymbolAtLocation(arg);
packages/compiler/src/frontend/lowering/lower-stmts.ts
+6
View File
@@ -2,6 +2,7 @@ import type { DiagnosticCode } from "../../diagnostics/codes.js";
import { keepsInitializerLayout, notePreservedLayout } from "./structural-instances.js";
import { jsGlobalValueByName } from "./global-values.js";
import { lowerTupleAliasDecl, stripTypeWrappers } from "./typed-arguments.js";
import { lowerIslandExpr } from "./lower-node-island.js";
import {
allBytesArms,
indexedUnionArms,
@@ -8032,6 +8033,11 @@ function lowerDereferencedElementWrite(lowerer: Lowerer, expr: ts.BinaryExpressi
export function lowerExprStatement(lowerer: Lowerer, expr: ts.Expression): IrStmt {
const stmtNode = expr.parent;
while (ts.isParenthesizedExpression(expr)) expr = expr.expression;
// Node islands (scope-B NI-4): writes and calls on island values.
if (lowerer.nodeIslands !== null) {
const island = lowerIslandExpr(lowerer, expr);
if (island !== null) return { kind: "exprStmt", expr: island, loc: locOf(expr) };
}
// A discarded conditional has no value to merge: represent its lazy
// branch selection directly as statement control flow. In particular,
// both arms may be void calls (`flag ? emitA() : emitB()`), for which a
packages/compiler/src/frontend/lowering/lowerer.ts
+106 -4
View File
@@ -44,6 +44,13 @@ import {
import { moduleNamespaceObject, moduleNamespaceValue } from "./module-namespace-elements.js";
import { keyedHelperName, moduleTagNumber, portableRelativePath } from "./stable-names.js";
import { positionTag, StructuralPositions } from "./structural-positions.js";
import {
activeNodeIslands,
isForeignType,
setActiveNodeIslands,
type NodeIslandSet,
} from "../../node-islands/boundary.js";
import type { IslandModuleBinding } from "./lower-node-island.js";
import { bindingSymbolOfRead, onlyForwarded, stripTypeWrappers } from "./typed-arguments.js";
import {
needsGeneralNodeStreams,
@@ -938,6 +945,9 @@ export interface LowerResult {
}
export interface LowerOptions {
/** Packages that run in Node islands (scope-B NI-4): their imports lower
* to island loads and their values to island handles. */
nodeIslands?: NodeIslandSet | null;
/** Parser/checker services owned by the load; required for npm graph analysis. */
frontendServices?: FrontendServices | undefined;
/** The project's tsconfig sets `checkJs` (see isCheckedJsSourceFile). */
@@ -965,6 +975,8 @@ export interface LowerOptions {
inlineRequireCycleCalls?: ReadonlySet<ts.Node> | undefined;
/** LoadResult.moduleEdges. */
moduleEdges?: ReadonlyMap<ts.SourceFile, readonly CycleEdge[]> | undefined;
/** LoadResult.dynamicImportRejections. */
dynamicImportRejections?: ReadonlyMap<ts.SourceFile, StartupCrash> | undefined;
/** LIBRARY mode's reachability roots: the profile-mapped exports of the
* entry module. Executable builds root at the entry's top level alone
* (an unreferenced export dead-strips); a library's exports are called from
@@ -1086,6 +1098,8 @@ export interface LowererMode {
dynNodeStreams?: boolean;
/** LoadResult.inlineRequireCycleCalls. */
inlineRequireCycleCalls?: ReadonlySet<ts.Node> | undefined;
/** LoadResult.dynamicImportRejections. */
dynamicImportRejections?: ReadonlyMap<ts.SourceFile, StartupCrash> | undefined;
/** LowerOptions.libraryCallbacks (see there). */
libraryCallbacks?: boolean;
projectCheckJs?: boolean;
@@ -1175,6 +1189,60 @@ export function lowerToIr(
entry: ts.SourceFile,
moduleOrder: ts.SourceFile[],
options: LowerOptions = {},
): LowerResult {
setActiveNodeIslands(options.nodeIslands ?? null);
try {
const result = lowerToIrWithIslands(program, entry, moduleOrder, options);
markModuleInitFences(program, result.runtimeFences);
return result;
} finally {
setActiveNodeIslands(null);
}
}
/** Marks the runtime fences in npm package code whose site is top-level
* module code (outside every function body): they throw when the module
* evaluates (ScrDiagnostic.moduleInit). */
function markModuleInitFences(program: ts.Program, fences: readonly ScrDiagnostic[]): void {
for (const d of fences) {
if (!d.loc.file.includes("/node_modules/")) continue;
// A fence without a source position belongs to synthesized code, such
// as the global object's lookup (global-values.ts), which runs when the
// program calls it, not when the module evaluates.
if (d.loc.start < 0) continue;
const sf = program.getSourceFile(d.loc.file);
if (sf === undefined) continue;
let inFunction = false;
let node: ts.Node | undefined = sf;
while (node !== undefined && !inFunction) {
let next: ts.Node | undefined;
ts.forEachChild(node, (child) => {
if (
next === undefined &&
ts.tokenStartOfNode(child) <= d.loc.start &&
d.loc.start < child.end
)
next = child;
return undefined;
});
if (next === undefined) break;
if (
ts.isFunctionLike(next) ||
(ts.isPropertyDeclaration(next) &&
!next.modifiers?.nodes.some((m) => m.kind === ts.SyntaxKind.StaticKeyword))
)
inFunction = true;
node = next;
}
if (!inFunction) d.moduleInit = true;
}
}
function lowerToIrWithIslands(
program: ts.Program,
entry: ts.SourceFile,
moduleOrder: ts.SourceFile[],
options: LowerOptions,
): LowerResult {
const phaseTiming = process.env["SCRIPTC_TIMING"] === "1";
const phaseStarted = performance.now();
@@ -1219,7 +1287,11 @@ export function lowerToIr(
// Inline require() calls closing a cycle inside the appended subgraphs
// join preflight's set (the same run-time refusal by name).
const inlineRequireCycleCalls = new Set<ts.Node>(options.inlineRequireCycleCalls ?? []);
const graph = { edges: options.moduleEdges, inlineCycleCalls: inlineRequireCycleCalls };
const graph = {
edges: options.moduleEdges,
inlineCycleCalls: inlineRequireCycleCalls,
rejected: options.dynamicImportRejections,
};
for (let pass = 0; pass < 32; pass++) {
const before = moduleOrder.length;
appendDynamicImportModules(program, moduleOrder, onExtraCycle, graph);
@@ -1240,6 +1312,7 @@ export function lowerToIr(
targetPlatform,
startupCrash,
inlineRequireCycleCalls,
dynamicImportRejections: options.dynamicImportRejections,
ffiImports,
libraryCallbacks,
forkTargets,
@@ -1264,6 +1337,7 @@ export function lowerToIr(
targetPlatform,
startupCrash,
inlineRequireCycleCalls,
dynamicImportRejections: options.dynamicImportRejections,
ffiImports,
libraryCallbacks,
forkTargets,
@@ -1329,6 +1403,7 @@ export function lowerToIr(
targetPlatform,
startupCrash,
inlineRequireCycleCalls,
dynamicImportRejections: options.dynamicImportRejections,
ffiImports,
libraryCallbacks,
forkTargets,
@@ -1374,6 +1449,7 @@ export function lowerToIr(
targetPlatform,
startupCrash,
inlineRequireCycleCalls,
dynamicImportRejections: options.dynamicImportRejections,
ffiImports,
libraryCallbacks,
forkTargets,
@@ -1515,6 +1591,7 @@ export function lowerToIr(
degradedTypes: resultLowerer.degradedTypes,
retriedSymbols: resultLowerer.retriedSymbols,
inlineRequireCycleCalls,
dynamicImportRejections: options.dynamicImportRejections,
targetPlatform,
ffiImports,
libraryCallbacks,
@@ -2135,6 +2212,8 @@ export class Lowerer {
/** Inline require() calls whose module edge closes a require cycle
* (LoadResult.inlineRequireCycleCalls). */
readonly inlineRequireCycleCalls: ReadonlySet<ts.Node>;
/** LoadResult.dynamicImportRejections. */
readonly dynamicImportRejections: ReadonlyMap<ts.SourceFile, StartupCrash>;
/** Immutable callable aliases and compiler-generated adapters. The map
* is symbol-keyed, so shadowing and cross-module imports keep the same
* exact binding discipline as globals/functions. */
@@ -3472,7 +3551,9 @@ export class Lowerer {
* collection; lowerDynamicImportCall reads it per site). */
readonly dynImports = new Map<
string,
{ kind: "program-module" } | { kind: "static-builtin"; module: string }
| { kind: "program-module" }
| { kind: "static-builtin"; module: string }
| { kind: "rejected"; crash: StartupCrash }
>();
readonly createRequireImports = new Map<
@@ -3533,6 +3614,11 @@ export class Lowerer {
/** The program calls into Node.js (@scriptc/node): the module carries
* the nodeIsland record, which the executable build fills in. */
usesNodeIsland = false;
/** The program's Node island packages (scope-B NI-4), or null. */
readonly nodeIslands: NodeIslandSet | null = activeNodeIslands();
/** Island module loads, by the importing file and specifier: the module
* global holding what the import form sees (lower-node-island.ts). */
readonly islandModules = new Map<string, IslandModuleBinding>();
/** The modules Node island calls load (lower-node-island.ts). */
readonly nodeIslandModules = new NodeIslandModuleCollector();
/** Each call's caller literal, set to its tree path when lowering ends. */
@@ -3543,6 +3629,8 @@ export class Lowerer {
caller: string;
target: string;
}[] = [];
/** Each island module load's path literal, set to its tree path when lowering ends. */
readonly nodeIslandLoads: { literal: IrExpr & { kind: "strLit" }; path: string }[] = [];
/** Per-directory answers of experimental-import.ts's declaration-file test. */
readonly experimentalFiles = new Map<string, boolean>();
/** Interned LifetimeRegion.run helpers, by callback type key. */
@@ -4027,9 +4115,15 @@ export class Lowerer {
// one lowers again rather than replaying a record.
checkedTable("nodeIslandCallers", () => this.nodeIslandCallers.length),
checkedTable("nodeIslandSpecifiers", () => this.nodeIslandSpecifiers.length),
// Island module loads hold literals of the unit's IR too, and their
// module globals (with each CommonJS named import's snapshot global)
// are created by the first load: a unit that adds either lowers again.
checkedTable("islandModules", () => this.islandModules.size),
checkedTable("nodeIslandLoads", () => this.nodeIslandLoads.length),
checkedTable("fnSigsBySymbol", () => this.fnSigsBySymbol.size),
checkedTable("cjsExportsAssignments", () => this.cjsExportsAssignments.size),
checkedTable("inlineRequireCycleCalls", () => this.inlineRequireCycleCalls.size),
checkedTable("dynamicImportRejections", () => this.dynamicImportRejections.size),
checkedTable("staticCallables", () => this.staticCallables.size),
checkedTable("genericFnsBySymbol", () => this.genericFnsBySymbol.size),
checkedTable("objLitGenericFns", () => this.objLitGenericFns.size),
@@ -4249,6 +4343,7 @@ export class Lowerer {
static readonly unjournaled: readonly string[] = [
// settings
"frontendServices",
"nodeIslands",
"ambient",
"overridesAmbient",
"remainder",
@@ -4781,6 +4876,7 @@ export class Lowerer {
this.forkTargets = mode.forkTargets ?? [];
this.frontendServices = mode.frontendServices;
this.inlineRequireCycleCalls = mode.inlineRequireCycleCalls ?? new Set();
this.dynamicImportRejections = mode.dynamicImportRejections ?? new Map();
this.libraryCallbacks = mode.libraryCallbacks ?? false;
this.projectCheckJs = mode.projectCheckJs ?? false;
this.reachable = mode.reachable ?? null;
@@ -4801,8 +4897,12 @@ export class Lowerer {
this.externalTypeSpecifiersByFile =
mode.externalTypeSpecifiersByFile ?? directExternalTypeSpecifiersByFile(this.externalTypes);
this.checker = program.getTypeChecker();
const islands = this.nodeIslands;
const shadow = canonShadowHook(this.checker);
this.typeCtx = {
...(islands !== null
? { foreignType: (t: ts.Type) => isForeignType(islands, t, this.checker) }
: {}),
checker: this.checker,
positionOf: (node) => this.positions.of(node),
shapes: this.shapes,
@@ -10550,14 +10650,16 @@ export class Lowerer {
}
/** The module's Node island record: the collected module tree, with each
* call's caller literal set to its path in that tree. The executable
* build fills in the libnode pack (index.ts). */
* call's caller literal and each load's path literal set to its path in
* that tree. The executable build fills in the libnode pack (index.ts). */
private nodeIslandRecord(): NonNullable<IrModule["nodeIsland"]> {
const modules = this.nodeIslandModules.tree();
for (const { literal, caller } of this.nodeIslandCallers)
literal.value = NodeIslandModuleCollector.callerPath(modules, caller);
for (const { literal, caller, target } of this.nodeIslandSpecifiers)
literal.value = NodeIslandModuleCollector.relativeSpecifier(caller, target);
for (const { literal, path } of this.nodeIslandLoads)
literal.value = NodeIslandModuleCollector.loadPath(modules, path);
return { libnode: "", nodeVersion: "", ...(modules === null ? {} : { modules }) };
}
packages/compiler/src/frontend/node-api-import.ts
+7 -3
View File
@@ -6,7 +6,11 @@ import * as ts from "./ts7/adapter.js";
export const NODE_API_MODULE = "@scriptc/node";
export type NodeApiMember = "call" | "callSync";
export type NodeApiMember = "call" | "callSync" | "requireModule";
function isNodeApiMember(name: string): name is NodeApiMember {
return name === "call" || name === "callSync" || name === "requireModule";
}
function moduleSpecifierText(decl: ts.Node | undefined): string | null {
return ts.isImportDeclaration(decl) && ts.isStringLiteral(decl.moduleSpecifier)
@@ -28,7 +32,7 @@ export function nodeApiBindingOf(
if (!ts.isImportSpecifier(decl)) return null;
if (moduleSpecifierText(decl.parent?.parent?.parent) !== NODE_API_MODULE) return null;
const member = decl.propertyName?.text ?? decl.name.text;
return member === "call" || member === "callSync" ? member : null;
return isNodeApiMember(member) ? member : null;
}
/** The @scriptc/node export an identifier is a named-import binding of. */
@@ -55,6 +59,6 @@ export function nodeApiCalleeOf(program: ts.Program, callee: ts.Expression): Nod
)
return null;
const member = callee.name.text;
if (member !== "call" && member !== "callSync") return null;
if (!isNodeApiMember(member)) return null;
return isNodeApiNamespace(program, callee.expression) ? member : null;
}
packages/compiler/src/frontend/pipeline.ts
+59 -40
View File
@@ -49,6 +49,7 @@ import { NODE_API_MODULE } from "./node-api-import.js";
import { erasedTypeOnlyImport } from "./type-only-imports.js";
import { EXPERIMENTAL_MODULE } from "./experimental-import.js";
import { readProjectConfig } from "../project-config.js";
import { setFrontendIslands } from "../node-islands/boundary.js";
import { typePatchDiagnostics } from "./type-patches/layer.js";
import { packageDirOf } from "../node-islands/origin.js";
import {
@@ -128,12 +129,18 @@ export interface Frontend {
}
/** Hosts own transport lifetimes; shared compiler stages own the frontend. */
/** What a frontend computes beyond the build's own needs. */
/** How a frontend loads the program, beyond its npm package selection. */
export interface FrontendOptions {
/** Packages that run in Node islands (scope-B NI-4): never compiled,
* typed by their declarations. */
islandPackages?: ReadonlySet<string>;
/** The Node island report (coverage): the project's scriptc settings and
* the detector's scan of the program and its npm packages. Builds do not
* read either. */
nodeIslands?: boolean;
/** The detector's scan alone, for a build that may run packages in Node
* islands (executable/prepare.ts decides which). */
islandScan?: boolean;
}
export type FrontendFactory = (
@@ -196,6 +203,8 @@ function addDynamicImportGraph(load: LoadResult, scanned: Set<SourceFile>): void
npmStaticDepSf7(load.program, sf, spec, kind);
if (dep === null || scanned.has(dep)) continue;
if (dep.fileName.includes("/node_modules/") && !isJsSourceFileName(dep.fileName)) continue;
// An import() whose graph Node refuses rejects: nothing in it loads.
if (kind === "import" && load.dynamicImportRejections?.has(dep)) continue;
scanned.add(dep);
added.add(dep);
queue.push(dep);
@@ -480,15 +489,21 @@ export function runFrontend(
transitive.add(base);
return base;
});
return loadFrontend(
entryPath,
trackedLoader,
"lib",
externalTypes,
[...new Set(names)],
transitive,
options,
);
const islands = options.islandPackages ?? new Set<string>();
setFrontendIslands(islands);
try {
return loadFrontend(
entryPath,
trackedLoader,
"lib",
externalTypes,
[...new Set(names)].filter((name) => !islands.has(name)),
transitive,
options,
);
} finally {
setFrontendIslands(new Set());
}
} catch (error) {
for (const load of active) {
// Preserve the original failure and attempt every remaining cleanup.
@@ -518,8 +533,10 @@ function loadFrontend(
clearResolveCaches();
const statuses: NpmStaticStatus[] = [];
const npmSites = new Map<string, SrcLoc>();
// Node island detection (report-only): how each package was refused, and
// the modules of packages removed from the program.
// Node island detection (the report, or a build that may run islands):
// how each package was refused, and the modules of packages removed from
// the program.
const scanIslands = options.nodeIslands === true || options.islandScan === true;
const refusals: RefusalRecord[] = [];
const droppedScan: { modules: NodeIslandScan["modules"]; edges: NodeIslandScan["edges"] } = {
modules: [],
@@ -530,7 +547,11 @@ function loadFrontend(
};
// Explicit library attempts bypass only the auto-selection heuristics.
// The shared preflight/lowering refusal and dependency-closure checks remain.
// Node island packages are judged already: they stay out of the
// program, typed by their declarations (scope-B NI-4).
const judged = new Set<string>(libraryNpmStatic);
if (options.islandPackages !== undefined)
for (const name of options.islandPackages) judged.add(name);
let requested: string[] = [];
let reusableScout: ReturnType<typeof loadProgram> | null = null;
let reusablePreflight: ScrDiagnostic[] | null = null;
@@ -656,7 +677,7 @@ function loadFrontend(
}
const dropping = [...reasons.keys()].filter((p) => effective.has(p));
if (dropping.length === 0) break;
if (options.nodeIslands === true) scanDroppedPackages(load, dropping, droppedScan);
if (scanIslands) scanDroppedPackages(load, dropping, droppedScan);
for (const p of dropping) {
effective.delete(p);
statuses.push({ package: p, status: "refused", detail: reasons.get(p)! });
@@ -708,15 +729,13 @@ function loadFrontend(
probe.dispose();
if (probeDiags.some((d) => d.code === "SC0001")) dropWithNote(p);
}
if (options.nodeIslands === true)
scanDroppedPackages(load, lastResortDropped.splice(0), droppedScan);
if (scanIslands) scanDroppedPackages(load, lastResortDropped.splice(0), droppedScan);
load.dispose();
load = loadProgram(entryPath, { npmStatic: [...effective], externalTypes });
preflight = checkPreflight(load);
if (preflight.some((d) => d.code === "SC0001") && effective.size > 0) {
for (const p of [...effective]) dropWithNote(p);
if (options.nodeIslands === true)
scanDroppedPackages(load, lastResortDropped.splice(0), droppedScan);
if (scanIslands) scanDroppedPackages(load, lastResortDropped.splice(0), droppedScan);
load.dispose();
load = loadProgram(entryPath, { npmStatic: [...effective], externalTypes });
preflight = checkPreflight(load);
@@ -802,6 +821,7 @@ function loadFrontend(
startupCrash: finalLoad.startupCrash ?? null,
inlineRequireCycleCalls: finalLoad.inlineRequireCycleCalls,
moduleEdges: finalLoad.moduleEdges,
dynamicImportRejections: finalLoad.dynamicImportRejections,
projectCheckJs: finalLoad.typeWorld?.projectCheckJs ?? false,
externalTypes: finalLoad.externalTypes,
externalTypeSpecifiersByFile: finalLoad.externalTypeSpecifiersByFile,
@@ -817,29 +837,28 @@ function loadFrontend(
},
npmStatic: statuses,
npmImportSites: npmSites,
nodeIslandScan:
options.nodeIslands !== true
? undefined
: ({ userJs }) => {
const files = new Set<SourceFile>([finalLoad.entry, ...finalLoad.moduleOrder]);
addDynamicImportGraph(finalLoad, files);
// Package edges matter only when the program has npm packages: one
// that loads, one that was refused, or one judged and dropped.
const npm =
statuses.length > 0 ||
droppedScan.modules.length > 0 ||
[...files].some((sf) => sf.fileName.includes("/node_modules/"));
const scanned = scanFiles(finalLoad.program, [...files], {
userJs,
dropped: false,
edges: npm,
});
return {
modules: [...droppedScan.modules, ...scanned.modules],
edges: [...droppedScan.edges, ...scanned.edges],
refusals,
};
},
nodeIslandScan: !scanIslands
? undefined
: ({ userJs }) => {
const files = new Set<SourceFile>([finalLoad.entry, ...finalLoad.moduleOrder]);
addDynamicImportGraph(finalLoad, files);
// Package edges matter only when the program has npm packages: one
// that loads, one that was refused, or one judged and dropped.
const npm =
statuses.length > 0 ||
droppedScan.modules.length > 0 ||
[...files].some((sf) => sf.fileName.includes("/node_modules/"));
const scanned = scanFiles(finalLoad.program, [...files], {
userJs,
dropped: false,
edges: npm,
});
return {
modules: [...droppedScan.modules, ...scanned.modules],
edges: [...droppedScan.edges, ...scanned.edges],
refusals,
};
},
dispose: finalLoad.dispose,
};
}
packages/compiler/src/frontend/process-fork.ts
+3
View File
@@ -8,6 +8,9 @@ export interface ProcessFork {
exit(code: number): void;
/** The worker's exit status, 128 plus a signal number, or -1. */
wait(pid: number): number;
/** The pid of a child process that has ended, waiting for one if none
* has, or -1. It stays unreaped: `wait` reaps it. */
exited(): number;
/** Opens the lock file checker requests from workers take. */
openLock(path: string): boolean;
lock(): void;
packages/compiler/src/frontend/program.ts
+95 -6
View File
@@ -39,6 +39,7 @@
* JSON.parse's wording rather than tsc's, the one message-text delta on
* that path (no snapshot pins it). */
import { isFrontendIsland } from "../node-islands/boundary.js";
import { builtinModules } from "node:module";
import { dirname, isAbsolute, resolve } from "node:path";
import * as ts from "./ts7/adapter.js";
@@ -97,6 +98,7 @@ import {
configurePackageShims,
isPackageShimFile,
isShimmedImport,
isUnloadedPackage,
npmStaticOmitsPackage,
} from "./package-shims.js";
import { provenanceEntryFor, provenancePaths } from "./provenance-registry.js";
@@ -123,6 +125,7 @@ import {
isRelativeSpecifier,
isWorkspacePackageName,
npmPackageNameOf,
packageNameOfSpecifier,
registerWorkspacePackage,
workspacePackageOfPath,
} from "./workspace-registry.js";
@@ -387,6 +390,11 @@ export interface LoadResult {
* literal import(), fork() or Worker reaches with them. Filled by
* checkPreflight. */
moduleEdges?: ReadonlyMap<ts.SourceFile, readonly CycleEdge[]> | undefined;
/** Program modules only a literal import() loads whose static graph
* Node's resolution refuses: each such import() rejects with the refusal
* instead of failing the build, and the module never joins the compiled
* graph. Filled by checkPreflight. */
dynamicImportRejections?: ReadonlyMap<ts.SourceFile, StartupCrash> | undefined;
/** Node's startup refusal, when the graph carries an import edge Node
* rejects before ANY module evaluates — a RESOLUTION refusal (a bare
* specifier nothing installed resolves, an exports/imports map that
@@ -539,6 +547,8 @@ export interface StartupCrash {
message: string;
className: "%Error" | "%TypeError" | "%SyntaxError";
loc: { file: string; start: number; end: number };
/** Node's error code, for the refusals that carry one. */
code?: string;
}
/** True when a coverage mapping names one exact bare host module. TypeScript
@@ -1408,12 +1418,19 @@ function declBoundaryDroppedDiag(file: string): ScrDiagnostic {
* preflight diagnostics. Preserve the caller's complete load shape when
* mutating it, including its owned services and disposal callback. */
export function checkPreflight<T extends LoadResult>(load: T): ScrDiagnostic[] {
const { diags, moduleOrder, startupCrash, inlineRequireCycleCalls, moduleEdges } =
preflight7(load);
const {
diags,
moduleOrder,
startupCrash,
inlineRequireCycleCalls,
moduleEdges,
dynamicImportRejections,
} = preflight7(load);
load.moduleOrder = moduleOrder;
load.startupCrash = startupCrash;
load.inlineRequireCycleCalls = inlineRequireCycleCalls;
load.moduleEdges = moduleEdges;
load.dynamicImportRejections = dynamicImportRejections;
return diags;
}
@@ -5352,6 +5369,7 @@ function preflight7(load: LoadResult): {
startupCrash: StartupCrash | null;
inlineRequireCycleCalls: ReadonlySet<ts.Node>;
moduleEdges?: ReadonlyMap<ts.SourceFile, readonly CycleEdge[]>;
dynamicImportRejections?: ReadonlyMap<ts.SourceFile, StartupCrash>;
} {
const { program, entry } = load;
const diags: ScrDiagnostic[] = [...load.configDiags];
@@ -5844,6 +5862,16 @@ function preflight7(load: LoadResult): {
);
continue;
}
// A re-export from a Node island package that tsconfig `paths` maps
// to its installed directory: its declarations are no program
// module (the lowering reports the island boundary).
if (
projectReexport !== null &&
projectReexport.isDeclarationFile &&
!isRelativeSpecifier(fromSpec) &&
isFrontendIsland(packageNameOfSpecifier(fromSpec))
)
continue;
const reDep = projectReexport;
// `export * as ns from "./m"` re-exports the module NAMESPACE
// object under a name: importers' `x.ns.member` reads resolve
@@ -5909,9 +5937,10 @@ function preflight7(load: LoadResult): {
message: string,
className: StartupCrash["className"],
fallbackFeature: string,
code?: string,
): void => {
refusals.get(sf)!.push({
crash: { message, className, loc: locOf7(stmt) },
crash: { message, className, loc: locOf7(stmt), ...(code !== undefined ? { code } : {}) },
fallback: unsupportedDiag("SC1010", locOf7(stmt), fallbackFeature),
pos: ts.tokenStartOfNode(stmt),
});
@@ -5954,6 +5983,7 @@ function preflight7(load: LoadResult): {
refusal.message,
"%Error",
`the '${spec}' package (its runtime resolution fails: ${refusal.message})`,
refusal.code,
);
}
continue;
@@ -5966,6 +5996,16 @@ function preflight7(load: LoadResult): {
// Node's startup crash with Node's exact message; what keeps a compile
// fence is unsupported builtin or types-only surface.
if (isBare && npmStaticDep === null) {
// A Node island package that tsconfig `paths` maps to its
// installed directory resolves to its declarations here: it loads
// in Node, typed by them, like an island package resolved through
// node_modules (scope-B NI-4).
if (
projDep !== null &&
projDep.isDeclarationFile &&
isFrontendIsland(packageNameOfSpecifier(spec))
)
continue;
if (projDep !== null && projDep.isDeclarationFile) {
diags.push(
unsupportedDiag(
@@ -5991,6 +6031,7 @@ function preflight7(load: LoadResult): {
`Invalid module "#" is not a valid internal imports specifier name imported from ${sf.fileName}`,
"%TypeError",
`the '#' import (Node rejects the bare '#' specifier as invalid)`,
"ERR_INVALID_MODULE_SPECIFIER",
);
continue;
}
@@ -6011,6 +6052,7 @@ function preflight7(load: LoadResult): {
`Package import specifier "${spec}" is not defined${pkgJson !== null ? ` in package ${pkgJson}` : ""} imported from ${sf.fileName}`,
"%TypeError",
`the '${spec}' import (no package.json imports-field entry maps it)`,
"ERR_PACKAGE_IMPORT_NOT_DEFINED",
);
continue;
}
@@ -6019,7 +6061,7 @@ function preflight7(load: LoadResult): {
const ambientNote = ambientDeclared(spec)
? `the '${spec}' module (it exists only as an ambient 'declare module' type surface — there is no runtime module to import)`
: `the '${spec}' package (nothing installed resolves it)`;
refuse(refusal.message, "%Error", ambientNote);
refuse(refusal.message, "%Error", ambientNote, refusal.code);
continue;
}
// Runtime-resolvable (or a shape the probe stays conservative
@@ -6125,7 +6167,12 @@ function preflight7(load: LoadResult): {
),
);
}
} else if (dep === null && !(!isRelative && ambientModules.has(spec))) {
} else if (
dep === null &&
!(!isRelative && ambientModules.has(spec)) &&
// A Node island module's namespace (scope-B NI-4).
!(!isRelative && isFrontendIsland(packageNameOfSpecifier(spec)))
) {
diags.push(unsupportedDiag("SC1013", locOf7(clause.namedBindings)));
}
}
@@ -6485,6 +6532,7 @@ function preflight7(load: LoadResult): {
// them). A found resolution crash wins over the CJS link check below:
// Node's fetch/resolve phase runs before instantiate.
let resolveCrash: StartupCrash | null = null;
const dynamicImportRejections = new Map<ts.SourceFile, StartupCrash>();
{
const seen = new Set<ts.SourceFile>();
const walk = (sf: ts.SourceFile): StartupCrash | null => {
@@ -6504,6 +6552,45 @@ function preflight7(load: LoadResult): {
return null;
};
resolveCrash = walk(entry);
// A module only a literal import() loads resolves its static graph when
// that import runs: Node rejects the import's promise with the first
// refusal it meets, before any module of that graph evaluates. Such a
// module never joins the compiled graph, and the refusals inside its
// graph are that rejection, not build failures.
const reach = (from: ts.SourceFile, into: Set<ts.SourceFile>): void => {
const stack = [from];
while (stack.length > 0) {
const sf = stack.pop()!;
if (into.has(sf)) continue;
into.add(sf);
for (const { dep } of depPositions.get(sf) ?? []) stack.push(dep);
}
};
const startup = new Set<ts.SourceFile>();
reach(entry, startup);
const covered = new Set<ts.SourceFile>();
for (const sf of userFiles) {
if (!sf.text.includes("import")) continue;
for (const call of moduleSourceCandidates(program, sf).calls) {
const arg = call.arguments.nodes[0];
if (call.expression.kind !== ts.SyntaxKind.ImportKeyword || arg === undefined) continue;
if (!ts.isStringLiteralLike(arg) || isUnloadedPackage(arg.text)) continue;
const root =
resolveImport7(program, sf, arg.text) ?? npmStaticDepSf7(program, sf, arg.text);
if (root === null || root.isDeclarationFile || startup.has(root)) continue;
// A module inside an already-rejected graph is `covered`, but a
// literal import() of it needs its OWN rejection entry: its graph
// rejects when that import runs, so it must not join the compiled
// graph (where lowering would fail) merely because another rejected
// import also reaches it. Walk each direct root independently.
if (dynamicImportRejections.has(root)) continue;
seen.clear();
const crash = walk(root);
if (crash === null) continue;
dynamicImportRejections.set(root, crash);
reach(root, covered);
}
}
// The crash channel engages only for programs that OTHERWISE pass
// preflight: when any other diagnostic already fails the build, every
// refusal keeps its compile-fence fallback instead — the report names
@@ -6511,7 +6598,8 @@ function preflight7(load: LoadResult): {
// get a diagnostic. (No reachable refusal → same: fallbacks only.)
if (resolveCrash === null || diags.length > 0) {
resolveCrash = null;
for (const cands of refusals.values()) {
for (const [sf, cands] of refusals) {
if (covered.has(sf) && !startup.has(sf)) continue;
for (const c of cands) diags.push(c.fallback);
}
}
@@ -6532,6 +6620,7 @@ function preflight7(load: LoadResult): {
startupCrash: resolveCrash ?? linkCrash,
inlineRequireCycleCalls,
moduleEdges: edges,
dynamicImportRejections,
};
}
packages/compiler/src/frontend/resolve.ts
+4
View File
@@ -24,6 +24,7 @@ import {
} from "./npm-static.js";
import {
clearPackageShimCaches,
isUnloadedPackage,
packageShimFor,
throughBundledShimForwarder,
} from "./package-shims.js";
@@ -1026,6 +1027,9 @@ export function resolveWorkspaceSourceModule(
) {
return null;
}
// A workspace package the project marks never loaded (package-shims.ts)
// resolves like a missing one.
if (isUnloadedPackage(specifier)) return null;
const source = resolveBareModule(fromFile, specifier, "runtime-source");
if (
source?.workspaceDir === undefined ||
packages/compiler/src/frontend/type-mapper.ts
+127 -1
View File
@@ -19,6 +19,7 @@ import {
CRYPTOHMAC_T,
DATE_T,
DYN,
FOREIGN,
F64,
funcOf,
isSupportedArrayElem,
@@ -1366,6 +1367,10 @@ export type TypeParamTsResolver = (t: ts.Type) => ts.Type | null;
* once per Lowerer and threaded through the recursion — adding a hook means
* one new field here, not a new parameter at every call site. */
export interface TypeMapperCtx {
/** Node islands (scope-B NI-4): whether a type's values are island
* handles, which map to the `foreign` type (a dyn value). Absent in
* programs without islands. */
foreignType?: (type: ts.Type) => boolean;
/** Set only while a union maps an arm that is the type inferred from an
* empty object literal, beside another non-nullish arm: that arm holds
* only the literal's objects, so it maps to the empty record (see the
@@ -2006,6 +2011,7 @@ function mapTypeEntry(type: ts.Type, ctx: TypeMapperCtx): IrType | null {
return STRING;
if (flags & (ts.TypeFlags.Boolean | ts.TypeFlags.BooleanLiteral)) return BOOL;
}
if (ctx.foreignType !== undefined && ctx.foreignType(type)) return FOREIGN;
const localValue = ctx.localClassValue?.(type);
if (localValue) {
contextResolutions++;
@@ -2290,6 +2296,11 @@ function mapTypeInner(type: ts.Type, ctx: TypeMapperCtx): IrType | null {
return viaConditional;
}
}
const viaDistribution = mapDistributedConditional(type, ctx);
if (viaDistribution !== null) {
contextResolutions++;
return viaDistribution;
}
}
}
const widened = checker.getBaseTypeOfLiteralType(type);
@@ -5302,11 +5313,19 @@ function mapTypeInner(type: ts.Type, ctx: TypeMapperCtx): IrType | null {
!checker.isNeverType(part) &&
!isEmptyObjectLiteralType(part, ctx),
);
let neverConditionalPart = false;
let undefinedPart: ts.Type | undefined;
for (const part of parts) {
if (hasArrayPart && emptyTuple(part)) continue;
// TypeScript's client can retain impossible intersections in a
// distributed union. They have no inhabitants and no runtime tag.
if (checker.isNeverType(part)) continue;
// A conditional over a bound parameter whose every member selects
// `never` contributes no arm either.
if (distributedConditionalParts(part, ctx)?.length === 0) {
neverConditionalPart = true;
continue;
}
// A `void` PART is inhabited only by undefined (`Promise<void> |
// void` return types, `string | void`): it becomes the undefinedT
// unit arm, exactly like an undefined part — the value either
@@ -5314,6 +5333,7 @@ function mapTypeInner(type: ts.Type, ctx: TypeMapperCtx): IrType | null {
// void stays VOID (the return-position mapping above).
if (part.flags & (ts.TypeFlags.Undefined | ts.TypeFlags.Void)) {
byKey.set(typeKey(UNDEFINED_T), UNDEFINED_T);
undefinedPart ??= part;
continue;
}
if (part.flags & ts.TypeFlags.Null) {
@@ -5356,6 +5376,11 @@ function mapTypeInner(type: ts.Type, ctx: TypeMapperCtx): IrType | null {
}
const arms = [...byKey.values()];
if (arms.length === 0) return F64; // same unreachable placeholder as standalone never
// `Cond<T> | undefined` whose conditional selects never for the
// instance's binding is the plain `undefined` the concrete
// instantiation's optional member has.
if (neverConditionalPart && arms.length === 1 && undefinedPart !== undefined)
return mapType(undefinedPart, ctx);
// A single surviving UNIT arm cannot stand alone (degenerate — the
// checker collapsed everything else away); anything else single is
// just that type. A single arm UNDER A MINTED PLACEHOLDER cannot
@@ -6442,6 +6467,104 @@ function mapGenericIndexedAccess(type: ts.Type, ctx: TypeMapperCtx): IrType | nu
return { kind: "union", unionId: ctx.unions.intern(arms) };
}
/** A distributive conditional (`T extends "a" ? A : T extends "b" ? B :
* never`, its check type a naked type parameter) over a parameter the
* instance binds: evaluated once per member of the binding (`"a" | "b"`,
* or the open `"a" | (string & {})` spelling), as the checker distributes
* it. Answers the non-never results (empty when every member selects
* never, so an optional member of that type holds only undefined, as in
* the concrete instantiation), or null when the conditional is another
* shape or a member does not evaluate. */
function distributedConditionalParts(type: ts.Type, ctx: TypeMapperCtx): ts.Type[] | null {
const { checker, resolveTypeParamTs } = ctx;
if (!resolveTypeParamTs || !type.isConditionalType()) return null;
const checkType = type.getCheckType();
if (!checkType || !(checkType.flags & ts.TypeFlags.TypeParameter)) return null;
const symbol = checkType.getSymbol();
const bound = resolveTypeParamTs(checkType);
if (!symbol || !bound) return null;
// The answer depends on the instance's binding: a mapping that consults
// it, even one that drops an all-never part, is not context-free.
contextResolutions++;
const parts: ts.Type[] = [];
for (const member of bound.isUnionType() ? ts.constituentTypes(bound) : [bound]) {
// `string & {}` (the open-union spelling) extends exactly what `string`
// extends: the empty object adds nothing to a primitive.
const emptyObject = (part: ts.Type): boolean =>
(part.flags & ts.TypeFlags.Object) !== 0 &&
checker.getPropertiesOfType(part).length === 0 &&
checker.getIndexInfosOfType(part).length === 0 &&
checker.getCallSignatures(part).length === 0 &&
checker.getConstructSignatures(part).length === 0;
const primitive = member.isIntersectionType()
? ts.constituentTypes(member).filter((part) => !emptyObject(part))
: [member];
if (primitive.length !== 1) return null;
const evaluated = evalMappedTemplate(type, new Map(), new Map([[symbol, primitive[0]!]]), ctx);
if (!evaluated) return null;
if (checker.isNeverType(evaluated)) continue;
if (!parts.includes(evaluated)) parts.push(evaluated);
}
return parts;
}
/** A unit-only type once its conditionals over bound parameters are
* distributed: `Cond<T> | undefined` where every member of T's binding
* selects never (the optional member's type in a generic body). */
function neverConditionalUnitOnly(t: ts.Type, ctx: TypeMapperCtx): boolean {
const UNIT = ts.TypeFlags.Undefined | ts.TypeFlags.Void | ts.TypeFlags.Null;
if (!t.isUnionType()) return false;
let conditional = false;
for (const part of ts.constituentTypes(t)) {
if ((part.flags & UNIT) !== 0) continue;
if (distributedConditionalParts(part, ctx)?.length !== 0) return false;
conditional = true;
}
return conditional;
}
/** mapType of a distributed conditional (distributedConditionalParts): the
* union of its members' results, as the concrete conditional's union maps
* them. An `unknown` or `any` result absorbs the rest (the checker's union
* is that type), as does a result that maps to the checked-dynamic value
* (`{}`, `object`, `Object`: the checker's union of one with a concrete
* result is the top type), and `undefined` and `null` results are the unit
* arms. Null when it does not distribute, when a result does not map, when
* every member selects never (the enclosing union drops such a part), or
* when only units remain. */
function mapDistributedConditional(type: ts.Type, ctx: TypeMapperCtx): IrType | null {
const parts = distributedConditionalParts(type, ctx);
if (!parts || parts.length === 0) return null;
const top = parts.find((part) => (part.flags & (ts.TypeFlags.Any | ts.TypeFlags.Unknown)) !== 0);
if (top !== undefined) return mapType(top, ctx);
const arms: IrType[] = [];
for (const part of parts) {
if (part.flags & (ts.TypeFlags.Undefined | ts.TypeFlags.Void)) {
if (!arms.some((a) => typeEquals(a, UNDEFINED_T))) arms.push(UNDEFINED_T);
continue;
}
if (part.flags & ts.TypeFlags.Null) {
if (!arms.some((a) => typeEquals(a, NULL_T))) arms.push(NULL_T);
continue;
}
const mapped = mapType(part, ctx);
if (!mapped) return null;
// A top-type result (`{}`, `object`, `Object`): the checker's union with
// a concrete result is the top type itself, so the member is
// checked-dynamic. A `dyn` arm would be invalid union IR (SC9001).
if (mapped.kind === "dyn") return DYN;
const members =
mapped.kind === "union" ? (ctx.unions.get(mapped.unionId)?.arms ?? null) : [mapped];
if (!members) return null;
for (const arm of members) if (!arms.some((a) => typeEquals(a, arm))) arms.push(arm);
}
if (arms.every(isUnitType)) return null;
if (arms.length === 1) return arms[0]!;
if (!unionContainerArmsOk(arms)) return null;
arms.sort((a, b) => (typeKey(a) < typeKey(b) ? -1 : 1));
return { kind: "union", unionId: ctx.unions.intern(arms) };
}
/** `keyof T extends never ? A : B` over a bound record T (the lib's
* Object.entries element spelling): decided by whether the binding has any
* key. */
@@ -8577,7 +8700,10 @@ function mapRecordTypeInner(
// absent-field idiom — and `{ p: undefined }` spellings): the
// unit-only union. The runtime value is the interned unit, JSON
// omits the undefined arm exactly like an omitted optional.
if (pt?.kind === "void" && isUnitOnlyTsType(fieldTs, ctx.resolveTypeParam))
if (
pt?.kind === "void" &&
(isUnitOnlyTsType(fieldTs, ctx.resolveTypeParam) || neverConditionalUnitOnly(fieldTs, ctx))
)
pt = unitOnlyUnion(ctx.unions);
if (pt !== null && isUnitOnlyIr(pt, ctx) && jsObjectLiteralProperty(p)) pt = DYN;
if (pt?.kind === "array" && jsNeverArrayProp(p, fieldTs, checker)) pt = DYN;
packages/compiler/src/index.ts
+12 -2
View File
@@ -1,3 +1,4 @@
import { createRequire } from "node:module";
import { shapeOverlayEnabled } from "./frontend/shape-inference/index.js";
import { nodeFrontendServices } from "./frontend/services-node.js";
import {
@@ -9,6 +10,8 @@ import { optimizationField, type NativeOptimization } from "./backend/optimizati
import { targetCpuField, targetCpuTargetRefusal } from "./backend/target-cpu.js";
import { compilationTiming, type CompilationTiming, type TimingClock } from "./timing.js";
import { prepareExecutableModule } from "./executable/prepare.js";
import { nodeIslandOptionsKey } from "./node-islands/policy.js";
import { setIslandResolver } from "./node-islands/resolve.js";
import {
prepareLibrary,
libraryLocalizeSymbols,
@@ -458,6 +461,10 @@ export async function analyzeCached(
return result;
}
// Island code resolves through Node (node-islands/resolve.ts; the
// self-hosted compiler builds no islands).
setIslandResolver((fromFile, spec) => createRequire(fromFile).resolve(spec));
const nodeFrontend: FrontendFactory = (
entry,
npmStatic,
@@ -926,7 +933,7 @@ async function prepareExecutableInput(
if (!moduleRuntimeFeatures(mod).nodeIsland) {
delete mod.nodeIsland;
} else if (outputKind === "exe") {
const recorded = libnodeRecord(opts.libnode, opts.outPath);
const recorded = libnodeRecord(opts.libnode, opts.outPath, process.env, dirname(entryPath));
if (typeof recorded === "string")
return { ok: false, diagnostics: [nodeIslandNodeDiag(recorded, entryPath)], sourceTexts };
mod.nodeIsland = { ...mod.nodeIsland, ...recorded };
@@ -1762,7 +1769,7 @@ async function compileTracked(
...(opts.detachedExit === true ? { detachedExit: true as const } : {}),
// Calls into Node record the libnode pack; their entries record its
// identity (node-islands/libnode.ts) and a hit requires it unchanged.
libnodeIdentity: () => libnodeCacheKey(opts.libnode),
libnodeIdentity: () => libnodeCacheKey(opts.libnode, process.env, dirname(entryPath)),
...(opts.largePages === true ? { largePages: true as const } : {}),
...targetCpuField(opts.targetCpu),
...(opts.cpuDispatch === true ? { cpuDispatch: true as const } : {}),
@@ -1770,6 +1777,9 @@ async function compileTracked(
...(functionLayoutDigest === undefined ? {} : { functionLayout: functionLayoutDigest }),
...checkerField(),
...(youngNurseryRequested() ? { youngNursery: true as const } : {}),
...(nodeIslandOptionsKey(opts.nodeIslands) !== ""
? { nodeIslands: nodeIslandOptionsKey(opts.nodeIslands) }
: {}),
...(heapCensusRequested() ? { heapCensus: true as const } : {}),
npmStatic: opts.npmStatic ?? null,
...(shapeOverlayEnabled() ? { shapeOverlay: true as const } : {}),
packages/compiler/src/ir/builtin-effects.ts
+11
View File
@@ -10,6 +10,17 @@ import type { IrLibFn } from "./builtin-signatures.js";
export const MAY_THROW_LIB_FNS: ReadonlySet<IrLibFn> = new Set([
// The copied error of a synchronous call into Node (@scriptc/node).
"nodeIsland.callSync",
// Island module evaluation and member operations run JavaScript.
"nodeIsland.require",
"nodeIsland.get",
"nodeIsland.getKey",
"nodeIsland.set",
"nodeIsland.setKey",
"nodeIsland.invoke",
"nodeIsland.apply",
"nodeIsland.applyTo",
"nodeIsland.construct",
"nodeIsland.instanceof",
// Node's argument, cipher-name, IV, tag-length and key-length errors.
"crypto.cipherNew",
// Node's curve, key, point, encoding and format errors.
packages/compiler/src/ir/builtin-signatures.ts
+17
View File
@@ -185,6 +185,23 @@ function buildLibFnSignatures() {
argTypes: [STRING, STRING, STRING, DYN],
result: { kind: "promise", inner: DYN },
},
/** Node islands (scope-B NI-4, scr_node_island_handle.c): island values
* are dyn values (`foreign` lowers as dyn), objects are handle boxes
* whose identity is the V8 object's. require/import load an island
* module by its absolute (or `$ORIGIN/`) path (also @scriptc/node
* requireModule); the rest are member reads,
* writes and calls on island values, argument lists as one dyn array. */
"nodeIsland.require": { argTypes: [STRING], result: DYN },
"nodeIsland.import": { argTypes: [STRING], result: { kind: "promise", inner: DYN } },
"nodeIsland.get": { argTypes: [DYN, STRING], result: DYN },
"nodeIsland.getKey": { argTypes: [DYN, DYN], result: DYN },
"nodeIsland.set": { argTypes: [DYN, STRING, DYN], result: VOID },
"nodeIsland.setKey": { argTypes: [DYN, DYN, DYN], result: VOID },
"nodeIsland.invoke": { argTypes: [DYN, STRING, DYN], result: DYN },
"nodeIsland.apply": { argTypes: [DYN, DYN], result: DYN },
"nodeIsland.applyTo": { argTypes: [DYN, DYN, DYN], result: DYN },
"nodeIsland.construct": { argTypes: [DYN, DYN], result: DYN },
"nodeIsland.instanceof": { argTypes: [DYN, DYN], result: BOOL },
/** Keyed WRITE on a dyn value — `h.onDone = cb` / `h["k"] = v` on a
* checked-dynamic object (args: receiver, key string, value — all
* borrowed; the runtime copies the key and retains the value in). An
packages/compiler/src/ir/inline-array-sources.ts
+83 -32
View File
@@ -24,9 +24,18 @@
* Arrays are followed shallowly: only the array objects matter, not the
* elements, which stay ordinary mutable instances. */
import type { IrExpr, IrFunction, IrGlobal, IrModule, IrStmt, IrType, SrcLoc } from "./ir.js";
import type {
IrClassDef,
IrExpr,
IrFunction,
IrGlobal,
IrModule,
IrStmt,
IrType,
SrcLoc,
} from "./ir.js";
import { typeKey } from "./ir.js";
import type { InlineArrayPlan } from "./inline-arrays.js";
import { readsArguments, type InlineArrayPlan } from "./inline-arrays.js";
import { everyExprChild, everyStmtChild } from "./traverse.js";
/** A use that breaks an InlineArray field's contract. */
@@ -88,25 +97,6 @@ const RETURNS_RECEIVER: ReadonlySet<string> = new Set([
"copyWithin",
]);
/** Library calls that only read their arguments: printing, inspection,
* and JSON.stringify without a replacer function. */
function readsArguments(e: IrExpr & { kind: "libCall" }): boolean {
if (/^(console|insp|inspect)\./.test(e.fn)) return true;
switch (e.fn) {
case "array.hasProperties":
case "array.hasNamedKey":
case "array.namedKeys":
case "json.stringifyValue":
return true;
case "json.stringifyReplacer": {
const replacer = e.args[1];
return replacer?.kind === "dynFrom" && replacer.value.kind === "unitLit";
}
default:
return false;
}
}
/** Expressions whose value is one of their children's. */
const PASS_THROUGH: ReadonlySet<IrExpr["kind"]> = new Set([
"upcast",
@@ -510,6 +500,8 @@ class SourceAnalysis {
* one). */
private readonly elementRoots = new Set<string>();
private readonly bases = new Map<string, string | undefined>();
private readonly classDefs = new Map<string, IrClassDef>();
private readonly subclasses = new Map<string, string[]>();
private readonly globals = new Map<string, IrGlobal>();
/** The slot graph (see "Arrays fields hold"). */
private readonly edges = new Map<string, Set<string>>();
@@ -557,7 +549,15 @@ class SourceAnalysis {
if (!list) this.methodImpls.set(m, (list = []));
list.push(`%${cls.name}.${m}`);
}
for (const cls of mod.classes ?? []) this.bases.set(cls.name, cls.base);
for (const cls of mod.classes ?? []) {
this.bases.set(cls.name, cls.base);
this.classDefs.set(cls.name, cls);
if (cls.base !== undefined) {
const list = this.subclasses.get(cls.base);
if (list === undefined) this.subclasses.set(cls.base, [cls.name]);
else list.push(cls.name);
}
}
for (const g of mod.globals ?? []) this.globals.set(g.id, g);
for (const cls of plan.classes.values()) this.elementRoots.add(this.rootOf(cls.elemClass));
}
@@ -1219,7 +1219,12 @@ class SourceAnalysis {
/** The field read whose array values of type `t` in `slots`, or the
* arrays, Maps, Sets and records inside them, may hold. */
private originWithin(slots: readonly string[], t: IrType, depth = 0): SrcLoc | undefined {
private originWithin(
slots: readonly string[],
t: IrType,
objects: boolean,
depth = 0,
): SrcLoc | undefined {
if (depth > 4) return undefined;
if (depth > 0 && this.holdsFieldArray(t))
for (const slot of slots) {
@@ -1230,29 +1235,67 @@ class SourceAnalysis {
switch (t.kind) {
case "array":
case "set":
return this.originWithin(elements, t.elem, depth + 1);
return this.originWithin(elements, t.elem, objects, depth + 1);
case "map":
return (
this.originWithin(elements, t.key, depth + 1) ??
this.originWithin(elements, t.value, depth + 1)
this.originWithin(elements, t.key, objects, depth + 1) ??
this.originWithin(elements, t.value, objects, depth + 1)
);
case "union":
for (const arm of this.arms(t)) {
const origin = this.originWithin(slots, arm, depth + 1);
const origin = this.originWithin(slots, arm, objects, depth + 1);
if (origin !== undefined) return origin;
}
return undefined;
case "record":
for (const f of (this.mod.records ?? []).find((r) => r.id === t.shapeId)?.fields ?? []) {
const origin = this.originWithin([`s:${t.shapeId}\0${f.name}`], f.type, depth + 1);
const origin = this.originWithin(
[`s:${t.shapeId}\0${f.name}`],
f.type,
objects,
depth + 1,
);
if (origin !== undefined) return origin;
}
return undefined;
// Code that reads an instance's fields by name (a dynamic view) can
// reach the arrays they hold. Other code reads them through field
// reads, which the graph follows.
case "object":
return objects ? this.objectOrigin(t.className, depth) : undefined;
default:
return undefined;
}
}
/** `originWithin` through objects for an instance of `className` or of a
* subclass: the arrays its fields may hold. An inline field is left out: its reads are
* followed where they are. By `className\0depth`. */
private readonly objectOrigins = new Map<string, SrcLoc | null>();
private objectOrigin(className: string, depth: number): SrcLoc | undefined {
const key = `${className}\0${depth}`;
const known = this.objectOrigins.get(key);
if (known !== undefined) return known ?? undefined;
// A cycle through fields finds nothing new.
this.objectOrigins.set(key, null);
let found: SrcLoc | undefined;
const visit = (name: string): void => {
const cls = this.classDefs.get(name);
if (cls === undefined) return;
const inline = this.plan.classes.get(name)?.field;
for (const f of cls.fields) {
if (found !== undefined) return;
if (f.name === inline) continue;
found = this.originWithin([this.fieldSlot(name, f.name)], f.type, true, depth + 1);
}
for (const sub of this.subclasses.get(name) ?? []) if (found === undefined) visit(sub);
};
visit(className);
this.objectOrigins.set(key, found ?? null);
return found;
}
/** The slots of the parameters of `fn` that only ever hold the caller's
* arguments (never assigned or captured), by index. */
private ownParams(fn: IrFunction): Map<string, number> {
@@ -1505,11 +1548,19 @@ class SourceAnalysis {
};
// `deep`: arrays inside the value escape too (dynamic and library code
// reads them other than through the reads this pass follows).
const escape = (value: IrExpr, loc: SrcLoc, what: string, deep = true): void => {
// `objects`: through the fields of class instances too (a dynamic view
// reads them by name).
const escape = (
value: IrExpr,
loc: SrcLoc,
what: string,
deep = true,
objects = false,
): void => {
if (check === null) return;
const origin =
this.origin(fn, value) ??
(deep ? this.originWithin(this.slotsOf(fn, value), value.type) : undefined);
(deep ? this.originWithin(this.slotsOf(fn, value), value.type, objects) : undefined);
if (origin !== undefined)
check({
loc,
@@ -1648,10 +1699,10 @@ class SourceAnalysis {
}
case "dynFrom":
case "caughtToDyn":
if (!readSinks.has(e)) escape(e.value, e.loc, "a dynamic value");
if (!readSinks.has(e)) escape(e.value, e.loc, "a dynamic value", true, true);
break;
case "callValue":
for (const a of e.args) escape(a, e.loc, "a call through a function value", false);
for (const a of e.args) escape(a, e.loc, "a call through a function value");
break;
case "intrinsic":
// Printing reads its arguments; publishing freezes them.
packages/compiler/src/ir/inline-arrays.ts
+102 -19
View File
@@ -60,6 +60,8 @@ export interface InlineArrayRejection {
readonly declaringClass: string;
readonly field: string;
readonly reason: string;
/** The use that rules the layout out, when it is one site. */
readonly loc?: SrcLoc;
}
export interface InlineArrayPlan {
@@ -97,8 +99,8 @@ export function planInlineArrays(
const at = entry.indexOf("\0");
const declaringClass = entry.slice(0, at);
const field = entry.slice(at + 1);
const reject = (reason: string): void => {
rejected.push({ declaringClass, field, reason });
const reject = (reason: string, loc?: SrcLoc): void => {
rejected.push({ declaringClass, field, reason, ...(loc !== undefined ? { loc } : {}) });
};
const decl = classes.get(declaringClass);
if (decl === undefined) {
@@ -147,9 +149,9 @@ export function planInlineArrays(
}
lengthParams.set(name, param);
}
reason ??= otherUses(mod, members, field, declaringClass);
if (reason !== null) {
reject(reason);
const other = reason === null ? otherUses(mod, members, field, declaringClass) : null;
if (reason !== null || other !== null) {
reject(reason ?? other!.reason, other?.loc);
continue;
}
for (const name of subtree)
@@ -258,17 +260,13 @@ function otherUses(
members: ReadonlySet<string>,
field: string,
declaringClass: string,
): string | null {
const reaches = (type: IrType): boolean => {
let found = false;
const visit = (t: IrType): boolean => {
if (t.kind === "object" && members.has(t.className)) found = true;
return !found && everyTypeChild(t, visit);
};
visit(type);
return found;
};
): { reason: string; loc?: SrcLoc } | null {
const reaches = holdersOf(mod, members, declaringClass);
let reason: string | null = null;
let loc: SrcLoc | undefined;
// Dynamic values that printing, JSON.stringify without a replacer and
// publishing only read, and the options a worker copies its data from.
const readOnly = new Set<IrExpr>();
for (const fn of mod.functions) {
const isDeclaringCtor = fn.name === `%${declaringClass}.constructor`;
everyStmtList(fn.body, {
@@ -276,9 +274,21 @@ function otherUses(
if (reason !== null) return false;
if (e.kind === "classRef" && members.has(e.className))
reason = `${e.className} is used as a value`;
else if ((e.kind === "dynFrom" || e.kind === "caughtToDyn") && reaches(e.value.type))
reason = "an instance becomes a dynamic value";
else if (e.kind === "fieldIncDec" && e.field === field && members.has(e.className))
else if (
(e.kind === "libCall" && (readsArguments(e) || e.fn === "worker.new")) ||
(e.kind === "intrinsic" &&
(e.name.startsWith("console.") || e.name === "threads.publish"))
)
for (const a of e.args) readOnly.add(a);
else if (
(e.kind === "dynFrom" || e.kind === "caughtToDyn") &&
!readOnly.has(e) &&
reaches(e.value.type)
) {
reason =
"an instance, or a value that can hold one, becomes a dynamic value (a dynamic view reads the field as a new array each time)";
loc = e.loc;
} else if (e.kind === "fieldIncDec" && e.field === field && members.has(e.className))
reason = "the field is written outside the constructor";
else if (
e.kind === "bin" &&
@@ -301,11 +311,84 @@ function otherUses(
return reason === null;
},
});
if (reason !== null) return reason;
if (reason !== null) return loc === undefined ? { reason } : { reason, loc };
}
return null;
}
/** Whether values of a type may be, or hold, an instance of `members`
* (the subtree of `declaringClass`): a value typed as a member or as a base
* of the declaring class, or a class instance, record, union, array, Map or
* Set whose fields or elements may hold one. A dynamic view of any of them
* reads the instance's fields by name. */
function holdersOf(
mod: IrModule,
members: ReadonlySet<string>,
declaringClass: string,
): (type: IrType) => boolean {
const classes = new Map((mod.classes ?? []).map((c) => [c.name, c]));
const arms = new Map((mod.unions ?? []).map((u) => [u.id, u.arms]));
const records = new Map((mod.records ?? []).map((r) => [r.id, r.fields]));
const holderClasses = new Set(members);
const holderShapes = new Set<string>();
const addWithBases = (name: string): boolean => {
let added = false;
for (let at: string | undefined = name; at !== undefined; at = classes.get(at)?.base)
if (!holderClasses.has(at)) {
holderClasses.add(at);
added = true;
}
return added;
};
addWithBases(declaringClass);
const holds = (type: IrType): boolean => {
const seen = new Set<string>();
let found = false;
const visit = (t: IrType): boolean => {
if (t.kind === "object") found = holderClasses.has(t.className);
else if (t.kind === "record") found = holderShapes.has(t.shapeId);
else if (t.kind === "union" && !seen.has(t.unionId)) {
seen.add(t.unionId);
for (const arm of arms.get(t.unionId) ?? []) if (!visit(arm)) break;
}
return !found && everyTypeChild(t, visit);
};
visit(type);
return found;
};
for (let changed = true; changed;) {
changed = false;
for (const cls of mod.classes ?? [])
if (!holderClasses.has(cls.name) && cls.fields.some((f) => holds(f.type)))
changed = addWithBases(cls.name) || changed;
for (const [shapeId, fields] of records)
if (!holderShapes.has(shapeId) && fields.some((f) => holds(f.type))) {
holderShapes.add(shapeId);
changed = true;
}
}
return holds;
}
/** Library calls that only read their arguments: printing, inspection,
* and JSON.stringify without a replacer function. */
export function readsArguments(e: IrExpr & { kind: "libCall" }): boolean {
if (/^(console|insp|inspect)\./.test(e.fn)) return true;
switch (e.fn) {
case "array.hasProperties":
case "array.hasNamedKey":
case "array.namedKeys":
case "json.stringifyValue":
return true;
case "json.stringifyReplacer": {
const replacer = e.args[1];
return replacer?.kind === "dynFrom" && replacer.value.kind === "unitLit";
}
default:
return false;
}
}
function readsField(e: IrExpr, members: ReadonlySet<string>, field: string): boolean {
return e.kind === "fieldGet" && e.field === field && members.has(e.className);
}
packages/compiler/src/ir/ir.ts
+8
View File
@@ -618,6 +618,14 @@ export const CHILDWRITER_T: IrType = { kind: "childWriter" };
export const PROCSTREAM_T: IrType = { kind: "procStream" };
export const VOID: IrType = { kind: "void" };
export const DYN: IrType = { kind: "dyn" };
/** The `foreign` type (scope-B NI-4): a value owned by a Node island,
* typed by the island package's declarations. Objects stay in V8 and cross
* as handles (identity is the V8 object's); member operations compile from
* the declarations into `nodeIsland.*` bridge calls. NI-4 represents it
* exactly as `dyn` (a handle box with the island tag, primitives as plain
* dyn values), so foreign -> any is the identity and the generic dyn paths
* (equality, inspection, any slots) apply unchanged. */
export const FOREIGN: IrType = DYN;
export const JSVAL: IrType = { kind: "jsval" };
export const CAUGHT: IrType = { kind: "caught" };
export const UNDEFINED_T: IrType = { kind: "undefinedT" };
packages/compiler/src/ir/node-index.ts
+7
View File
@@ -196,6 +196,13 @@ export class IrNodeIndex {
return scan;
}
/** The number of nodes recorded for `fn`, or undefined when `fn` is not
* recorded. */
nodeCount(fn: IrFunction): number | undefined {
const f = this.functionIndex.get(fn);
return f === undefined ? undefined : this.starts[f + 1]! - this.starts[f]!;
}
/** Calls `note` with each recorded node of `fn` whose kind `scan` names,
* in walk order, until it answers false. False (nothing called) when
* `fn` is not recorded. */
packages/compiler/src/native/cli.ts
+2
View File
@@ -36,6 +36,7 @@ declare function compilerNativeCollectCycles(): void;
declare function compilerNativeFork(): number;
declare function compilerNativeExit(code: number): void;
declare function compilerNativeWait(pid: number): number;
declare function compilerNativeExited(): number;
declare function compilerNativeLockOpen(path: string): boolean;
declare function compilerNativeLock(): void;
declare function compilerNativeUnlock(): void;
@@ -73,6 +74,7 @@ async function main(): Promise<number> {
fork: () => compilerNativeFork(),
exit: (code) => compilerNativeExit(code),
wait: (pid) => compilerNativeWait(pid),
exited: () => compilerNativeExited(),
openLock: (path) => compilerNativeLockOpen(path),
lock: () => compilerNativeLock(),
unlock: () => compilerNativeUnlock(),
packages/compiler/src/native/prepare.ts
+1
View File
@@ -603,5 +603,6 @@ function emitForkPrimitives(): ForkPrimitives | undefined {
fork: () => primitives.fork(),
exit: (code) => primitives.exit(code),
wait: (pid) => primitives.wait(pid),
exited: () => primitives.exited(),
};
}
packages/compiler/src/node-islands/boundary.ts
+157
View File
@@ -0,0 +1,157 @@
/* The Node island boundary (DESIGN.md §2.4–§2.5, scope-B NI-4): which
* values of a compiled program are island values, and what crosses by
* copy. Rule (decision 2): a value crosses by reference unless copying it
* is unobservable. Primitives and strings are copied; every object stays a
* handle whose identity is the V8 object's.
*
* Island values are typed by the island package's own declarations. A
* checker type is FOREIGN when it is an object type declared in an island
* package (its .d.ts, or its @types twin): a class, interface, type alias
* or type literal there, the type of an exported function or class value,
* or an island module's namespace. Values of foreign types are held as
* island handles (the `foreign` IR type, represented as a dyn value), and
* member operations on them compile from the declarations into bridge
* calls. A union is foreign when every arm that is not null or undefined
* is foreign (the dyn representation holds null and undefined). */
import * as ts from "../frontend/ts7/adapter.js";
import { packageDirOf } from "./origin.js";
/** One package the build runs in a Node island. */
export interface IslandPackage {
name: string;
version: string | null;
/** The package directory (…/node_modules/<name>). */
dir: string;
/** The island class (runtime, object-model, gap, packaging) and the first
* reason, for the SC8001 note. */
class: string;
reason: string;
}
/** The program's island packages, by package name. */
export interface NodeIslandSet {
packages: ReadonlyMap<string, IslandPackage>;
}
/** The island package that owns a declaration or JavaScript file, or null.
* `@types/<name>` declarations type the package `<name>`. */
export function islandPackageOfFile(islands: NodeIslandSet, file: string): IslandPackage | null {
const at = packageDirOf(file);
if (at === null) return null;
let name = at.name;
if (name.startsWith("@types/")) {
const bare = name.slice("@types/".length);
name = bare.includes("__") ? `@${bare.replace("__", "/")}` : bare;
}
return islands.packages.get(name) ?? null;
}
function declaredInIsland(islands: NodeIslandSet, symbol: ts.Symbol | null | undefined): boolean {
if (!symbol) return false;
const decls = symbol.declarations;
if (decls.length === 0) return false;
return decls.every(
(d) => islandPackageOfFile(islands, ts.getSourceFileOfNode(d).fileName) !== null,
);
}
const NULLISH = ts.TypeFlags.Null | ts.TypeFlags.Undefined | ts.TypeFlags.Void;
/** Whether values of `type` are island handles (see the header). An
* anonymous object type every property of which the island declares (the
* namespace TypeScript synthesizes for `import()` of a CommonJS island
* module) is foreign too. */
export function isForeignType(
islands: NodeIslandSet,
type: ts.Type,
checker?: ts.TypeChecker,
): boolean {
if (type.isUnionType()) {
const arms = ts.constituentTypes(type).filter((t) => (t.flags & NULLISH) === 0);
return arms.length > 0 && arms.every((t) => isForeignType(islands, t, checker));
}
// An intersection the island declares (execa's `ChildProcess &
// ExecaChildPromise & Promise<…>`), or one with an island-declared arm,
// is made by island code: its values are island objects.
if ((type.flags & ts.TypeFlags.Intersection) !== 0) {
if (declaredInIsland(islands, type.getAliasSymbol())) return true;
return ts.constituentTypes(type).some((t) => isForeignType(islands, t, checker));
}
if ((type.flags & ts.TypeFlags.Object) === 0) return false;
if (
declaredInIsland(islands, type.getAliasSymbol()) ||
declaredInIsland(islands, type.getSymbol())
)
return true;
if (checker === undefined || type.getAliasSymbol()) return false;
const symbol = type.getSymbol();
if (symbol && symbol.declarations.length > 0) return false;
// Synthetic members (the interop `default`) have no declarations.
let declared = 0;
for (const p of checker.getPropertiesOfType(type)) {
if (p.declarations.length === 0) continue;
if (!declaredInIsland(islands, p)) return false;
declared++;
}
return declared > 0;
}
/** How a value of a checker type crosses INTO an island as an argument or
* an assigned value: copied (primitives, strings, null/undefined), passed
* as its handle (foreign), passed as a dyn value the runtime converts
* (any/unknown: a handle crosses as its object, a primitive is copied, a
* fresh literal is copied), or not representable yet. */
export type BoundaryCrossing = "copy" | "handle" | "dyn" | "refused";
const COPIED =
ts.TypeFlags.NumberLike |
ts.TypeFlags.StringLike |
ts.TypeFlags.BooleanLike |
ts.TypeFlags.BigIntLike |
ts.TypeFlags.Null |
ts.TypeFlags.Undefined |
ts.TypeFlags.Void;
export function crossingOf(
islands: NodeIslandSet,
type: ts.Type,
checker?: ts.TypeChecker,
): BoundaryCrossing {
if (type.isUnionType()) {
const arms = ts.constituentTypes(type);
if (arms.every((t) => (t.flags & COPIED) !== 0)) return "copy";
if (isForeignType(islands, type, checker)) return "handle";
return "refused";
}
if ((type.flags & COPIED) !== 0) return "copy";
if ((type.flags & (ts.TypeFlags.Any | ts.TypeFlags.Unknown)) !== 0) return "dyn";
if (isForeignType(islands, type, checker)) return "handle";
return "refused";
}
/* The island set of the lowering in progress. lowerToIr sets it from its
* options for the duration of one synchronous lowering; every Lowerer it
* creates reads it at construction. Null in programs without islands, so
* their lowering is unchanged. */
let active: NodeIslandSet | null = null;
export function setActiveNodeIslands(islands: NodeIslandSet | null): void {
active = islands !== null && islands.packages.size > 0 ? islands : null;
}
export function activeNodeIslands(): NodeIslandSet | null {
return active;
}
/* The island package names of the frontend load in progress (runFrontend
* sets them for one synchronous load): preflight lets their imports
* through, typed by their declarations. */
let frontendIslands: ReadonlySet<string> = new Set();
export function setFrontendIslands(names: ReadonlySet<string>): void {
frontendIslands = names;
}
export function isFrontendIsland(pkg: string): boolean {
return frontendIslands.size > 0 && frontendIslands.has(pkg);
}
packages/compiler/src/node-islands/classify.ts
+1 -1
View File
@@ -25,7 +25,7 @@ const MESSAGE_REASONS: { code: string; pattern: RegExp; reason: string }[] = [
pattern: /createRequire's require with this argument shape/,
reason: "dynamic-require",
},
{ code: "SC2020", pattern: /native addon/, reason: "native-addon" },
{ code: "SC2020", pattern: /^native addon /, reason: "native-addon" },
{
code: "SC2020",
pattern: /through require\.cache|require\.extensions|Module\._|module\.register/,
packages/compiler/src/node-islands/libnode-pack.test.ts
+338
View File
@@ -0,0 +1,338 @@
import { createHash } from "node:crypto";
import {
existsSync,
mkdirSync,
mkdtempSync,
readFileSync,
readdirSync,
rmSync,
writeFileSync,
} from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { fileURLToPath } from "node:url";
import { afterEach, describe, expect, test } from "vitest";
import { compilerReleaseVersion } from "../library/sidecar.js";
import { NODE_BRIDGE_ABI, libnodeRecord } from "./libnode.js";
import {
LIBNODE_PACK_SCHEMA,
LibnodePackError,
findInstalledLibnodePack,
hostLibnodeTarget,
libnodePackPackage,
parseLibnodeSetting,
readLibnodePack,
writeLibnodeNotices,
} from "./libnode-pack.js";
const repo = fileURLToPath(new URL("../../../..", import.meta.url));
const dirs: string[] = [];
afterEach(() => {
for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true });
});
function temp(): string {
const dir = mkdtempSync(join(tmpdir(), "scriptc-libnode-pack-"));
dirs.push(dir);
return dir;
}
function entry(bytes: string) {
return {
size: Buffer.byteLength(bytes),
sha256: createHash("sha256").update(bytes).digest("hex"),
};
}
/** A published-style pack with stand-in libraries; `edit` adjusts the
* manifest build-pack.mjs would write. */
function fakePack(
dir: string,
edit: (manifest: Record<string, unknown>) => void = () => {},
name = "@scriptc/libnode-linux-x64-gnu",
): string {
mkdirSync(dir, { recursive: true });
const libnode = "stand-in libnode";
const bridge = "stand-in bridge";
writeFileSync(join(dir, "libnode.so.137"), libnode);
writeFileSync(join(dir, "libscr_node_bridge.so"), bridge);
writeFileSync(join(dir, "LICENSE"), "Node.js is licensed for use as follows:\n\nMIT\n");
writeFileSync(
join(dir, "package.json"),
JSON.stringify({ name, version: compilerReleaseVersion() }),
);
const manifest: Record<string, unknown> = {
schema: LIBNODE_PACK_SCHEMA,
nodeVersion: "24.22.0",
moduleVersion: 137,
bridgeAbi: NODE_BRIDGE_ABI,
platform: "linux",
arch: "x64",
libnode: "libnode.so.137",
bridge: "libscr_node_bridge.so",
package: name,
version: compilerReleaseVersion(),
target: "linux-x64-gnu",
license: "LICENSE",
files: { "libnode.so.137": entry(libnode), "libscr_node_bridge.so": entry(bridge) },
};
edit(manifest);
writeFileSync(join(dir, "manifest.json"), JSON.stringify(manifest));
return dir;
}
function thrown(run: () => unknown): LibnodePackError {
try {
run();
} catch (error) {
expect(error).toBeInstanceOf(LibnodePackError);
return error as LibnodePackError;
}
throw new Error("expected a LibnodePackError");
}
describe("the pack packages", () => {
test("every libnode platform package is the pack of one native target", () => {
const packages = readdirSync(join(repo, "packages"))
.filter((name) => name.startsWith("libnode-") && name !== "libnode-common")
.map(
(name) =>
JSON.parse(readFileSync(join(repo, "packages", name, "package.json"), "utf8")).name,
)
.sort();
const targets = ["linux-x64-gnu", "linux-arm64-gnu", "macos-arm64", "macos-x64"];
expect(targets.map((target) => libnodePackPackage(target)).sort()).toEqual(packages);
expect(libnodePackPackage("linux-x64-musl")).toBeNull();
expect(libnodePackPackage("windows-x64-msvc")).toBeNull();
expect(libnodePackPackage("wasm32-wasi")).toBeNull();
});
test("host targets map to native target names", () => {
expect(hostLibnodeTarget("linux", "x64")).toBe("linux-x64-gnu");
expect(hostLibnodeTarget("linux", "arm64")).toBe("linux-arm64-gnu");
expect(hostLibnodeTarget("darwin", "arm64")).toBe("macos-arm64");
expect(hostLibnodeTarget("darwin", "x64")).toBe("macos-x64");
});
});
describe("parseLibnodeSetting", () => {
test("accepts embedded, beside and absolute directories", () => {
expect(parseLibnodeSetting("embedded")).toEqual({ kind: "embedded" });
expect(parseLibnodeSetting("beside")).toEqual({ kind: "beside" });
expect(parseLibnodeSetting("/opt/scriptc/libnode-24.22.0")).toEqual({
kind: "path",
path: "/opt/scriptc/libnode-24.22.0",
});
});
test("rejects relative paths and other values", () => {
expect(parseLibnodeSetting("libnode")).toBe(
'must be "embedded", "beside" or an absolute directory, not the relative path "libnode"',
);
expect(parseLibnodeSetting(true)).toBe('must be "embedded", "beside" or an absolute directory');
});
});
describe("finding the installed pack", () => {
test("in the nearest node_modules above the project", () => {
const root = temp();
const pack = fakePack(join(root, "node_modules/@scriptc/libnode-linux-x64-gnu"));
const project = join(root, "apps/web/src");
mkdirSync(project, { recursive: true });
expect(
findInstalledLibnodePack({
targetName: "linux-x64-gnu",
projectDir: project,
resolveInstalled: () => null,
}),
).toBe(pack);
});
test("next to the compiler when the project has none", () => {
const pack = fakePack(join(temp(), "libnode-darwin-arm64"));
expect(
findInstalledLibnodePack({
targetName: "macos-arm64",
projectDir: temp(),
resolveInstalled: (name) => (name === "@scriptc/libnode-darwin-arm64" ? pack : null),
}),
).toBe(pack);
expect(
findInstalledLibnodePack({
targetName: "macos-arm64",
projectDir: temp(),
resolveInstalled: () => null,
}),
).toBeNull();
});
test("targets without packs have none", () => {
expect(findInstalledLibnodePack({ targetName: "linux-x64-musl", projectDir: temp() })).toBe(
null,
);
});
test("libnodeRecord uses the installed pack when nothing names one", () => {
const root = temp();
const name = libnodePackPackage(hostLibnodeTarget());
if (name === null) return;
const pack = fakePack(join(root, "node_modules", name), () => {}, name);
expect(libnodeRecord(undefined, join(root, "app"), {}, root)).toEqual({
libnode: pack,
nodeVersion: "24.22.0",
});
// SCRIPTC_LIBNODE still wins.
const other = fakePack(join(temp(), "other"), (m) => {
m["nodeVersion"] = "24.13.1";
});
expect(libnodeRecord(undefined, join(root, "app"), { SCRIPTC_LIBNODE: other }, root)).toEqual({
libnode: other,
nodeVersion: "24.13.1",
});
});
});
test("an installed pack of another scriptc version is refused", () => {
const root = temp();
const name = libnodePackPackage(hostLibnodeTarget());
if (name === null) return;
const pack = fakePack(
join(root, "node_modules", name),
(m) => {
m["version"] = "0.0.1";
},
name,
);
expect(libnodeRecord(undefined, join(root, "app"), {}, root)).toBe(
`the installed libnode pack ${name} at ${pack} is version 0.0.1; this scriptc is ${compilerReleaseVersion()}: install ${name}@${compilerReleaseVersion()}`,
);
});
test("an installed pack whose files do not match their digests is refused", () => {
const root = temp();
const name = libnodePackPackage(hostLibnodeTarget());
if (name === null) return;
const pack = fakePack(join(root, "node_modules", name), () => {}, name);
// Same size, other bytes.
writeFileSync(join(pack, "libnode.so.137"), "stand-in libnodX");
expect(libnodeRecord(undefined, join(root, "app"), {}, root)).toBe(
`libnode.so.137 in the libnode pack at ${pack} does not match its manifest`,
);
});
describe("readLibnodePack", () => {
test("reads a pack in build-pack.mjs's format and checks its package", () => {
const pack = fakePack(join(temp(), "pack"));
expect(
readLibnodePack(pack, { packageName: "@scriptc/libnode-linux-x64-gnu", verifyHashes: true })
.manifest.nodeVersion,
).toBe("24.22.0");
expect(
thrown(() => readLibnodePack(pack, { packageName: "@scriptc/libnode-linux-arm64-gnu" }))
.message,
).toBe(
`the libnode pack at ${pack} is @scriptc/libnode-linux-x64-gnu; this target needs @scriptc/libnode-linux-arm64-gnu`,
);
});
test("refuses other manifest schemas and file names outside the pack", () => {
const schema = fakePack(join(temp(), "pack"), (m) => {
m["schema"] = "scriptc.libnode-pack.v0";
});
expect(thrown(() => readLibnodePack(schema)).kind).toBe("invalid");
for (const name of [
"lib/libnode.so.137",
"../libnode.so.137",
"..",
"/etc/libnode.so",
"a\\b",
"a\0b",
]) {
const pack = fakePack(join(temp(), "pack"), (m) => {
m["libnode"] = name;
m["files"] = { ...(m["files"] as object), [name]: entry("stand-in libnode") };
});
expect(thrown(() => readLibnodePack(pack)).message).toBe(
`the libnode pack at ${pack} names a file outside the pack (${JSON.stringify(name)})`,
);
}
});
test("checks sizes always and hashes on request", () => {
const pack = fakePack(join(temp(), "pack"));
writeFileSync(join(pack, "libnode.so.137"), "stand-in libnodE");
expect(readLibnodePack(pack).root).toBe(pack);
expect(thrown(() => readLibnodePack(pack, { verifyHashes: true })).message).toBe(
`libnode.so.137 in the libnode pack at ${pack} does not match its manifest`,
);
writeFileSync(join(pack, "libnode.so.137"), "short");
expect(thrown(() => readLibnodePack(pack)).kind).toBe("invalid");
});
test("a directory without a manifest is missing", () => {
const dir = temp();
expect(thrown(() => readLibnodePack(dir)).message).toBe(
`no libnode pack at ${dir} (manifest.json is missing or unreadable)`,
);
});
});
test("the license notices carry Node's LICENSE", () => {
const pack = readLibnodePack(fakePack(join(temp(), "pack")));
const out = join(temp(), "server");
const path = writeLibnodeNotices(pack, out);
expect(path).toBe(`${out}.licenses.txt`);
expect(readFileSync(path, "utf8")).toBe(
[
`server runs part of its code in Node.js 24.22.0, loaded from libnode.so.137 (@scriptc/libnode-linux-x64-gnu@${compilerReleaseVersion()}).`,
"Node.js and the components it bundles are licensed as follows. The scriptc bridge,",
"libscr_node_bridge.so, is licensed under Apache-2.0.",
"",
"Node.js is licensed for use as follows:",
"",
"MIT",
"",
].join("\n"),
);
});
test("beside refuses an installed pack whose manifest names a file outside it", () => {
const root = temp();
const name = libnodePackPackage(hostLibnodeTarget());
if (name === null) return;
const pack = fakePack(
join(root, "node_modules", name),
(m) => {
m["files"] = { ...(m["files"] as object), "../escaped": entry("payload") };
},
name,
);
// The pack's "../escaped" and the file it would replace next to the output.
writeFileSync(join(pack, "..", "escaped"), "payload");
writeFileSync(join(root, "escaped"), "victim");
expect(libnodeRecord("beside", join(root, "app"), {}, root)).toBe(
`the libnode pack at ${pack} names a file outside the pack ("../escaped")`,
);
expect(readFileSync(join(root, "escaped"), "utf8")).toBe("victim");
expect(existsSync(join(root, "app.libnode"))).toBe(false);
});
test("readLibnodePack refuses any listed file or license outside the pack", () => {
for (const edit of [
(m: Record<string, unknown>) => {
m["files"] = { ...(m["files"] as object), "../escaped": entry("payload") };
},
(m: Record<string, unknown>) => {
m["files"] = { ...(m["files"] as object), "/etc/passwd": entry("payload") };
},
(m: Record<string, unknown>) => {
m["license"] = "../../secret";
},
(m: Record<string, unknown>) => {
m["license"] = 7;
},
]) {
const pack = fakePack(join(temp(), "pack"), edit);
expect(thrown(() => readLibnodePack(pack)).kind).toBe("invalid");
}
});
packages/compiler/src/node-islands/libnode-pack.ts
+267
View File
@@ -0,0 +1,267 @@
/* The installed libnode packs (scope-B design §2.11, §2.13, §2.16.3):
* `@scriptc/libnode-<platform>`, built only by the libnode-packs workflow
* from the pinned Node release (packages/libnode-common) and installed like
* the runtime packs. A scriptc build never compiles Node.
*
* libnode.ts decides which pack a build records: --libnode, else
* SCRIPTC_LIBNODE, else the installed pack this module finds: the nearest
* `node_modules/@scriptc/libnode-<platform>` walking up from the project
* directory, else the one next to the compiler. The manifest format is the
* one packages/libnode-common/build-pack.mjs writes (schema, nodeVersion,
* moduleVersion, bridgeAbi, platform, arch, libnode, bridge, files); the
* published packs add package, version, target, license and build inputs.
*
* Also here: the `nodeIslands.libnode` setting ("embedded" | "beside" |
* <absolute directory>) for the project configuration, and the license
* notices a program that redistributes Node.js carries. */
import { createHash } from "node:crypto";
import { existsSync, readFileSync, statSync, writeFileSync } from "node:fs";
import { createRequire } from "node:module";
import { basename, dirname, isAbsolute, join, resolve } from "node:path";
export const LIBNODE_PACK_SCHEMA = "scriptc.libnode-pack.v1";
/** The pack package for each native target that can host Node islands. */
const PACK_PACKAGES: Readonly<Record<string, string>> = {
"linux-x64-gnu": "@scriptc/libnode-linux-x64-gnu",
"linux-arm64-gnu": "@scriptc/libnode-linux-arm64-gnu",
"macos-arm64": "@scriptc/libnode-darwin-arm64",
"macos-x64": "@scriptc/libnode-darwin-x64",
};
export type LibnodePlacement =
| { kind: "embedded" }
| { kind: "beside" }
| { kind: "path"; path: string };
export interface LibnodeFile {
size: number;
sha256: string;
}
export interface LibnodePackManifest {
schema: string;
nodeVersion: string;
moduleVersion: number;
bridgeAbi: number;
/** process.platform and process.arch of the pack. */
platform: string;
arch: string;
/** File names inside the pack directory. */
libnode: string;
bridge: string;
files: Record<string, LibnodeFile>;
/** Published packs only. */
package?: string;
version?: string;
target?: string;
license?: string;
}
export interface LibnodePack {
root: string;
manifest: LibnodePackManifest;
}
export class LibnodePackError extends Error {
readonly kind: "missing" | "invalid" | "unsupported";
constructor(message: string, kind: "missing" | "invalid" | "unsupported") {
super(message);
this.name = "LibnodePackError";
this.kind = kind;
}
}
/** The libnode pack package for a native target, or null when Node islands
* have no pack there (musl, Windows, WASI and mobile targets). */
export function libnodePackPackage(targetName: string): string | null {
return PACK_PACKAGES[targetName] ?? null;
}
/** The native target name of the host, as libnodePackPackage takes it. */
export function hostLibnodeTarget(
platform: string = process.platform,
arch: string = process.arch,
): string {
return platform === "darwin" ? `macos-${arch}` : `${platform}-${arch}-gnu`;
}
/** Validates a `nodeIslands.libnode` value; a string result is the reason
* it is invalid. */
export function parseLibnodeSetting(value: unknown): LibnodePlacement | string {
if (value === "embedded") return { kind: "embedded" };
if (value === "beside") return { kind: "beside" };
if (typeof value === "string" && isAbsolute(value)) return { kind: "path", path: value };
if (typeof value === "string") {
return `must be "embedded", "beside" or an absolute directory, not the relative path "${value}"`;
}
return `must be "embedded", "beside" or an absolute directory`;
}
/** Whether `name` names a file directly in the pack directory. */
function packFileName(name: string): boolean {
return name !== "" && name !== "." && name !== ".." && !/[/\\\0]/.test(name) && !isAbsolute(name);
}
function isRecord(value: unknown): value is Record<string, unknown> {
return value !== null && typeof value === "object" && !Array.isArray(value);
}
function isFileEntry(value: unknown): value is LibnodeFile {
return (
isRecord(value) &&
typeof value["size"] === "number" &&
typeof value["sha256"] === "string" &&
/^[0-9a-f]{64}$/.test(value["sha256"])
);
}
/** Parses a pack's manifest.json; throws LibnodePackError when it is not a
* pack manifest. Bridge ABI and Node version policy stay in libnode.ts. */
export function parseLibnodePackManifest(json: unknown, where: string): LibnodePackManifest {
const invalid = (detail: string): never => {
throw new LibnodePackError(`the libnode pack at ${where} ${detail}`, "invalid");
};
if (!isRecord(json)) return invalid("has a malformed manifest.json");
if (json["schema"] !== LIBNODE_PACK_SCHEMA) {
return invalid(
`has manifest schema ${String(json["schema"])}, expected ${LIBNODE_PACK_SCHEMA}`,
);
}
for (const field of ["nodeVersion", "platform", "arch", "libnode", "bridge"]) {
if (typeof json[field] !== "string" || json[field] === "") {
return invalid(`has no ${field} in manifest.json`);
}
}
if (typeof json["moduleVersion"] !== "number" || typeof json["bridgeAbi"] !== "number") {
return invalid("has no moduleVersion or bridgeAbi in manifest.json");
}
const files = json["files"];
if (!isRecord(files)) return invalid("lists no files in manifest.json");
// Every name the pack lists is a file directly in the pack directory.
const license = json["license"];
if (license !== undefined && typeof license !== "string") {
return invalid("has a malformed license in manifest.json");
}
for (const name of [...Object.keys(files), json["license"] ?? "LICENSE"] as string[]) {
if (!packFileName(name)) {
return invalid(`names a file outside the pack (${JSON.stringify(name)})`);
}
}
for (const name of [json["libnode"], json["bridge"]] as string[]) {
if (!packFileName(name)) {
return invalid(`names a file outside the pack (${JSON.stringify(name)})`);
}
if (!isFileEntry(files[name])) return invalid(`lists no size and SHA-256 for ${name}`);
}
return json as unknown as LibnodePackManifest;
}
/** Reads and checks the pack in `root`: its manifest, its package when
* `packageName` is given, and that each library exists with the recorded
* size (and SHA-256 when `verifyHashes`). */
export function readLibnodePack(
root: string,
options: { packageName?: string | undefined; verifyHashes?: boolean | undefined } = {},
): LibnodePack {
let json: unknown;
try {
json = JSON.parse(readFileSync(join(root, "manifest.json"), "utf8"));
} catch {
throw new LibnodePackError(
`no libnode pack at ${root} (manifest.json is missing or unreadable)`,
"missing",
);
}
const manifest = parseLibnodePackManifest(json, root);
if (options.packageName !== undefined && manifest.package !== options.packageName) {
throw new LibnodePackError(
`the libnode pack at ${root} is ${manifest.package ?? "unnamed"}; this target needs ${options.packageName}`,
"invalid",
);
}
for (const name of [manifest.libnode, manifest.bridge]) {
const path = join(root, name);
const entry = manifest.files[name]!;
let size: number;
try {
size = statSync(path).size;
} catch {
throw new LibnodePackError(`the libnode pack at ${root} is missing ${name}`, "invalid");
}
const hashMismatch =
options.verifyHashes === true &&
createHash("sha256").update(readFileSync(path)).digest("hex") !== entry.sha256;
if (size !== entry.size || hashMismatch) {
throw new LibnodePackError(
`${name} in the libnode pack at ${root} does not match its manifest`,
"invalid",
);
}
}
return { root, manifest };
}
/** The nearest `node_modules/<name>` directory with a built pack (a
* manifest.json) walking up from `from`. */
function findInstalled(name: string, from: string): string | null {
let dir = resolve(from);
for (;;) {
const candidate = join(dir, "node_modules", name);
if (existsSync(join(candidate, "manifest.json"))) return candidate;
const parent = dirname(dir);
if (parent === dir) return null;
dir = parent;
}
}
function resolveNextToCompiler(name: string): string | null {
try {
const dir = dirname(createRequire(import.meta.url).resolve(`${name}/package.json`));
// An unbuilt workspace package is not a pack.
return existsSync(join(dir, "manifest.json")) ? dir : null;
} catch {
return null;
}
}
/** The installed pack directory for `targetName`, or null when none is
* installed or the target has no packs. */
export function findInstalledLibnodePack(options: {
targetName: string;
projectDir: string;
/** Test hook for the compiler-relative lookup. */
resolveInstalled?: ((name: string) => string | null) | undefined;
}): string | null {
const packageName = libnodePackPackage(options.targetName);
if (packageName === null) return null;
return (
findInstalled(packageName, options.projectDir) ??
(options.resolveInstalled ?? resolveNextToCompiler)(packageName)
);
}
/** Writes the license notices a program that redistributes Node.js must
* carry, `<output>.licenses.txt` (design §2.13), and returns its path. */
export function writeLibnodeNotices(pack: LibnodePack, outPath: string): string {
const { manifest } = pack;
const nodeLicense = readFileSync(join(pack.root, manifest.license ?? "LICENSE"), "utf8");
const origin =
manifest.package !== undefined && manifest.version !== undefined
? ` (${manifest.package}@${manifest.version})`
: "";
const path = `${outPath}.licenses.txt`;
writeFileSync(
path,
[
`${basename(outPath)} runs part of its code in Node.js ${manifest.nodeVersion}, loaded from ${manifest.libnode}${origin}.`,
`Node.js and the components it bundles are licensed as follows. The scriptc bridge,`,
`${manifest.bridge}, is licensed under Apache-2.0.`,
"",
nodeLicense.trimEnd(),
"",
].join("\n"),
);
return path;
}
packages/compiler/src/node-islands/libnode.test.ts
+48 -5
View File
@@ -11,7 +11,9 @@ import { tmpdir } from "node:os";
import { join } from "node:path";
import { describe, expect, test } from "vitest";
import { USES_TIMERS_LIB_FNS } from "../backend/llvm/lib-shared.js";
import { libnodeCacheKey, libnodeRecord, NODE_BRIDGE_ABI } from "./libnode.js";
import { libnodeCacheKey, libnodeRecord, libnodeTarget, NODE_BRIDGE_ABI } from "./libnode.js";
import { libnodePackPackage } from "./libnode-pack.js";
import { findInstalledLibnodePack, hostLibnodeTarget } from "./libnode-pack.js";
function pack(nodeVersion: string, bridgeAbi = NODE_BRIDGE_ABI): string {
const dir = mkdtempSync(join(tmpdir(), "scriptc-libnode-"));
@@ -39,7 +41,21 @@ describe("the recorded libnode pack", () => {
});
test("refuses a missing, too old or incompatible pack", () => {
expect(libnodeRecord(undefined, "/out/app", {})).toMatch(/no libnode pack is installed/);
// Unless a workspace build of the host's @scriptc/libnode-<platform>
// resolves next to the compiler, which libnodeRecord then uses.
const installed = findInstalledLibnodePack({
targetName: hostLibnodeTarget(),
projectDir: "/nonexistent",
});
if (installed === null) {
expect(libnodeRecord(undefined, "/out/app", {}, "/nonexistent")).toMatch(
/no libnode pack is installed/,
);
} else {
expect(libnodeRecord(undefined, "/out/app", {}, "/nonexistent")).toMatchObject({
libnode: installed,
});
}
expect(libnodeRecord("/nonexistent", "/out/app", {})).toBe(
"no libnode pack at /nonexistent (manifest.json is missing or unreadable)",
);
@@ -143,9 +159,36 @@ describe("the recorded libnode pack", () => {
});
});
describe("the libnode pack in an executable's cache key", () => {
test("is absent without --libnode or SCRIPTC_LIBNODE", () => {
expect(libnodeCacheKey(undefined, {})).toBeUndefined();
describe("the libnode pack identity a cached build keeps", () => {
// A workspace build of the target's @scriptc/libnode-<platform> that
// resolves next to the compiler (CI assembles one) is installed for every
// project, so without a pack in the project the identity names it.
const nextToCompiler = findInstalledLibnodePack({
targetName: libnodeTarget({}),
projectDir: "/nonexistent",
});
const withoutProjectPack = (key: string | undefined): void => {
if (nextToCompiler === null) expect(key).toBeUndefined();
else expect(key).toContain(`|${nextToCompiler}|`);
};
test("is absent when no pack is named or installed", () => {
withoutProjectPack(libnodeCacheKey(undefined, {}, "/nonexistent"));
});
test("names the installed pack, and changes with it", () => {
const project = mkdtempSync(join(tmpdir(), "scriptc-libnode-project-"));
const name = libnodePackPackage(libnodeTarget({}));
if (name === null) return;
const installed = join(project, "node_modules", name);
mkdirSync(installed, { recursive: true });
writeFileSync(join(installed, "manifest.json"), JSON.stringify({ nodeVersion: "24.22.0" }));
const before = libnodeCacheKey(undefined, {}, project);
expect(before).toContain(installed);
writeFileSync(join(installed, "manifest.json"), JSON.stringify({ nodeVersion: "24.23.0" }));
expect(libnodeCacheKey(undefined, {}, project)).not.toBe(before);
rmSync(installed, { recursive: true, force: true });
withoutProjectPack(libnodeCacheKey(undefined, {}, project));
});
test("changes when the pack in the same directory is upgraded", () => {
packages/compiler/src/node-islands/libnode.ts
+66 -13
View File
@@ -6,8 +6,8 @@
* SCRIPTC_LIBNODE overrides the recorded directory at run time.
*
* Where the build finds the pack: `--libnode=<pack directory>`, else
* SCRIPTC_LIBNODE in the build's environment (until NI-13 publishes
* @scriptc/libnode-<platform> packages that an installation carries).
* SCRIPTC_LIBNODE in the build's environment, else the installed
* @scriptc/libnode-<platform> pack (libnode-pack.ts).
* `--libnode=beside` takes that pack and copies it next to the executable
* as `<output>.libnode/`, recorded as `$ORIGIN/<output>.libnode`, so the
* executable and its Node travel together; a recorded `$ORIGIN/` path is
@@ -24,6 +24,16 @@ import {
rmSync,
} from "node:fs";
import { basename, isAbsolute, join, resolve, sep } from "node:path";
import { compilerReleaseVersion } from "../library/sidecar.js";
import { nativeCodegenTarget } from "../backend/targets.js";
import {
findInstalledLibnodePack,
hostLibnodeTarget,
LibnodePackError,
libnodePackPackage,
readLibnodePack,
type LibnodePack,
} from "./libnode-pack.js";
/** The oldest Node.js release Node islands support: synchronous
* module.registerHooks, which shared modules need, is a release candidate
@@ -31,7 +41,7 @@ import { basename, isAbsolute, join, resolve, sep } from "node:path";
export const NODE_ISLAND_MIN_NODE = "24.13.1";
/** The bridge ABI this compiler's runtime speaks (scr_node_bridge.h). */
export const NODE_BRIDGE_ABI = 3;
export const NODE_BRIDGE_ABI = 5;
/** The prefix of a recorded path relative to the executable's directory. */
export const ORIGIN = "$ORIGIN/";
@@ -131,18 +141,56 @@ function realpathOr(path: string): string {
}
}
/** The native target whose libnode pack a build records: the codegen
* target (SCRIPTC_TARGET, and the host's libc), else the host. */
export function libnodeTarget(env: NodeJS.ProcessEnv = process.env): string {
return nativeCodegenTarget(env)?.name ?? hostLibnodeTarget();
}
/** Why the installed pack in `dir` cannot serve this compiler, or null: it
* must be the target's package at the compiler's own version. */
function installedPackProblem(dir: string, packageName: string): string | null {
let pack: LibnodePack;
try {
// An installed pack is found, not named: check its files' digests too.
pack = readLibnodePack(dir, { packageName, verifyHashes: true });
} catch (error) {
return error instanceof LibnodePackError ? error.message : String(error);
}
const version = compilerReleaseVersion();
if (pack.manifest.version !== version)
return `the installed libnode pack ${packageName} at ${dir} is version ${pack.manifest.version ?? "unknown"}; this scriptc is ${version}: install ${packageName}@${version}`;
return null;
}
/** What to record for a program that calls into Node, or the reason the
* build cannot (an SC8005 message). `option` is --libnode. */
export function libnodeRecord(
option: string | undefined,
outPath: string,
env: NodeJS.ProcessEnv = process.env,
projectDir: string = process.cwd(),
): LibnodeRecord | string {
const fromEnv = env["SCRIPTC_LIBNODE"];
const beside = option === "beside";
const source = option !== undefined && !beside ? option : fromEnv;
if (source === undefined || source === "")
return "this program calls into Node.js (@scriptc/node), but no libnode pack is installed: set SCRIPTC_LIBNODE or pass --libnode=<pack directory>";
const explicit = option !== undefined && !beside ? option : fromEnv;
const target = libnodeTarget(env);
// Without an explicit pack, the installed @scriptc/libnode-<target>.
const installed =
explicit !== undefined && explicit !== ""
? null
: findInstalledLibnodePack({ targetName: target, projectDir });
const source = installed ?? explicit;
if (source === undefined || source === null || source === "") {
const pack = libnodePackPackage(target);
return pack === null
? `this program calls into Node.js (@scriptc/node), but there is no libnode pack for ${target}: set SCRIPTC_LIBNODE or pass --libnode=<pack directory>`
: `this program calls into Node.js (@scriptc/node), but no libnode pack is installed: install ${pack}, set SCRIPTC_LIBNODE or pass --libnode=<pack directory>`;
}
if (installed !== null) {
const problem = installedPackProblem(installed, libnodePackPackage(target)!);
if (problem !== null) return problem;
}
if (source.startsWith(ORIGIN)) {
// A pack the deployment places next to the executable: nothing to read.
return { libnode: source, nodeVersion: "" };
@@ -159,18 +207,23 @@ export function libnodeRecord(
/** The identity of the libnode pack a build that calls into Node.js
* records, which the program's executable cache entry keeps
* (EarlyExecutableCacheOptions.libnodeIdentity), or undefined when neither
* --libnode nor SCRIPTC_LIBNODE is set. It holds the option, the pack
* directory and a digest of the pack's manifest.json, so that a pack
* upgraded in place (another Node version, bridge ABI or file list)
* invalidates the entry instead of restoring its old record. */
* (EarlyExecutableCacheOptions.libnodeIdentity): the option, the pack
* directory (named by --libnode or SCRIPTC_LIBNODE, or the installed pack
* found from `projectDir`), and a digest of the pack's manifest.json. A pack
* upgraded in place, removed, or replaced by another installed pack changes
* it. Undefined when no pack is named or installed. */
export function libnodeCacheKey(
option: string | undefined,
env: NodeJS.ProcessEnv = process.env,
projectDir: string = process.cwd(),
): string | undefined {
const fromEnv = env["SCRIPTC_LIBNODE"];
if (option === undefined && fromEnv === undefined) return undefined;
const source = option !== undefined && option !== "beside" ? option : (fromEnv ?? "");
const explicit = option !== undefined && option !== "beside" ? option : fromEnv;
const source =
explicit !== undefined && explicit !== ""
? explicit
: findInstalledLibnodePack({ targetName: libnodeTarget(env), projectDir });
if (source === undefined || source === null) return option === undefined ? undefined : option;
let manifest = "none";
if (isAbsolute(source)) {
try {
packages/compiler/src/node-islands/modules.test.ts
+46
View File
@@ -30,6 +30,10 @@ describe("module specifiers in JavaScript", () => {
export * from "./f";
export { g } from "./g";
const h = await import("./h.js");
const require2 = createRequire(import.meta.url);
const j = require2("./j.cjs");
const k = require2.resolve("k");
const i = import.meta.resolve("i");
import.meta.url;
obj.require("x");
const t = \`\${require("./t")}\`;
@@ -44,6 +48,9 @@ describe("module specifiers in JavaScript", () => {
"./f",
"./g",
"./h.js",
"./j.cjs",
"k",
"i",
"./t",
]);
expect(scan.computed).toEqual([]);
@@ -67,10 +74,19 @@ describe("module specifiers in JavaScript", () => {
expect(forms("require(name)")).toEqual(["require() with a computed specifier"]);
expect(forms("require('./a' + b)")).toEqual(["require() with a computed specifier"]);
expect(forms("import(`./x/${y}`)")).toEqual(["import() with a computed specifier"]);
expect(forms("import.meta.resolve(name)")).toEqual([
"import.meta.resolve() with a computed specifier",
]);
expect(forms("const r = require; r('./a')")).toEqual(["require as a value"]);
expect(forms("createRequire(import.meta.url)('./a')")).toEqual([
"a require() from createRequire()",
]);
expect(forms("const req = createRequire(import.meta.url); req(name)")).toEqual([
"require() with a computed specifier",
]);
expect(forms("const req = createRequire(import.meta.url); use(req)")).toEqual([
"require as a value",
]);
expect(forms("function require(x) {} typeof require")).toEqual([]);
});
});
@@ -138,6 +154,36 @@ describe("the collected module tree", () => {
]);
});
test("packages a package loads without declaring them ship too", () => {
const root = project({
"main.ts": "",
"node_modules/bridge/package.json": JSON.stringify({
type: "module",
imports: { "#mapped": { node: "mapped", default: "./local.js" } },
}),
"node_modules/bridge/index.js":
'import { a } from "zoo";\nexport const b = () => import("zoo/extra.js");',
"node_modules/bridge/index.d.ts": 'import type { T } from "typesonly";',
"node_modules/zoo/package.json": "{}",
"node_modules/zoo/index.js": "exports.a = 1;",
"node_modules/zoo/extra.js": "",
"node_modules/typesonly/index.d.ts": "",
"node_modules/mapped/index.js": "",
});
const collector = new NodeIslandModuleCollector();
expect(collector.addLoaded(join(root, "node_modules/bridge/index.js"))).toBeNull();
// The root stays above node_modules, where Node looks for packages.
expect(collector.tree()!.files.map((f) => f.path)).toEqual([
"node_modules/bridge/index.d.ts",
"node_modules/bridge/index.js",
"node_modules/bridge/package.json",
"node_modules/mapped/index.js",
"node_modules/zoo/extra.js",
"node_modules/zoo/index.js",
"node_modules/zoo/package.json",
]);
});
test("pnpm's links between package directories stay links inside the tree", () => {
const root = project({
"app/main.ts": "",
packages/compiler/src/node-islands/modules.ts
+111 -17
View File
@@ -5,18 +5,25 @@
* Each `call`/`callSync` names its module with a string literal. The build
* resolves it from the calling module as Node's require() does:
* - a relative or absolute specifier names a module of the program: the
* module and every module its string-literal `require()`, `import` and
* `export ... from` reach are collected; a computed specifier in such a
* module is refused, since the build cannot know what it loads;
* module and every module its string-literal `require()` (also through a
* `createRequire()` function), `import` and `export ... from` reach are
* collected; a computed specifier in such a module is refused, since the
* build cannot know what it loads;
* - a package specifier names a package: its whole directory is collected
* (computed requires inside packages keep working), and so are its
* `dependencies`, `optionalDependencies` and `peerDependencies`, found as
* Node finds them;
* `dependencies`, `optionalDependencies` and `peerDependencies`, the
* packages its "imports" map to and the packages its JavaScript names
* with string literals, all found as Node finds them;
* - a Node built-in module needs nothing, and neither does a module the
* build cannot find: loading it throws MODULE_NOT_FOUND when the call
* runs, as under Node.
* A load of a resolved file (`requireModule` and the compiler's own Node
* islands, lower-node-island.ts) collects the package that holds the file
* the same way, or, for a module of the program, the module and what it
* loads; the load then names the file by its path in the tree.
* Paths keep their layout relative to one root directory (the deepest
* directory that holds every collected path), with symbolic links between
* directory that holds every collected path, above any node_modules
* directory, where Node looks for packages), with symbolic links between
* package directories (pnpm's layout) kept as links. Only these relative
* paths enter the executable. Every read goes through the frontend input
* tracker, so an edit to an island module invalidates cached builds. */
@@ -33,6 +40,7 @@ import {
trackedRealpath,
} from "../frontend/input-tracker.js";
import type { IrNodeIslandModules } from "../ir/ir.js";
import { packageDirOf } from "./origin.js";
export type NodeIslandModuleTree = IrNodeIslandModules;
type NodeIslandModuleFile = IrNodeIslandModules["files"][number];
@@ -46,6 +54,9 @@ const FILE_EXTENSIONS = [".js", ".json", ".node"];
/** Module files whose imports are traced. */
const TRACED = /\.(?:[cm]?js|[cm]?ts)$/;
/** A package's JavaScript files, whose imports of other packages are followed. */
const SCRIPT = /\.[cm]?js$/;
function isBuiltin(specifier: string): boolean {
return specifier.startsWith("node:") || BUILTINS.has(specifier);
}
@@ -138,6 +149,16 @@ function commonDirectory(paths: readonly string[]): string {
return joined === "" ? sep : joined;
}
/** The string targets of a package.json "imports" map, through its
* conditions and fallback arrays. */
function importTargets(value: unknown): string[] {
if (typeof value === "string") return [value];
if (Array.isArray(value)) return value.flatMap(importTargets);
if (value !== null && typeof value === "object")
return Object.values(value).flatMap(importTargets);
return [];
}
function portable(path: string): string {
return sep === "/" ? path : path.split(sep).join("/");
}
@@ -151,6 +172,8 @@ export class NodeIslandModuleCollector {
private readonly traced = new Set<string>();
/** Real paths of the calling modules. */
private readonly callers = new Set<string>();
/** Real paths of the files loads name (addLoaded). */
private readonly loaded = new Set<string>();
/** Collects the module `specifier` names from `caller` (a real path).
* Returns null, or the reason the build cannot ship it. */
@@ -159,6 +182,19 @@ export class NodeIslandModuleCollector {
return this.resolveFrom(caller, specifier);
}
/** Collects the module a load names by its resolved path (a real path):
* the package that holds it, or a module of the program with what it
* loads. Returns null, or the reason the build cannot ship it. */
addLoaded(file: string): string | null {
this.loaded.add(file);
const at = packageDirOf(file);
if (at !== null) {
this.addPackage(at.dir);
return null;
}
return this.addModule(this.placed(file));
}
/** The collected tree, or null when no call loads a file. */
tree(): NodeIslandModuleTree | null {
if (this.files.size === 0 && this.links.size === 0) return null;
@@ -167,8 +203,14 @@ export class NodeIslandModuleCollector {
...this.links.keys(),
...this.links.values(),
...[...this.callers].map((caller) => dirname(caller)),
...[...this.loaded].map((file) => dirname(file)),
];
const root = commonDirectory(anchors);
// Node finds a package through the node_modules directory that holds
// it, so a root inside one moves above it.
let root = commonDirectory(anchors);
const parts = root.split(sep);
const outer = parts.indexOf("node_modules");
if (outer >= 0) root = parts.slice(0, outer).join(sep) || sep;
const files: NodeIslandModuleFile[] = [];
for (const file of [...this.files].sort()) {
const bytes = trackedReadBytes(file);
@@ -193,6 +235,12 @@ export class NodeIslandModuleCollector {
return tree === null ? basename(caller) : portable(relative(tree.root, caller));
}
/** The name a load gives a collected file: `./` and its path relative to
* the tree root (the runtime resolves it under the root). */
static loadPath(tree: NodeIslandModuleTree | null, file: string): string {
return tree === null ? file : `./${portable(relative(tree.root, file))}`;
}
private resolveFrom(fromFile: string, specifier: string): string | null {
if (isBuiltin(specifier)) return null;
if (specifier.startsWith("#"))
@@ -270,7 +318,21 @@ export class NodeIslandModuleCollector {
const dir = this.placed(found);
if (this.packages.has(dir)) return;
this.packages.add(dir);
this.addDirectory(dir, new Set());
const scripts: string[] = [];
this.addDirectory(dir, new Set(), scripts);
// A package may load another it does not declare (one installed beside
// it): Node finds that one by walking up node_modules, and so does the
// build, from each string-literal specifier of the package's JavaScript.
for (const file of scripts) {
const text = trackedReadFile(file);
if (text === null) continue;
for (const specifier of scanModuleSpecifiers(text).specifiers) {
if (isBuiltin(specifier) || isPathSpecifier(specifier)) continue;
if (specifier.startsWith("#") || /^[a-z][a-z0-9+.-]*:/i.test(specifier)) continue;
const other = packageDirectory(dirname(file), packageName(specifier));
if (other !== null) this.addPackage(other);
}
}
const manifest = trackedReadFile(join(dir, "package.json"));
if (manifest === null) return;
let parsed: Record<string, unknown>;
@@ -285,6 +347,9 @@ export class NodeIslandModuleCollector {
if (deps !== null && typeof deps === "object")
for (const name of Object.keys(deps)) names.add(name);
}
// A package import ("imports") may map to another package.
for (const target of importTargets(parsed["imports"]))
if (!isPathSpecifier(target) && !isBuiltin(target)) names.add(packageName(target));
for (const name of [...names].sort()) {
// A dependency that is not installed (optional, or a peer the program
// does not use) is not loaded either.
@@ -294,17 +359,21 @@ export class NodeIslandModuleCollector {
}
/** Every file below `dir`, except nested node_modules (dependencies come
* through package.json) and .git. */
private addDirectory(dir: string, visited: Set<string>): void {
* through package.json and the package's imports) and .git; the
* JavaScript files among them also go to `scripts`. */
private addDirectory(dir: string, visited: Set<string>, scripts: string[]): void {
const real = trackedRealpath(dir) ?? dir;
if (visited.has(real)) return;
visited.add(real);
const entries = trackedAccessibleEntries(dir);
if (entries === null) return;
for (const file of entries.files) this.files.add(join(dir, file));
for (const file of entries.files) {
this.files.add(join(dir, file));
if (SCRIPT.test(file)) scripts.push(join(dir, file));
}
for (const sub of entries.directories) {
if (sub === "node_modules" || sub === ".git") continue;
this.addDirectory(join(dir, sub), visited);
this.addDirectory(join(dir, sub), visited, scripts);
}
}
}
@@ -490,8 +559,8 @@ export function moduleTokens(text: string): Token[] {
}
export interface ModuleSpecifierScan {
/** String-literal specifiers of require(), require.resolve(), import()
* and import/export declarations, in source order. */
/** String-literal specifiers of require(), require.resolve(), import(),
* import.meta.resolve() and import/export declarations, in source order. */
specifiers: string[];
/** Loads whose module the source does not name with a string literal. */
computed: { at: number; form: string }[];
@@ -513,11 +582,27 @@ export function scanModuleSpecifiers(text: string): ModuleSpecifierScan {
(is(k + 2, "punct", ")") || is(k + 2, "punct", ","))
? at(k + 1)!.value
: null;
// `const r = createRequire(...)` declares a require function of the
// module: calls of `r` load modules as require() calls do.
const aliases = new Set<string>();
const declarations = new Set<number>();
for (let k = 0; k + 4 < tokens.length; k++) {
if (
(is(k, "name", "const") || is(k, "name", "let") || is(k, "name", "var")) &&
is(k + 1, "name") &&
is(k + 2, "punct", "=") &&
is(k + 3, "name", "createRequire") &&
is(k + 4, "punct", "(")
) {
aliases.add(at(k + 1)!.value);
declarations.add(k + 1).add(k + 3);
}
}
for (let k = 0; k < tokens.length; k++) {
const t = tokens[k]!;
if (t.kind !== "name") continue;
if (t.kind !== "name" || declarations.has(k)) continue;
const member = is(k - 1, "punct", ".") || is(k - 1, "punct", "?.");
if (t.value === "require" && !member && !is(k + 1, "punct", ":")) {
if ((t.value === "require" || aliases.has(t.value)) && !member && !is(k + 1, "punct", ":")) {
if (is(k - 1, "name", "function") || is(k - 1, "name", "typeof")) continue;
let call = k + 1;
let form = "require()";
@@ -541,7 +626,16 @@ export function scanModuleSpecifiers(text: string): ModuleSpecifierScan {
continue;
}
if (t.value === "import" && !member) {
if (is(k + 1, "punct", ".")) continue; // import.meta
if (is(k + 1, "punct", ".")) {
// import.meta.resolve() names a module, as require.resolve() does.
if (is(k + 2, "name", "meta") && is(k + 3, "punct", ".") && is(k + 4, "name", "resolve")) {
const literal = literalCall(k + 5);
if (literal !== null) specifiers.push(literal);
else if (is(k + 5, "punct", "("))
computed.push({ at: t.at, form: "import.meta.resolve() with a computed specifier" });
}
continue;
}
if (is(k + 1, "punct", "(")) {
const literal = literalCall(k + 1);
if (literal === null)
packages/compiler/src/node-islands/resolve.ts
+16
View File
@@ -0,0 +1,16 @@
/** `require.resolve(spec)` from `fromFile` for island code (scope-B NI-4).
* The Node-hosted compiler installs the resolver (index.ts, Node's own
* createRequire); the self-hosted compiler builds no islands and never
* installs one. Throws when the specifier does not resolve. */
type IslandResolver = (fromFile: string, spec: string) => string;
let islandResolver: IslandResolver | null = null;
export function setIslandResolver(resolver: IslandResolver): void {
islandResolver = resolver;
}
export function resolveFrom(fromFile: string, spec: string): string {
if (islandResolver === null) throw new Error(`no module resolver for '${spec}'`);
return islandResolver(fromFile, spec);
}
packages/compiler/test/inline-array-fields.test.ts
+60 -16
View File
@@ -191,6 +191,36 @@ byPos.set(list.pos, list.items);
}
});
test("arrays fields hold do not reach unchecked code inside other values", async () => {
for (const [use, where] of [
// Inside an array passed through a function value.
[
`const cb = (xs: Item[][]): void => {
xs[0]!.push(new Item(9));
};
const f: (xs: Item[][]) => void = Math.random() < 2 ? cb : () => {};
f([new List(0, [new Item(1)]).items as Item[]]);`,
"a call through a function value",
],
// In a field of a class instance that becomes a dynamic value.
[
`class Box {
readonly arr: readonly Item[];
constructor(arr: readonly Item[]) {
this.arr = arr;
}
}
const u: unknown = new Box(new List(0, [new Item(1)]).items);
(u as { arr: Item[] }).arr.push(new Item(9));`,
"a dynamic value",
],
]) {
const result = await build(classes + use);
expect(result.ok, use).toBe(false);
expect(result.messages.join("\n"), use).toContain(`reaches ${where}`);
}
});
test("a write names the field read whose array it may reach", async () => {
const result = await build(
classes +
@@ -438,12 +468,7 @@ console.log(new List(2).items.length);
expect(computed.messages.join("\n")).toContain("constructor parameter");
});
test.runIf(lowRegion)(
"an instance inside a dynamic value shows the field as an array",
async () => {
const { ok, llvm } = await build(
classes +
`
const holder = `
class Holder {
readonly list: List;
constructor(list: List) {
@@ -451,17 +476,36 @@ class Holder {
}
}
const list = new List(3, [new Item(1), new Item(2)]);
console.log(JSON.stringify({ list }), JSON.stringify(new Holder(list)));
`,
`;
test("an instance inside a dynamic value is refused", async () => {
for (const use of [
"const u: unknown = new Holder(list);\n(u as { list: { items: Item[] } }).list.items.push(new Item(3));",
"const u: unknown = [{ list }];\nconsole.log(u);",
"const u: unknown = new Map([[1, new Holder(list)]]);\nconsole.log(u);",
]) {
const result = await build(classes + holder + use);
expect(result.ok, use).toBe(false);
expect(result.messages.join("\n"), use).toContain(
"an instance, or a value that can hold one, becomes a dynamic value",
);
expect(ok).toBe(true);
// The instance's dynamic view reads the field as an array of the elements.
const views = llvm
.split(/\n(?=define )/)
.filter((f) => f.startsWith("define") && f.includes("_materialize("));
expect(views.some((f) => f.includes("call ptr @scr_arr_view_cref("))).toBe(true);
},
);
expect(result.messages.join("\n"), use).toContain("(at ");
}
});
test.runIf(lowRegion)("printing and JSON.stringify show the field as an array", async () => {
const { ok, llvm } = await build(
classes +
holder +
"console.log(JSON.stringify({ list }), JSON.stringify(new Holder(list)));\nconsole.log(new Holder(list));",
);
expect(ok).toBe(true);
// The instance's dynamic view reads the field as an array of the elements.
const views = llvm
.split(/\n(?=define )/)
.filter((f) => f.startsWith("define") && f.includes("_materialize("));
expect(views.some((f) => f.includes("call ptr @scr_arr_view_cref("))).toBe(true);
});
test("the constructor stores the array parameter and does nothing else with it", async () => {
const result = await build(`
packages/compiler/test/node-index.test.ts
+27 -1
View File
@@ -3,7 +3,8 @@ import { tmpdir } from "node:os";
import { join } from "node:path";
import { beforeAll, expect, test } from "vitest";
import { compile, deserializeModule } from "../src/index.js";
import type { IrExpr, IrModule, IrStmt, IrType } from "../src/ir/ir.js";
import type { IrExpr, IrFunction, IrModule, IrStmt, IrType } from "../src/ir/ir.js";
import { emitRangeStarts } from "../src/backend/llvm/emit-module.js";
import { eachNodeOfKinds, IrNodeIndex, type IrNodeKind } from "../src/ir/node-index.js";
import { everyModuleNode, everyStmtList } from "../src/ir/traverse.js";
@@ -125,6 +126,31 @@ test("a false answer stops the visit, with or without the index", () => {
}
});
test("a function's node count is the number of nodes its visit notes", () => {
const all = index.scanExcept(NO_KINDS);
for (const fn of mod.functions) expect(index.nodeCount(fn)).toBe(noted(fn, all).length);
expect(index.nodeCount({ ...mod.functions[0]! })).toBeUndefined();
});
test("emission ranges cover the functions in order, none empty, of about equal cost", () => {
const count = (fn: IrFunction): number | undefined => index.nodeCount(fn);
for (const ranges of [1, 2, 3, mod.functions.length, mod.functions.length + 4]) {
const starts = emitRangeStarts(mod.functions, ranges, count);
expect(starts[0]).toBe(0);
expect(starts.at(-1)).toBe(mod.functions.length);
expect(starts.length - 1).toBeLessThanOrEqual(ranges);
for (let i = 1; i < starts.length; i++) expect(starts[i]!).toBeGreaterThan(starts[i - 1]!);
}
// Equal costs split into equal ranges. A range ends after the function
// that reaches its share of the cost, so a large function ends its range.
const functions = Array.from({ length: 12 }, () => mod.functions[0]!);
expect(emitRangeStarts(functions, 4, () => 0)).toEqual([0, 3, 6, 9, 12]);
const sizes = [0, 0, 1000, 0, 0, 0];
const uneven = sizes.map((_, i) => ({ ...mod.functions[0]!, name: `f${i}` }));
const bySize = new Map(uneven.map((fn, i) => [fn, sizes[i]!]));
expect(emitRangeStarts(uneven, 3, (fn) => bySize.get(fn))).toEqual([0, 3, 4, 6]);
});
test("a function the index did not record is walked", () => {
const all = index.scanExcept(NO_KINDS);
for (const fn of mod.functions) {
packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6804-node-island-signals/main.mjs.json
+6
View File
@@ -0,0 +1,6 @@
{
"order": [
"<repo>/tests/corpus/6804-node-island-signals/main.mjs"
],
"diags": []
}
packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6805-node-island-exit-code/main.ts.json
+6
View File
@@ -0,0 +1,6 @@
{
"order": [
"<repo>/tests/corpus/6805-node-island-exit-code/main.ts"
],
"diags": []
}
packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6806-node-island-exit-in-call/main.ts.json
+6
View File
@@ -0,0 +1,6 @@
{
"order": [
"<repo>/tests/corpus/6806-node-island-exit-in-call/main.ts"
],
"diags": []
}
packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6807-node-island-exit-in-island-timer/main.ts.json
+6
View File
@@ -0,0 +1,6 @@
{
"order": [
"<repo>/tests/corpus/6807-node-island-exit-in-island-timer/main.ts"
],
"diags": []
}
packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6808-node-island-exit-from-program/main.ts.json
+6
View File
@@ -0,0 +1,6 @@
{
"order": [
"<repo>/tests/corpus/6808-node-island-exit-from-program/main.ts"
],
"diags": []
}
packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6809-node-island-stdio/main.ts.json
+6
View File
@@ -0,0 +1,6 @@
{
"order": [
"<repo>/tests/corpus/6809-node-island-stdio/main.ts"
],
"diags": []
}
packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6810-node-island-children/main.ts.json
+6
View File
@@ -0,0 +1,6 @@
{
"order": [
"<repo>/tests/corpus/6810-node-island-children/main.ts"
],
"diags": []
}
packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6811-node-island-uncaught-listener/main.mjs.json
+6
View File
@@ -0,0 +1,6 @@
{
"order": [
"<repo>/tests/corpus/6811-node-island-uncaught-listener/main.mjs"
],
"diags": []
}
packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6812-node-island-handles/main.mjs.json
+6
View File
@@ -0,0 +1,6 @@
{
"order": [
"<repo>/tests/corpus/6812-node-island-handles/main.mjs"
],
"diags": []
}
packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6813-node-island-handle-errors/main.mjs.json
+6
View File
@@ -0,0 +1,6 @@
{
"order": [
"<repo>/tests/corpus/6813-node-island-handle-errors/main.mjs"
],
"diags": []
}
packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6814-node-island-handle-bytes/main.mjs.json
+6
View File
@@ -0,0 +1,6 @@
{
"order": [
"<repo>/tests/corpus/6814-node-island-handle-bytes/main.mjs"
],
"diags": []
}
packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6815-node-island-handle-promises/main.mjs.json
+6
View File
@@ -0,0 +1,6 @@
{
"order": [
"<repo>/tests/corpus/6815-node-island-handle-promises/main.mjs"
],
"diags": []
}
packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6816-node-island-objects/main.ts.json
+6
View File
@@ -0,0 +1,6 @@
{
"order": [
"<repo>/tests/corpus/6816-node-island-objects/main.ts"
],
"diags": []
}
packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6817-node-island-esm/main.ts.json
+6
View File
@@ -0,0 +1,6 @@
{
"order": [
"<repo>/tests/corpus/6817-node-island-esm/main.ts"
],
"diags": []
}
packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6822-node-island-gc/main.ts.json
+6
View File
@@ -0,0 +1,6 @@
{
"order": [
"<repo>/tests/corpus/6822-node-island-gc/main.ts"
],
"diags": []
}
packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6823-node-island-nested-stack/main.ts.json
+6
View File
@@ -0,0 +1,6 @@
{
"order": [
"<repo>/tests/corpus/6823-node-island-nested-stack/main.ts"
],
"diags": []
}
packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6839-node-island-path-mapped/main.ts.json
+18
View File
@@ -0,0 +1,18 @@
{
"order": [
"<repo>/tests/corpus/6839-node-island-path-mapped/main.ts"
],
"diags": [
{
"code": "SC1010",
"message": "the 'pathkit' import (it resolves only to type declarations — there is no runtime implementation to compile) is not supported yet",
"loc": {
"file": "<repo>/tests/corpus/6839-node-island-path-mapped/main.ts",
"start": 234,
"end": 294
},
"milestone": "M4",
"hint": "relative imports (./file, ../dir/file), package.json-mediated project imports (#alias via the imports field, self-name references via exports), statically compiled npm packages, and the built-in fs, fs/promises, path, os, url, crypto, zlib, child_process, net, http, tls, https, http2, dgram, dns, dns/promises, util, util/types, string_decoder, querystring, readline, events, stream, stream/promises, stream/consumers, stream/web, buffer, assert, assert/strict, worker_threads, cluster, tty, async_hooks, timers, timers/promises, diagnostics_channel, perf_hooks, module, v8, and console modules (bare or node:-prefixed) and node:test (node:-prefixed only, like in Node) are supported"
}
]
}
packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6890-node-island-timer-refresh/main.ts.json
+6
View File
@@ -0,0 +1,6 @@
{
"order": [
"<repo>/tests/corpus/6890-node-island-timer-refresh/main.ts"
],
"diags": []
}
packages/compiler/test/ts7/baselines/order-parity/tests/corpus/generic-conditional-member-bindings.ts.json
+6
View File
@@ -0,0 +1,6 @@
{
"order": [
"<repo>/tests/corpus/generic-conditional-member-bindings.ts"
],
"diags": []
}
packages/compiler/test/ts7/baselines/order-parity/tests/corpus/generic-conditional-member-units.ts.json
+6
View File
@@ -0,0 +1,6 @@
{
"order": [
"<repo>/tests/corpus/generic-conditional-member-units.ts"
],
"diags": []
}
packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/unloaded-dynamic-import/main.ts.json
Vendored
+6
View File
@@ -0,0 +1,6 @@
{
"order": [
"<repo>/tests/fixtures/npm/cases/unloaded-dynamic-import/main.ts"
],
"diags": []
}
packages/experimental/README.md
+2 -2
View File
@@ -144,8 +144,8 @@ function visit(list: NodeList): void {
- The constructor of the class that declares the field writes it once, from one of its parameters. A subclass constructor passes one of its own parameters there through `super(...)`. Nothing else writes the field.
- Indexing the field, reading its `length` and iterating it with `for...of` read the elements from the instance, directly or through a `const` local used only in those ways. Any other use, such as passing the field to a function or calling one of its methods, gets an array of the elements, which is frozen in scriptc builds.
- The array passed to the constructor is one that nothing else keeps: a new array (a literal, a copy, or the result of a function that returns only such arrays or its own argument), a local array at its last use, a parameter whose callers each pass such an array, an array that an InlineArray field already holds, or a `const` array that is never written.
- The arrays fields hold are never written, through a read of the field (a cast) or through any other reference. scriptc follows them through locals, parameters, return values, fields, the elements of arrays, Maps and Sets, and closures, and reports one that reaches a write, a dynamic value, a call through a function value, or a library call other than printing and `JSON.stringify` without a replacer.
- The arrays fields hold are never written, through a read of the field (a cast) or through any other reference. scriptc follows them through locals, parameters, return values, fields, the elements of arrays, Maps and Sets, and closures, and reports one that reaches a write, a dynamic value (also inside the fields of an object that becomes one), a call through a function value (also inside an array, Map or Set passed there), or a library call other than printing and `JSON.stringify` without a replacer.
- Two reads of the field need not give the same array, so the arrays fields hold are never compared by identity: with `===` or `!==`, as an `indexOf` or `includes` argument, or as a Map key or Set element. Compare the elements instead, or keep a copy made with `slice()` where identity matters.
- Instances of the class are not constructed through a class value, and do not become dynamic values (for example through `JSON.stringify` or a conversion to `unknown`).
- Instances of the class are not constructed through a class value, and do not become dynamic values (for example through a conversion to `unknown`), directly or inside another value that can hold one: an instance of a class with such a field, a record, array, Map or Set, or a value typed as a base class. A dynamic view reads the field as a new array each time, so a class whose instances can become dynamic values cannot hold an InlineArray field. Printing, `JSON.stringify` without a replacer, `publish` and a worker's `workerData` only read or copy the value and are allowed.
scriptc reports a field that does not keep these rules (SC7009). Under Node.js the field holds the array the constructor stored. In builds with sanitizers, in programs that use `LifetimeRegion`, and on platforms other than 64-bit Linux, the elements stay in an ordinary array object.
packages/libnode-common/bridge/scr_node_bridge.cc
+992 -45
View File
File diff suppressed because it is too large Load Diff
packages/libnode-common/build-pack.mjs
+108 -40
View File
@@ -1,8 +1,7 @@
#!/usr/bin/env node
/* Assembles a libnode pack: the directory a scriptc program with Node
* islands loads Node from (scope-B design §2.16.3). Until the pack CI
* matrix (NI-13) publishes @scriptc/libnode-<platform> packages, packs are
* assembled by hand from a Node `--shared` build and the matching headers:
* islands loads Node from (scope-B design §2.16.3), from a Node `--shared`
* build and the headers of the same release:
*
* node packages/libnode-common/build-pack.mjs \
* --libnode <path to libnode.so.<ABI> or libnode.<ABI>.dylib> \
@@ -10,21 +9,32 @@
* --out <pack directory> [--license <Node's LICENSE>] [--cxx <compiler>]
*
* The pack holds libnode, the bridge (bridge/scr_node_bridge.cc) compiled
* against those headers, Node's license and manifest.json, which the build
* of a program with Node islands reads (packages/compiler/src/node-islands/
* libnode.ts). Programs find a pack through SCRIPTC_LIBNODE or --libnode. */
* against those headers and linked to libnode through $ORIGIN /
* @loader_path (so the directory can move), Node's license and
* manifest.json, which the build of a program with Node islands reads
* (packages/compiler/src/node-islands/libnode.ts). Programs find a pack
* through --libnode, SCRIPTC_LIBNODE, or the installed
* @scriptc/libnode-<platform> package.
*
* The @scriptc/libnode-<platform> packs call buildPack() from
* scripts/build.mjs after building libnode from the pinned release; run by
* hand, this is the bridge-only rebuild path (seconds) and the way a
* self-builder pairs the bridge with a compatible Node build of their own. */
import { execFileSync } from "node:child_process";
import { copyFileSync, mkdirSync, readFileSync, statSync, writeFileSync } from "node:fs";
import { createHash } from "node:crypto";
import { basename, join, resolve } from "node:path";
import { basename, dirname, join, resolve } from "node:path";
import { fileURLToPath } from "node:url";
import { parseArgs } from "node:util";
const here = fileURLToPath(new URL(".", import.meta.url));
const runtimeSrc = join(here, "../runtime/src");
/** The manifest schema; the compiler's pack reader accepts exactly this. */
export const LIBNODE_PACK_SCHEMA = "scriptc.libnode-pack.v1";
/** The bridge ABI the runtime expects (scr_node_bridge.h). */
function bridgeAbi() {
export function bridgeAbi() {
const header = readFileSync(join(runtimeSrc, "scr_node_bridge.h"), "utf8");
return Number(/#define SCR_NODE_BRIDGE_ABI (\d+)u/.exec(header)[1]);
}
@@ -46,38 +56,41 @@ function fileEntry(path) {
};
}
if (process.argv[1] === fileURLToPath(import.meta.url)) {
const { values } = parseArgs({
options: {
libnode: { type: "string" },
headers: { type: "string" },
out: { type: "string" },
license: { type: "string" },
cxx: { type: "string", default: process.platform === "darwin" ? "clang++" : "g++" },
},
});
if (!values.libnode || !values.headers || !values.out) {
console.error("usage: build-pack.mjs --libnode <lib> --headers <include/node> --out <dir>");
process.exit(2);
}
const out = resolve(values.out);
const headers = resolve(values.headers);
mkdirSync(out, { recursive: true });
const libnodeName = basename(values.libnode);
copyFileSync(values.libnode, join(out, libnodeName));
const bridgeName =
process.platform === "darwin" ? "libscr_node_bridge.dylib" : "libscr_node_bridge.so";
const link =
process.platform === "darwin"
? ["-dynamiclib", join(out, libnodeName), "-Wl,-rpath,@loader_path"]
: ["-shared", `-L${out}`, `-l:${libnodeName}`, "-Wl,-rpath,$ORIGIN"];
/** The bridge library name on a platform. */
export function bridgeName(os = process.platform) {
return os === "darwin" ? "libscr_node_bridge.dylib" : "libscr_node_bridge.so";
}
/** Compiles the bridge against `headers` (an include/node directory) and
* links it to `libnode`, which must already sit in the bridge's directory. */
export function buildBridge({ headers, libnode, out, cxx }) {
const darwin = process.platform === "darwin";
const compiler = cxx ?? process.env.CXX ?? (darwin ? "clang++" : "g++");
const link = darwin
? [
"-dynamiclib",
libnode,
"-Wl,-rpath,@loader_path",
`-Wl,-install_name,@rpath/${basename(out)}`,
]
: [
"-shared",
`-L${dirname(libnode)}`,
`-l:${basename(libnode)}`,
"-Wl,-rpath,$ORIGIN",
"-Wl,-z,defs",
"-Wl,--as-needed",
"-ldl",
"-pthread",
];
execFileSync(
values.cxx,
compiler,
[
"-std=c++20",
"-O2",
"-fPIC",
"-fvisibility=hidden",
"-fno-rtti",
"-I",
runtimeSrc,
"-I",
@@ -85,24 +98,79 @@ if (process.argv[1] === fileURLToPath(import.meta.url)) {
join(here, "bridge/scr_node_bridge.cc"),
...link,
"-o",
join(out, bridgeName),
out,
],
{ stdio: "inherit" },
);
if (values.license) copyFileSync(values.license, join(out, "LICENSE"));
if (darwin) {
execFileSync("strip", ["-x", out], { stdio: "inherit" });
// Stripping invalidated the linker's ad-hoc signature, which arm64
// macOS requires to map the library.
execFileSync("codesign", ["--force", "--sign", "-", out], { stdio: "inherit" });
} else {
execFileSync("strip", ["--strip-unneeded", out], { stdio: "inherit" });
}
}
/**
* Assembles the pack in `out` and returns its manifest. `extra` adds fields
* to manifest.json (the published packs record their package, target and
* build inputs); schema, nodeVersion, moduleVersion, bridgeAbi, platform,
* arch, libnode, bridge and files are always present.
*
* @param {{ libnode: string, headers: string, out: string, license?: string,
* cxx?: string, extra?: Record<string, unknown> }} options
*/
export function buildPack(options) {
const out = resolve(options.out);
const headers = resolve(options.headers);
mkdirSync(out, { recursive: true });
const libnodeName = basename(options.libnode);
const libnode = join(out, libnodeName);
if (resolve(options.libnode) !== libnode) copyFileSync(options.libnode, libnode);
const bridge = join(out, bridgeName());
buildBridge({ headers, libnode, out: bridge, cxx: options.cxx });
if (options.license) copyFileSync(options.license, join(out, "LICENSE"));
const { nodeVersion, moduleVersion } = nodeVersionOf(headers);
const manifest = {
schema: "scriptc.libnode-pack.v1",
schema: LIBNODE_PACK_SCHEMA,
nodeVersion,
moduleVersion,
bridgeAbi: bridgeAbi(),
platform: process.platform,
arch: process.arch,
libnode: libnodeName,
bridge: bridgeName(),
...options.extra,
files: {
[libnodeName]: fileEntry(join(out, libnodeName)),
[bridgeName]: fileEntry(join(out, bridgeName)),
[libnodeName]: fileEntry(libnode),
[bridgeName()]: fileEntry(bridge),
},
};
writeFileSync(join(out, "manifest.json"), `${JSON.stringify(manifest, null, 2)}\n`);
console.log(`libnode pack for Node ${nodeVersion} in ${out}`);
return manifest;
}
if (process.argv[1] === fileURLToPath(import.meta.url)) {
const { values } = parseArgs({
options: {
libnode: { type: "string" },
headers: { type: "string" },
out: { type: "string" },
license: { type: "string" },
cxx: { type: "string" },
},
});
if (!values.libnode || !values.headers || !values.out) {
console.error("usage: build-pack.mjs --libnode <lib> --headers <include/node> --out <dir>");
process.exit(2);
}
const manifest = buildPack({
libnode: values.libnode,
headers: values.headers,
out: values.out,
license: values.license,
cxx: values.cxx,
});
console.log(`libnode pack for Node ${manifest.nodeVersion} in ${resolve(values.out)}`);
}
packages/libnode-common/fetch-dev-pack.mjs
+3 -2
View File
@@ -2,8 +2,9 @@
/* Assembles a development libnode pack from the pinned, checksum-verified
* inputs in dev-pack.json: a prebuilt libnode of the repository's Node
* release and Node's official headers, with the bridge built against them
* by build-pack.mjs. CI's Linux test lanes use it (.github/actions/
* libnode-dev-pack) until the NI-13 packs are published.
* by build-pack.mjs. CI's Linux x64 and macOS test lanes fall back to it
* (.github/actions/libnode) until the libnode-packs workflow has cached a
* build of @scriptc/libnode-<platform> for their ref.
*
* node packages/libnode-common/fetch-dev-pack.mjs --out <pack directory> */
import { execFileSync } from "node:child_process";
packages/libnode-common/libnode.json
+7
View File
@@ -0,0 +1,7 @@
{
"nodeVersion": "24.22.0",
"tarballSha256": "41f97cff2c666f1f8293dc5f85c65e8dd605c16b43fd9febd49ede32ee1e0329",
"moduleVersion": 137,
"configureFlags": ["--shared", "--ninja"],
"icu": "full"
}
packages/libnode-common/package.json
+6
View File
@@ -0,0 +1,6 @@
{
"name": "@scriptc/libnode-common",
"version": "0.0.1",
"private": true,
"type": "module"
}
packages/libnode-common/scripts/build.mjs
+336
View File
@@ -0,0 +1,336 @@
#!/usr/bin/env node
// Builds one libnode pack into its package directory:
//
// 1. libnode: the pinned Node.js release (libnode.json) from its official
// source tarball, SHA-256 verified, configured `--shared` with full ICU,
// only the `libnode` target built, stripped, staged with its headers
// and LICENSE in a cache directory keyed by libnodeCacheKey(). A cache
// hit skips the Node build entirely (about 26 minutes on 8 cores).
// 2. the bridge, compiled against those headers (seconds, every run);
// 3. the pack: libnode, the bridge, Node's LICENSE and manifest.json in
// packages/libnode-<platform>/, plus `--compress` copies (.zst) of both
// libraries in <cache>/compressed/ for size reports and embedding.
//
// This never runs in `pnpm -r build` or the test suites: each platform
// package runs it through its explicit `build:native` script, and the
// libnode-packs workflow runs that on matching hosts.
//
// Environment:
// SCRIPTC_LIBNODE_CACHE cache root (default node_modules/.cache/scriptc-libnode)
// SCRIPTC_LIBNODE_SOURCE an extracted source tree of the pinned release to
// build in (seeds the cache from an earlier build tree)
// SCRIPTC_LIBNODE_TOOLCHAIN a name for the toolchain, part of the cache key
// (CI: "manylinux_2_28-gcc14", "macos-15-clang")
// SCRIPTC_LIBNODE_JOBS ninja parallelism (default: available cores)
// CC, CXX the compilers for Node and the bridge
import { execFileSync } from "node:child_process";
import { createHash } from "node:crypto";
import {
copyFileSync,
createReadStream,
existsSync,
mkdirSync,
readFileSync,
renameSync,
rmSync,
statSync,
writeFileSync,
} from "node:fs";
import { availableParallelism } from "node:os";
import { join, resolve } from "node:path";
import { fileURLToPath } from "node:url";
import { parseArgs } from "node:util";
import { buildPack } from "../build-pack.mjs";
import {
bridgeAbi,
bridgeHash,
libnodeCacheKey,
packFileNames,
platformInfo,
readPin,
repoRoot,
} from "./pin.mjs";
function log(message) {
process.stdout.write(`libnode: ${message}\n`);
}
function run(command, args, options = {}) {
log(`$ ${command} ${args.join(" ")}`);
execFileSync(command, args, { stdio: "inherit", ...options });
}
async function sha256File(path) {
const hash = createHash("sha256");
for await (const chunk of createReadStream(path)) hash.update(chunk);
return hash.digest("hex");
}
/** Downloads and verifies the pinned source tarball, then extracts it. */
async function fetchSource(cacheRoot, pin) {
const name = `node-v${pin.nodeVersion}`;
const tree = join(cacheRoot, "src", name);
if (existsSync(join(tree, "configure.py"))) return tree;
const tarball = join(cacheRoot, "src", `${name}.tar.xz`);
mkdirSync(join(cacheRoot, "src"), { recursive: true });
if (!existsSync(tarball) || (await sha256File(tarball)) !== pin.tarballSha256) {
const url = `https://nodejs.org/dist/v${pin.nodeVersion}/${name}.tar.xz`;
run("curl", ["-fsSL", "--retry", "3", "-o", `${tarball}.part`, url]);
renameSync(`${tarball}.part`, tarball);
}
const actual = await sha256File(tarball);
if (actual !== pin.tarballSha256) {
rmSync(tarball, { force: true });
throw new Error(`${name}.tar.xz has SHA-256 ${actual}, expected ${pin.tarballSha256}`);
}
rmSync(tree, { recursive: true, force: true });
run("tar", ["-xJf", tarball, "-C", join(cacheRoot, "src")]);
return tree;
}
/** The version a source tree's node_version.h declares. */
function sourceVersion(tree) {
const text = readFileSync(join(tree, "src", "node_version.h"), "utf8");
const define = (name) => new RegExp(`#define ${name} (\\d+)`).exec(text)?.[1];
return `${define("NODE_MAJOR_VERSION")}.${define("NODE_MINOR_VERSION")}.${define("NODE_PATCH_VERSION")}`;
}
function moduleVersionOf(headers) {
const text = readFileSync(join(headers, "include", "node", "node_version.h"), "utf8");
return Number(/#define NODE_MODULE_VERSION (\d+)/.exec(text)?.[1]);
}
/** The highest GLIBC_x.y symbol version a Linux library needs. */
function minimumGlibc(library) {
const dynamic = execFileSync("objdump", ["-T", library], {
encoding: "utf8",
maxBuffer: 256 * 1024 * 1024,
});
const versions = [...new Set(dynamic.match(/GLIBC_\d+(\.\d+)+/g) ?? [])].map((v) =>
v.slice("GLIBC_".length),
);
versions.sort((a, b) => {
const x = a.split(".").map(Number);
const y = b.split(".").map(Number);
for (let i = 0; i < Math.max(x.length, y.length); i++) {
if ((x[i] ?? 0) !== (y[i] ?? 0)) return (x[i] ?? 0) - (y[i] ?? 0);
}
return 0;
});
return versions.at(-1) ?? null;
}
/** Configures and builds libnode in `tree`, then stages it into `stage`. */
function buildLibnode(tree, stage, platform, pin) {
const { os } = platformInfo(platform);
if (sourceVersion(tree) !== pin.nodeVersion) {
throw new Error(`${tree} is Node ${sourceVersion(tree)}, the pin is ${pin.nodeVersion}`);
}
const python = process.env.PYTHON ?? "python3";
// An earlier tree configured with exactly these flags is reused as is, so
// a seeded build tree is a ninja no-op instead of a reconfigure.
const status = join(tree, "config.status");
const configured =
existsSync(status) &&
existsSync(join(tree, "out", "Release", "build.ninja")) &&
readFileSync(status, "utf8")
.trimEnd()
.endsWith(`configure ${pin.configureFlags.join(" ")}`);
const started = Date.now();
if (!configured) run("./configure", pin.configureFlags, { cwd: tree });
const gypi = readFileSync(join(tree, "config.gypi"), "utf8");
for (const [setting, expected] of [
["node_shared", "true"],
["node_use_node_snapshot", "true"],
["icu_small", pin.icu === "small" ? "true" : "false"],
]) {
if (!gypi.includes(`"${setting}": "${expected}"`)) {
throw new Error(`config.gypi does not have ${setting}: ${expected}`);
}
}
const jobs = process.env.SCRIPTC_LIBNODE_JOBS ?? String(availableParallelism());
run("ninja", ["-C", join(tree, "out", "Release"), "-j", jobs, "libnode"]);
const buildSeconds = Math.round((Date.now() - started) / 1000);
const names = packFileNames(platform, pin.moduleVersion);
const built = [
join(tree, "out", "Release", "lib", names.libnode),
join(tree, "out", "Release", names.libnode),
].find((path) => existsSync(path));
if (built === undefined) throw new Error(`the Node build did not produce ${names.libnode}`);
const partial = `${stage}.partial`;
rmSync(partial, { recursive: true, force: true });
mkdirSync(join(partial, "lib"), { recursive: true });
const library = join(partial, "lib", names.libnode);
if (os === "darwin") {
copyFileSync(built, library);
run("install_name_tool", ["-id", `@rpath/${names.libnode}`, library]);
// -x keeps the global symbols addons and the bridge bind to.
run("strip", ["-x", library]);
// Editing the library invalidated the linker's ad-hoc signature, which
// arm64 macOS requires to map it.
run("codesign", ["--force", "--sign", "-", library]);
} else {
run("strip", ["--strip-unneeded", "-o", library, built]);
}
run(
python,
[
join(tree, "tools", "install.py"),
"install",
"--headers-only",
"--silent",
"--root-dir",
tree,
"--dest-dir",
join(partial, "headers"),
"--prefix",
"/",
],
{ cwd: tree },
);
const moduleVersion = moduleVersionOf(join(partial, "headers"));
if (moduleVersion !== pin.moduleVersion) {
throw new Error(
`Node ${pin.nodeVersion} has NODE_MODULE_VERSION ${moduleVersion}, the pin says ${pin.moduleVersion}`,
);
}
copyFileSync(join(tree, "LICENSE"), join(partial, "LICENSE"));
writeFileSync(
join(partial, "build.json"),
`${JSON.stringify(
{
nodeVersion: pin.nodeVersion,
moduleVersion,
configureFlags: pin.configureFlags,
icu: pin.icu,
toolchain: process.env.SCRIPTC_LIBNODE_TOOLCHAIN ?? "default",
buildSeconds,
builtBytes: statSync(built).size,
minimumGlibc: os === "linux" ? minimumGlibc(library) : null,
},
null,
2,
)}\n`,
);
rmSync(stage, { recursive: true, force: true });
renameSync(partial, stage);
}
/** zstd copies of the pack's libraries, for size reports and embedding. */
function compress(pack, names, out) {
mkdirSync(out, { recursive: true });
const sizes = {};
for (const name of [names.libnode, names.bridge]) {
const target = join(out, `${name}.zst`);
const started = Date.now();
run("zstd", ["-19", "-T0", "-q", "-f", join(pack, name), "-o", target]);
sizes[name] = { zstdBytes: statSync(target).size, seconds: (Date.now() - started) / 1000 };
}
return sizes;
}
/**
* @param {{ platform: string, packageRoot: string, compress?: boolean, skipLibnode?: boolean }} options
*/
export async function buildLibnodePack(options) {
const { platform, packageRoot } = options;
const { os, arch } = platformInfo(platform);
const pin = readPin();
const cacheRoot = resolve(
process.env.SCRIPTC_LIBNODE_CACHE ??
join(repoRoot, "node_modules", ".cache", "scriptc-libnode"),
);
const key = libnodeCacheKey(platform);
const stage = join(cacheRoot, `${platform}-${pin.nodeVersion}-${key.slice(0, 16)}`);
const names = packFileNames(platform, pin.moduleVersion);
if (existsSync(join(stage, "build.json"))) {
log(`cached libnode ${pin.nodeVersion} for ${platform} in ${stage}`);
} else {
if (options.skipLibnode) throw new Error(`no cached libnode build in ${stage}`);
const tree = process.env.SCRIPTC_LIBNODE_SOURCE
? resolve(process.env.SCRIPTC_LIBNODE_SOURCE)
: await fetchSource(cacheRoot, pin);
buildLibnode(tree, stage, platform, pin);
}
const build = JSON.parse(readFileSync(join(stage, "build.json"), "utf8"));
for (const name of [names.libnode, names.bridge, "LICENSE", "manifest.json"]) {
rmSync(join(packageRoot, name), { force: true });
}
const packageJson = JSON.parse(readFileSync(join(packageRoot, "package.json"), "utf8"));
const bridgeStarted = Date.now();
const manifest = buildPack({
libnode: join(stage, "lib", names.libnode),
headers: join(stage, "headers", "include", "node"),
out: packageRoot,
license: join(stage, "LICENSE"),
extra: {
package: packageJson.name,
version: packageJson.version,
target: platform,
license: "LICENSE",
build: {
libnodeKey: key,
bridgeHash: bridgeHash(),
configureFlags: build.configureFlags,
icu: build.icu,
toolchain: build.toolchain,
minimumGlibc: build.minimumGlibc,
},
},
});
const bridgeSeconds = (Date.now() - bridgeStarted) / 1000;
if (
manifest.nodeVersion !== pin.nodeVersion ||
manifest.moduleVersion !== pin.moduleVersion ||
manifest.bridgeAbi !== bridgeAbi() ||
manifest.platform !== os ||
manifest.arch !== arch ||
manifest.libnode !== names.libnode ||
manifest.bridge !== names.bridge
) {
throw new Error(`the assembled pack does not match ${platform}: ${JSON.stringify(manifest)}`);
}
const report = {
platform,
nodeVersion: pin.nodeVersion,
libnodeBuildSeconds: build.buildSeconds,
libnodeBuiltBytes: build.builtBytes,
libnodeStrippedBytes: manifest.files[names.libnode].size,
bridgeBytes: manifest.files[names.bridge].size,
bridgeSeconds,
minimumGlibc: manifest.build.minimumGlibc,
};
if (options.compress) {
report.compressed = compress(packageRoot, names, join(cacheRoot, "compressed", platform));
}
log(`pack ${packageJson.name} for Node ${pin.nodeVersion}: ${JSON.stringify(report)}`);
return report;
}
if (process.argv[1] === fileURLToPath(import.meta.url)) {
const { values } = parseArgs({
options: {
platform: { type: "string" },
"package-root": { type: "string" },
compress: { type: "boolean", default: false },
"skip-libnode": { type: "boolean", default: false },
report: { type: "string" },
},
});
if (!values.platform) {
console.error("usage: build.mjs --platform <platform> [--package-root <dir>] [--compress]");
process.exit(2);
}
const packageRoot = resolve(
values["package-root"] ?? join(repoRoot, "packages", `libnode-${values.platform}`),
);
const report = await buildLibnodePack({
platform: values.platform,
packageRoot,
compress: values.compress,
skipLibnode: values["skip-libnode"],
});
if (values.report) writeFileSync(values.report, `${JSON.stringify(report, null, 2)}\n`);
}
packages/libnode-common/scripts/cache-key.mjs
+29
View File
@@ -0,0 +1,29 @@
#!/usr/bin/env node
// Prints the CI cache identity of one platform's libnode build as
// GitHub Actions outputs: `key` (Node release, configure flags, platform,
// toolchain), `path` (the staged build directory under the cache root,
// relative to it) and `bridge` (the bridge source hash, which is not part
// of the key: a bridge change reuses the cached libnode).
//
// node packages/libnode-common/scripts/cache-key.mjs --platform <p> [--toolchain <t>]
import { parseArgs } from "node:util";
import { bridgeHash, libnodeCacheKey, readPin } from "./pin.mjs";
const { values } = parseArgs({
options: { platform: { type: "string" }, toolchain: { type: "string" } },
});
if (!values.platform) {
console.error("usage: cache-key.mjs --platform <platform> [--toolchain <name>]");
process.exit(2);
}
const pin = readPin();
const key = libnodeCacheKey(values.platform, values.toolchain);
process.stdout.write(
[
`key=libnode-${values.platform}-${pin.nodeVersion}-${key}`,
`path=${values.platform}-${pin.nodeVersion}-${key.slice(0, 16)}`,
`bridge=${bridgeHash()}`,
`node=${pin.nodeVersion}`,
"",
].join("\n"),
);
packages/libnode-common/scripts/pin.mjs
+90
View File
@@ -0,0 +1,90 @@
// The libnode pin and the platform table shared by the libnode pack build,
// its CI cache keys, the pack verifier, and the smoke test.
//
// A libnode pack (`@scriptc/libnode-<platform>`) holds a stripped Node.js
// shared library built from the pinned release, the C++ bridge compiled
// against that release's headers, Node's license, and manifest.json. The
// scriptc runtime dlopens the bridge on its first call into Node and talks
// to it only through the C ABI in packages/runtime/src/scr_node_bridge.h.
import { createHash } from "node:crypto";
import { readFileSync } from "node:fs";
import { dirname, join } from "node:path";
import { fileURLToPath } from "node:url";
export const commonRoot = dirname(dirname(fileURLToPath(import.meta.url)));
export const repoRoot = dirname(dirname(commonRoot));
export const bridgeSource = join(commonRoot, "bridge", "scr_node_bridge.cc");
export const bridgeHeader = join(repoRoot, "packages", "runtime", "src", "scr_node_bridge.h");
export { LIBNODE_PACK_SCHEMA, bridgeAbi } from "../build-pack.mjs";
/** Bump to invalidate every cached libnode build (a change to how the
* library is built or staged that the pin does not capture). */
const BUILD_RECIPE = 1;
/**
* @typedef {{
* nodeVersion: string,
* tarballSha256: string,
* moduleVersion: number,
* configureFlags: string[],
* icu: "full" | "small",
* }} LibnodePin
*/
/** @returns {LibnodePin} */
export function readPin() {
return JSON.parse(readFileSync(join(commonRoot, "libnode.json"), "utf8"));
}
/** Pack platforms: the npm package suffix, the Node `os`/`arch`, and the
* file names inside the pack. The darwin packs build only on macOS hosts. */
export const LIBNODE_PLATFORMS = {
"linux-x64-gnu": { os: "linux", arch: "x64", target: "linux-x64-gnu" },
"linux-arm64-gnu": { os: "linux", arch: "arm64", target: "linux-arm64-gnu" },
"darwin-arm64": { os: "darwin", arch: "arm64", target: "macos-arm64" },
"darwin-x64": { os: "darwin", arch: "x64", target: "macos-x64" },
};
/** @param {string} platform */
export function platformInfo(platform) {
const info = LIBNODE_PLATFORMS[/** @type {keyof typeof LIBNODE_PLATFORMS} */ (platform)];
if (info === undefined) throw new Error(`unknown libnode pack platform ${platform}`);
return info;
}
/** The library names inside a pack for one platform and module version. */
export function packFileNames(platform, moduleVersion) {
const { os } = platformInfo(platform);
return os === "darwin"
? { libnode: `libnode.${moduleVersion}.dylib`, bridge: "libscr_node_bridge.dylib" }
: { libnode: `libnode.so.${moduleVersion}`, bridge: "libscr_node_bridge.so" };
}
function sha256(text) {
return createHash("sha256").update(text).digest("hex");
}
/** The cache key of the libnode build itself: the Node release, how it is
* configured, the platform and toolchain. The bridge is not part of it; a
* bridge change rebuilds only the bridge against the cached headers. */
export function libnodeCacheKey(platform, toolchain = process.env.SCRIPTC_LIBNODE_TOOLCHAIN) {
const pin = readPin();
platformInfo(platform);
return sha256(
JSON.stringify({
recipe: BUILD_RECIPE,
nodeVersion: pin.nodeVersion,
tarballSha256: pin.tarballSha256,
configureFlags: pin.configureFlags,
icu: pin.icu,
platform,
toolchain: toolchain ?? "default",
}),
);
}
/** The hash of everything the bridge is compiled from. */
export function bridgeHash() {
return sha256(readFileSync(bridgeSource, "utf8") + "\0" + readFileSync(bridgeHeader, "utf8"));
}
packages/libnode-common/scripts/smoke.mjs
+108
View File
@@ -0,0 +1,108 @@
#!/usr/bin/env node
// Runs the libnode pack smoke test (smoke/smoke.c): compiles the C program
// with the host's C compiler, runs it against a pack directory and compares
// its stdout with the expected transcript, which pins the Node version,
// the built-in startup snapshot, full ICU, and synchronous, asynchronous
// and throwing calls through the bridge's C ABI.
//
// node packages/libnode-common/scripts/smoke.mjs [--pack <dir>] [--platform <p>]
//
// The pack defaults to SCRIPTC_LIBNODE, else packages/libnode-<platform>.
import { execFileSync, spawnSync } from "node:child_process";
import { createHash } from "node:crypto";
import { mkdtempSync, readFileSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { dirname, join, resolve } from "node:path";
import { fileURLToPath } from "node:url";
import { parseArgs } from "node:util";
import { bridgeAbi, bridgeHeader, commonRoot, readPin, repoRoot } from "./pin.mjs";
/** The host's pack platform name. */
export function hostPlatform() {
if (process.platform === "darwin") return `darwin-${process.arch}`;
return `${process.platform}-${process.arch}-gnu`;
}
/** The transcript smoke.c prints for a correct pack of the pinned Node. */
export function expectedTranscript(pin = readPin(), abi = bridgeAbi()) {
const icuSmall = pin.icu === "small";
return [
`bridge ABI ${abi}, Node ${pin.nodeVersion}, NODE_MODULE_VERSION ${pin.moduleVersion}`,
"libnode mapped before dlopen: no",
"1 + 2 => R3",
`process.versions.node => R"${pin.nodeVersion}"`,
"process.config.variables.node_use_node_snapshot => Rtrue",
`process.config.variables.icu_small => R${icuSmall}`,
`new Intl.NumberFormat('de-DE').format(1234567.891) => R"${icuSmall ? "1,234,567.891" : "1.234.567,891"}"`,
`new Intl.DateTimeFormat('ja-JP', { timeZone: 'UTC', dateStyle: 'full' }).format(0) => R"${icuSmall ? "Thursday, January 1, 1970" : "1970年1月1日木曜日"}"`,
`require('node:crypto').createHash('sha256').update('scriptc').digest('hex') => R"${sha256Hex("scriptc")}"`,
`require('node:zlib').gunzipSync(require('node:zlib').gzipSync('round trip')).toString() => R"round trip"`,
`[typeof require('node:fs').readFileSync, require('node:path').join('a', 'b')] => R["function","a/b"]`,
`null.property => T{"error":{"name":"TypeError","message":"Cannot read properties of null (reading 'property')"}}`,
`await new Promise((resolve) => setTimeout(() => resolve('timer fired'), 5)) => R"timer fired"`,
"await require('node:fs/promises').readFile(process.execPath).then((b) => b.length > 0) => Rtrue",
`await Promise.reject(new RangeError('rejected')) => T{"error":{"name":"RangeError","message":"rejected"}}`,
"ok",
"",
].join("\n");
}
function sha256Hex(text) {
return createHash("sha256").update(text).digest("hex");
}
/** Compiles and runs the smoke test; returns its stdout and stderr. */
export function runSmoke(pack, cc = process.env.CC ?? "cc") {
const work = mkdtempSync(join(tmpdir(), "scriptc-libnode-smoke-"));
try {
const binary = join(work, "smoke");
execFileSync(
cc,
[
"-std=c11",
...(process.platform === "linux" ? ["-D_POSIX_C_SOURCE=200809L", "-D_DEFAULT_SOURCE"] : []),
"-O2",
"-Wall",
"-Werror",
"-I",
dirname(bridgeHeader),
join(commonRoot, "smoke", "smoke.c"),
...(process.platform === "linux" ? ["-ldl"] : []),
"-o",
binary,
],
{ stdio: "inherit" },
);
const result = spawnSync(binary, [pack], { encoding: "utf8", env: { ...process.env } });
return { status: result.status, stdout: result.stdout, stderr: result.stderr };
} finally {
rmSync(work, { recursive: true, force: true });
}
}
if (process.argv[1] === fileURLToPath(import.meta.url)) {
const { values } = parseArgs({
options: { pack: { type: "string" }, platform: { type: "string" } },
});
// An explicit --pack or --platform names the pack; SCRIPTC_LIBNODE only
// replaces the host's default.
const pack = resolve(
values.pack ??
(values.platform !== undefined
? join(repoRoot, "packages", `libnode-${values.platform}`)
: (process.env.SCRIPTC_LIBNODE ?? join(repoRoot, "packages", `libnode-${hostPlatform()}`))),
);
const manifest = JSON.parse(readFileSync(join(pack, "manifest.json"), "utf8"));
const { status, stdout, stderr } = runSmoke(pack);
process.stderr.write(stderr);
const expected = expectedTranscript();
if (status !== 0 || stdout !== expected) {
process.stderr.write(
`smoke test failed (exit ${status}) for ${pack}\n--- expected\n${expected}--- actual\n${stdout}`,
);
process.exit(1);
}
process.stdout.write(
`libnode smoke test passed: ${manifest.package} (Node ${manifest.nodeVersion})\n`,
);
}
packages/libnode-common/scripts/verify.mjs
+69
View File
@@ -0,0 +1,69 @@
#!/usr/bin/env node
// Verifies a built libnode pack before it is packed or published: the
// manifest matches the package, the pin and the bridge ABI; every library
// it lists exists with the recorded size and SHA-256; the license files are
// present.
//
// node packages/libnode-common/scripts/verify.mjs <package directory>
import { createHash } from "node:crypto";
import { existsSync, readFileSync, statSync } from "node:fs";
import { join, resolve } from "node:path";
import { fileURLToPath } from "node:url";
import { LIBNODE_PACK_SCHEMA, bridgeAbi, packFileNames, platformInfo, readPin } from "./pin.mjs";
/** The problems with the pack in `root`; empty when it is complete. */
export function verifyPack(root) {
const problems = [];
const packageJson = JSON.parse(readFileSync(join(root, "package.json"), "utf8"));
const manifestPath = join(root, "manifest.json");
if (!existsSync(manifestPath)) return [`${manifestPath} is missing; run build:native first`];
const manifest = JSON.parse(readFileSync(manifestPath, "utf8"));
const pin = readPin();
const platform = String(packageJson.name).replace(/^@scriptc\/libnode-/, "");
const expect = (field, actual, expected) => {
if (actual !== expected) problems.push(`manifest ${field} is ${actual}, expected ${expected}`);
};
expect("schema", manifest.schema, LIBNODE_PACK_SCHEMA);
expect("package", manifest.package, packageJson.name);
expect("version", manifest.version, packageJson.version);
expect("target", manifest.target, platform);
expect("platform", manifest.platform, platformInfo(platform).os);
expect("arch", manifest.arch, platformInfo(platform).arch);
expect("nodeVersion", manifest.nodeVersion, pin.nodeVersion);
expect("moduleVersion", manifest.moduleVersion, pin.moduleVersion);
expect("bridgeAbi", manifest.bridgeAbi, bridgeAbi());
const names = packFileNames(platform, pin.moduleVersion);
expect("libnode", manifest.libnode, names.libnode);
expect("bridge", manifest.bridge, names.bridge);
for (const name of [names.libnode, names.bridge]) {
const entry = manifest.files?.[name];
const path = join(root, name);
if (entry === undefined) problems.push(`manifest lists no ${name}`);
else if (!existsSync(path)) problems.push(`${name} is missing`);
else if (statSync(path).size !== entry.size) problems.push(`${name} has the wrong size`);
else if (createHash("sha256").update(readFileSync(path)).digest("hex") !== entry.sha256) {
problems.push(`${name} has the wrong SHA-256`);
}
if (!packageJson.files.includes(name)) problems.push(`package.json files omits ${name}`);
}
for (const name of ["LICENSE", "SCRIPTC_LICENSE", "THIRD_PARTY_NOTICES"]) {
if (!existsSync(join(root, name))) problems.push(`${name} is missing`);
}
if (existsSync(join(root, "LICENSE"))) {
const license = readFileSync(join(root, "LICENSE"), "utf8");
if (!license.startsWith("Node.js is licensed for use as follows:")) {
problems.push("LICENSE is not the Node.js LICENSE file");
}
}
return problems;
}
if (process.argv[1] === fileURLToPath(import.meta.url)) {
const root = resolve(process.argv[2] ?? ".");
const problems = verifyPack(root);
if (problems.length > 0) {
for (const problem of problems) console.error(`libnode pack ${root}: ${problem}`);
process.exit(1);
}
console.log(`libnode pack ${root} verified`);
}
packages/libnode-common/smoke/smoke.c
+205
View File
@@ -0,0 +1,205 @@
/* Smoke test for a libnode pack, written the way the scriptc runtime loads
* one: a plain C program that never includes a Node or V8 header, dlopens
* the pack's bridge, checks its ABI and Node version, starts Node (one
* environment from the built-in startup snapshot), evaluates expressions
* synchronously and asynchronously through the bridge's C ABI, runs Node's
* event loop until the asynchronous ones settle, and tears Node down.
*
* smoke <pack directory> (or SCRIPTC_LIBNODE=<pack directory> smoke)
*
* stdout is deterministic and compared by scripts/smoke.mjs; timings and
* memory go to stderr. */
#include "scr_node_bridge.h"
#include <dlfcn.h>
#include <poll.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/resource.h>
#include <time.h>
#if defined(__APPLE__)
#define BRIDGE_NAME "libscr_node_bridge.dylib"
#else
#define BRIDGE_NAME "libscr_node_bridge.so"
#endif
/* The bootstrap runs as the environment's main script. It hands the bridge
* a dispatcher that evaluates `name` as an expression and answers the
* bridge's outcome strings: "R<json>" when it returns, "T<json>" when it
* throws; an asynchronous call settles through the host. The smoke test
* schedules no runtime jobs and listens to no process events, so the rest
* of the dispatcher the bridge requires does nothing. */
static const char kBootstrap[] =
"const host = globalThis.__scriptcNodeIslandHost;\n"
"globalThis.require = require;\n"
"const outcome = (ok, value) => (ok ? 'R' : 'T') + JSON.stringify(\n"
" ok ? value : { error: { name: value?.name, message: String(value?.message ?? value) } });\n"
"globalThis.__scriptcNodeIsland = {\n"
" call(id, sync, file, specifier, name, args) {\n"
" let value;\n"
" try { value = (0, eval)(name); } catch (error) { return outcome(false, error); }\n"
" if (sync) return outcome(true, value);\n"
" Promise.resolve(value).then((v) => host.settle(id, outcome(true, v)),\n"
" (e) => host.settle(id, outcome(false, e)));\n"
" },\n"
" setTimer() { return 0; }, clearTimer() {}, refTimer() {}, refreshTimer() {},\n"
" setImmediate() { return 0; }, clearImmediate() {}, refImmediate() {},\n"
" nextTick() {}, emit() {}, load() {},\n"
"};\n";
static double now_ms(void) {
struct timespec ts;
clock_gettime(CLOCK_MONOTONIC, &ts);
return ts.tv_sec * 1e3 + ts.tv_nsec / 1e6;
}
static long max_rss_kb(void) {
struct rusage usage;
getrusage(RUSAGE_SELF, &usage);
#if defined(__APPLE__)
return usage.ru_maxrss / 1024;
#else
return usage.ru_maxrss;
#endif
}
/* Whether a library whose path contains `needle` is mapped (Linux only). */
static int mapped(const char *needle) {
FILE *maps = fopen("/proc/self/maps", "r");
if (maps == NULL) return -1;
char line[4096];
int found = 0;
while (!found && fgets(line, sizeof line, maps) != NULL) found = strstr(line, needle) != NULL;
fclose(maps);
return found;
}
/* The asynchronous expressions, by call id, and their outcomes. */
static const char *const kAsync[] = {
"new Promise((resolve) => setTimeout(() => resolve('timer fired'), 5))",
"require('node:fs/promises').readFile(process.execPath).then((b) => b.length > 0)",
"Promise.reject(new RangeError('rejected'))",
};
#define ASYNC_COUNT (sizeof kAsync / sizeof kAsync[0])
static char *settled[ASYNC_COUNT];
static void host_settle(uint32_t id, const char *outcome, size_t len) {
if (id >= ASYNC_COUNT || settled[id] != NULL) return;
settled[id] = malloc(len + 1);
memcpy(settled[id], outcome, len);
settled[id][len] = '\0';
}
static void host_exit(int code) {
fflush(stdout);
exit(code);
}
/* The smoke test never calls host_loop, so Node's loop runs alone: no
* runtime jobs, I/O or listeners, and V8 keeps the thread's stack limit. */
static void host_run(uint32_t kind, uint32_t token) { (void)kind, (void)token; }
static void host_io_state(ScrNodeIoState *state) {
state->nfds = 0;
state->alive = 0;
state->ready = 0;
state->cap_ms = -1;
}
static void host_io_dispatch(void) {}
static void host_microtasks_done(void) {}
static void host_listen(uint32_t event, const char *name, uint32_t listener, int add, int once) {
(void)event, (void)name, (void)listener, (void)add, (void)once;
}
static uintptr_t host_stack_limit(void) { return 0; }
/* The smoke test runs on its own thread's stack. */
static uintptr_t host_stack_start(void) { return 0; }
static const ScrNodeBridgeHost kHost = {
host_settle, host_exit, host_run, host_io_state,
host_io_dispatch, host_microtasks_done, host_listen, host_stack_limit,
host_stack_start,
};
static const ScrNodeBridge *bridge;
static void eval_sync(const char *expression) {
size_t len = 0;
char *outcome = bridge->call(0, 1, "", "", expression, "[]", 2, &len);
printf("%s => %s\n", expression, outcome);
bridge->release(outcome);
}
/* Starts every asynchronous expression, then runs Node's event loop until
* nothing keeps it alive, as the runtime does at the end of a program. */
static void eval_async_all(void) {
for (uint32_t id = 0; id < ASYNC_COUNT; id++) {
size_t len = 0;
char *immediate = bridge->call(id, 0, "", "", kAsync[id], "[]", 2, &len);
if (immediate != NULL) bridge->release(immediate);
bridge->checkpoint(0);
}
bridge->run_loop(0);
for (uint32_t id = 0; id < ASYNC_COUNT; id++) {
printf("await %s => %s\n", kAsync[id], settled[id] != NULL ? settled[id] : "(never settled)");
free(settled[id]);
settled[id] = NULL;
}
}
int main(int argc, char **argv) {
const char *dir = argc > 1 ? argv[1] : getenv("SCRIPTC_LIBNODE");
if (dir == NULL || dir[0] == '\0') {
fprintf(stderr, "usage: smoke <pack directory> (or set SCRIPTC_LIBNODE)\n");
return 2;
}
char path[4096];
snprintf(path, sizeof path, "%s/%s", dir, BRIDGE_NAME);
int before = mapped("/libnode.");
double t0 = now_ms();
void *library = dlopen(path, RTLD_NOW | RTLD_GLOBAL);
if (library == NULL) {
fprintf(stderr, "smoke: %s\n", dlerror());
return 1;
}
double t1 = now_ms();
const ScrNodeBridge *(*entry)(void) =
(const ScrNodeBridge *(*)(void))dlsym(library, SCR_NODE_BRIDGE_ENTRY);
if (entry == NULL) {
fprintf(stderr, "smoke: %s has no %s\n", path, SCR_NODE_BRIDGE_ENTRY);
return 1;
}
bridge = entry();
if (bridge->abi != SCR_NODE_BRIDGE_ABI) {
fprintf(stderr, "smoke: bridge ABI %u, expected %u\n", bridge->abi, SCR_NODE_BRIDGE_ABI);
return 1;
}
printf("bridge ABI %u, Node %s, NODE_MODULE_VERSION %u\n", bridge->abi, bridge->node_version,
bridge->module_version);
printf("libnode mapped before dlopen: %s\n", before == 1 ? "yes" : "no");
char *error = bridge->start(&kHost, kBootstrap, argv[0], 0);
if (error != NULL) {
fprintf(stderr, "smoke: %s\n", error);
return 1;
}
double t2 = now_ms();
eval_sync("1 + 2");
double t3 = now_ms();
eval_sync("process.versions.node");
eval_sync("process.config.variables.node_use_node_snapshot");
eval_sync("process.config.variables.icu_small");
eval_sync("new Intl.NumberFormat('de-DE').format(1234567.891)");
eval_sync("new Intl.DateTimeFormat('ja-JP', { timeZone: 'UTC', dateStyle: 'full' }).format(0)");
eval_sync("require('node:crypto').createHash('sha256').update('scriptc').digest('hex')");
eval_sync("require('node:zlib').gunzipSync(require('node:zlib').gzipSync('round trip')).toString()");
eval_sync("[typeof require('node:fs').readFileSync, require('node:path').join('a', 'b')]");
eval_sync("null.property");
eval_async_all();
fprintf(stderr,
"smoke: dlopen %.1f ms, start %.1f ms, first result %.2f ms, dlopen to first result "
"%.1f ms, max RSS %ld KB\n",
t1 - t0, t2 - t1, t3 - t2, t3 - t0, max_rss_kb());
bridge->stop();
printf("ok\n");
return 0;
}
packages/libnode-common/test/bridge-smoke.c
+29 -9
View File
@@ -2,10 +2,10 @@
* check its ABI, start Node from the snapshot with the runtime's embedded
* bootstrap) and makes calls through the bridge: synchronous results and
* errors, and an asynchronous call settled by Node's own loop, which this
* host drives like the runtime's loop slot. bridge.test.ts compares the
* transcript. Usage: bridge-smoke <pack dir> <module path> */
* host runs the way the runtime's hosted loop does (host_loop, a
* checkpoint, run_loop) with no runtime work of its own. bridge.test.ts
* compares the transcript. Usage: bridge-smoke <pack dir> <module path> */
#include <dlfcn.h>
#include <poll.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
@@ -32,7 +32,29 @@ static void on_exit_code(int code) {
_Exit(code);
}
static const ScrNodeBridgeHost host = {settle, on_exit_code};
static void run(uint32_t kind, uint32_t token) {
printf("run %u %u\n", kind, token);
}
static void io_state(ScrNodeIoState *state) {
state->nfds = 0;
state->alive = 0;
state->ready = 0;
state->cap_ms = -1;
}
static void io_dispatch(void) {}
static void microtasks_done(void) {}
static void listen(uint32_t event, const char *name, uint32_t listener, int add, int once) {
printf("listen %u %s %u %d %d\n", event, name, listener, add, once);
}
static uintptr_t stack_limit(void) { return 0; }
static uintptr_t stack_start(void) { return 0; }
static const ScrNodeBridgeHost host = {settle, on_exit_code, run, io_state, io_dispatch,
microtasks_done, listen, stack_limit, stack_start};
static void call(const ScrNodeBridge *b, uint32_t id, int sync, const char *module,
const char *name, const char *args) {
@@ -70,11 +92,9 @@ int main(int argc, char **argv) {
call(b, 0, 1, file, "fails", "[]");
call(b, 0, 1, file, "missing", "[]");
call(b, 7, 0, file, "later", "[5]");
while (settled == 0 || b->alive()) {
struct pollfd fd = {b->backend_fd(), POLLIN, 0};
(void)poll(&fd, 1, b->backend_timeout());
b->pump();
}
b->host_loop();
b->checkpoint(0);
b->run_loop(0);
call(b, 0, 1, file, "quit", "[3]");
return 0;
}
packages/libnode-common/test/bridge.test.ts
+2 -1
View File
@@ -3,6 +3,7 @@
* calls through the bridge with the runtime's embedded bootstrap. Needs a
* pack in SCRIPTC_LIBNODE; required in the Linux x64 and macOS CI lanes
* (tests/harness/libnode.ts). */
import { bridgeAbi } from "../build-pack.mjs";
import { execFile } from "node:child_process";
import { mkdirSync, writeFileSync } from "node:fs";
import { join } from "node:path";
@@ -45,7 +46,7 @@ exports.quit = (code) => process.exit(code);
expect(result.stderr).toBe("");
expect(result.stdout).toBe(
[
"abi 3 ok 1",
`abi ${bridgeAbi()} ok 1`,
"sync add -> R5",
'sync fails -> T{"error":{"kind":2,"message":"out of range","code":"ERR_X"}}',
`sync missing -> T{"error":{"kind":1,"message":"'missing' exported by './smoke.cjs' is not a function"}}`,
packages/libnode-common/test/pin.test.ts
+90
View File
@@ -0,0 +1,90 @@
import { readFileSync, readdirSync } from "node:fs";
import { join } from "node:path";
import { describe, expect, test } from "vitest";
import {
LIBNODE_PLATFORMS,
bridgeAbi,
bridgeHash,
libnodeCacheKey,
packFileNames,
readPin,
repoRoot,
} from "../scripts/pin.mjs";
import { expectedTranscript } from "../scripts/smoke.mjs";
const readJson = (path: string) => JSON.parse(readFileSync(join(repoRoot, path), "utf8"));
describe("the libnode pin", () => {
test("is the repository's pinned Node release", () => {
const pin = readPin();
expect(pin.nodeVersion).toBe(readJson("internal/compatibility/node-v24.json").version);
expect(pin.nodeVersion).toBe(readFileSync(join(repoRoot, ".node-version"), "utf8").trim());
expect(pin.tarballSha256).toMatch(/^[0-9a-f]{64}$/);
});
test("builds the shared library with full ICU (design §8 question 2)", () => {
const pin = readPin();
expect(pin.configureFlags).toContain("--shared");
expect(pin.configureFlags.some((flag: string) => flag.startsWith("--with-intl"))).toBe(false);
expect(pin.icu).toBe("full");
});
});
describe("the platform packages", () => {
const dirs = readdirSync(join(repoRoot, "packages")).filter(
(name) => name.startsWith("libnode-") && name !== "libnode-common",
);
test("match the platform table", () => {
expect(dirs.map((dir) => dir.slice("libnode-".length)).sort()).toEqual(
Object.keys(LIBNODE_PLATFORMS).sort(),
);
});
test.each(Object.entries(LIBNODE_PLATFORMS))("%s ships its libraries", (platform, info) => {
const manifest = readJson(`packages/libnode-${platform}/package.json`);
const names = packFileNames(platform, readPin().moduleVersion);
expect(manifest.name).toBe(`@scriptc/libnode-${platform}`);
expect(manifest.version).toBe(readJson("packages/cli/package.json").version);
expect(manifest.os).toEqual([info.os]);
expect(manifest.cpu).toEqual([info.arch]);
expect(manifest.files).toEqual([
names.libnode,
names.bridge,
"manifest.json",
"LICENSE",
"SCRIPTC_LICENSE",
"THIRD_PARTY_NOTICES",
]);
// Never built by `pnpm -r build`.
expect(manifest.scripts.build).toBe("true");
expect(manifest.scripts["build:native"]).toBe("node scripts/build.mjs");
});
});
describe("cache identity", () => {
test("the libnode key depends on platform and toolchain, not on the bridge", () => {
const a = libnodeCacheKey("linux-x64-gnu", "manylinux_2_28-gcc-toolset-14");
expect(a).toMatch(/^[0-9a-f]{64}$/);
expect(libnodeCacheKey("linux-x64-gnu", "manylinux_2_28-gcc-toolset-14")).toBe(a);
expect(libnodeCacheKey("linux-arm64-gnu", "manylinux_2_28-gcc-toolset-14")).not.toBe(a);
expect(libnodeCacheKey("linux-x64-gnu", "other")).not.toBe(a);
expect(bridgeHash()).toMatch(/^[0-9a-f]{64}$/);
});
test("unknown platforms are refused", () => {
expect(() => libnodeCacheKey("linux-x64-musl", "x")).toThrow(
"unknown libnode pack platform linux-x64-musl",
);
});
});
test("the smoke transcript pins the Node version, the snapshot and full ICU", () => {
const lines = expectedTranscript().split("\n");
expect(lines[0]).toBe(
`bridge ABI ${bridgeAbi()}, Node ${readPin().nodeVersion}, NODE_MODULE_VERSION ${readPin().moduleVersion}`,
);
expect(lines).toContain("process.config.variables.node_use_node_snapshot => Rtrue");
expect(lines).toContain(`new Intl.NumberFormat('de-DE').format(1234567.891) => R"1.234.567,891"`);
expect(lines.at(-2)).toBe("ok");
});
packages/libnode-common/test/smoke.test.ts
+27
View File
@@ -0,0 +1,27 @@
// Loads a real libnode pack from a C program (smoke/smoke.c) the way the
// scriptc runtime does. Packs are not built by the ordinary build, so this
// runs only when SCRIPTC_LIBNODE names a pack directory (a CI artifact of
// the libnode-packs workflow, or a local build:native output). CI's
// fallback development pack (dev-pack.json) is not a package: it has no
// package.json and may carry a different Node patch release, so it is
// exercised by the Node island tests instead.
import { existsSync } from "node:fs";
import { join } from "node:path";
import { describe, expect, test } from "vitest";
import { verifyPack } from "../scripts/verify.mjs";
import { expectedTranscript, runSmoke } from "../scripts/smoke.mjs";
const pack = process.env["SCRIPTC_LIBNODE"];
const isPackage = pack !== undefined && pack !== "" && existsSync(join(pack, "package.json"));
describe.skipIf(!isPackage)("libnode pack smoke test", () => {
test("the pack is complete", () => {
expect(verifyPack(pack!)).toEqual([]);
});
test("a C program starts Node from the pack and evaluates expressions", () => {
const { status, stdout } = runSmoke(pack!);
expect(stdout).toBe(expectedTranscript());
expect(status).toBe(0);
}, 60_000);
});
packages/libnode-darwin-arm64/SCRIPTC_LICENSE
+1
View File
@@ -0,0 +1 @@
Copyright scriptc contributors. Licensed under Apache-2.0.
packages/libnode-darwin-arm64/THIRD_PARTY_NOTICES
+15
View File
@@ -0,0 +1,15 @@
This package contains libnode.137.dylib, a stripped build of the Node.js shared
library (https://nodejs.org) from the official source release named in
manifest.json, built without modification. Node.js is licensed under the
MIT license. It bundles third-party components, among them V8, ICU,
OpenSSL, libuv, zlib, brotli, nghttp2, nghttp3, ngtcp2, llhttp, c-ares,
simdutf, ada and uvwasi, under the BSD, Apache-2.0, Unicode, MIT and
other permissive licenses. LICENSE is the LICENSE file of that Node.js
release, which carries the complete license text of Node.js and of every
bundled component.
libscr_node_bridge.dylib is the scriptc bridge between the scriptc runtime and Node.js,
licensed under Apache-2.0 (SCRIPTC_LICENSE).
A program that `scriptc build` links against this package redistributes
Node.js; scriptc writes these notices next to such a program.
packages/libnode-darwin-arm64/package.json
+35
View File
@@ -0,0 +1,35 @@
{
"name": "@scriptc/libnode-darwin-arm64",
"version": "0.4.5",
"private": true,
"description": "Prebuilt Node.js shared library and scriptc bridge for Node islands on macOS arm64",
"license": "SEE LICENSE IN LICENSE",
"homepage": "https://scriptc.dev",
"repository": {
"type": "git",
"url": "git+https://github.com/vercel-labs/scriptc.git",
"directory": "packages/libnode-darwin-arm64"
},
"os": [
"darwin"
],
"cpu": [
"arm64"
],
"files": [
"libnode.137.dylib",
"libscr_node_bridge.dylib",
"manifest.json",
"LICENSE",
"SCRIPTC_LICENSE",
"THIRD_PARTY_NOTICES"
],
"scripts": {
"build": "true",
"build:native": "node scripts/build.mjs",
"prepack": "node ../libnode-common/scripts/verify.mjs ."
},
"publishConfig": {
"access": "public"
}
}
packages/libnode-darwin-arm64/scripts/build.mjs
+22
View File
@@ -0,0 +1,22 @@
#!/usr/bin/env node
// Builds @scriptc/libnode-darwin-arm64: libnode for the pinned Node.js release
// and the bridge (packages/libnode-common/scripts/build.mjs). The Node
// build takes about half an hour on 8 cores and is cached by Node version,
// configure flags and toolchain; only the libnode-packs workflow and an
// explicit `pnpm --filter @scriptc/libnode-darwin-arm64 build:native` run it.
import { dirname } from "node:path";
import { fileURLToPath } from "node:url";
import { buildLibnodePack } from "../../libnode-common/scripts/build.mjs";
if (process.platform !== "darwin" || process.arch !== "arm64") {
process.stdout.write("@scriptc/libnode-darwin-arm64: skipped on this host\n");
process.exit(0);
}
await buildLibnodePack({
platform: "darwin-arm64",
packageRoot: dirname(dirname(fileURLToPath(import.meta.url))),
compress: process.argv.includes("--compress"),
// Assemble from a restored libnode build only (CI test lanes); never
// start a Node build.
skipLibnode: process.argv.includes("--from-cache"),
});
packages/libnode-darwin-x64/SCRIPTC_LICENSE
+1
View File
@@ -0,0 +1 @@
Copyright scriptc contributors. Licensed under Apache-2.0.
packages/libnode-darwin-x64/THIRD_PARTY_NOTICES
+15
View File
@@ -0,0 +1,15 @@
This package contains libnode.137.dylib, a stripped build of the Node.js shared
library (https://nodejs.org) from the official source release named in
manifest.json, built without modification. Node.js is licensed under the
MIT license. It bundles third-party components, among them V8, ICU,
OpenSSL, libuv, zlib, brotli, nghttp2, nghttp3, ngtcp2, llhttp, c-ares,
simdutf, ada and uvwasi, under the BSD, Apache-2.0, Unicode, MIT and
other permissive licenses. LICENSE is the LICENSE file of that Node.js
release, which carries the complete license text of Node.js and of every
bundled component.
libscr_node_bridge.dylib is the scriptc bridge between the scriptc runtime and Node.js,
licensed under Apache-2.0 (SCRIPTC_LICENSE).
A program that `scriptc build` links against this package redistributes
Node.js; scriptc writes these notices next to such a program.

Some files were not shown because too many files have changed in this diff Show More