diff --git a/.github/actions/libnode-dev-pack/action.yml b/.github/actions/libnode-dev-pack/action.yml deleted file mode 100644 index 0619b1c45..000000000 --- a/.github/actions/libnode-dev-pack/action.yml +++ /dev/null @@ -1,31 +0,0 @@ -name: libnode development pack (Linux x64, macOS) -description: >- - Provide a libnode pack for the Node island tests and export - SCRIPTC_LIBNODE: restored from the cache, or on a miss assembled from the - pinned, checksum-verified inputs in packages/libnode-common/dev-pack.json - for the runner's platform (a prebuilt libnode and Node's headers; the - bridge builds in seconds). Temporary until the NI-13 packs exist. Fails - the job if no pack results: the island tests must never skip in CI. -runs: - using: composite - steps: - - name: Name the runner's platform - id: platform - shell: bash - run: node -p '`name=${process.platform}-${process.arch}`' >> "$GITHUB_OUTPUT" - - uses: actions/cache@v4 - id: libnode - with: - path: ${{ runner.temp }}/libnode-dev-pack - key: libnode-dev-pack-${{ steps.platform.outputs.name }}-${{ hashFiles('packages/libnode-common/dev-pack.json', 'packages/libnode-common/build-pack.mjs', 'packages/libnode-common/fetch-dev-pack.mjs', 'packages/libnode-common/bridge/**', 'packages/runtime/src/scr_node_bridge.h') }} - - name: Assemble the libnode pack - if: steps.libnode.outputs.cache-hit != 'true' - shell: bash - run: node packages/libnode-common/fetch-dev-pack.mjs --out "$RUNNER_TEMP/libnode-dev-pack" - - name: Export SCRIPTC_LIBNODE - shell: bash - run: | - case "$RUNNER_OS" in macOS) bridge=libscr_node_bridge.dylib ;; *) bridge=libscr_node_bridge.so ;; esac - test -f "$RUNNER_TEMP/libnode-dev-pack/manifest.json" - test -f "$RUNNER_TEMP/libnode-dev-pack/$bridge" - echo "SCRIPTC_LIBNODE=$RUNNER_TEMP/libnode-dev-pack" >> "$GITHUB_ENV" diff --git a/.github/actions/libnode/action.yml b/.github/actions/libnode/action.yml new file mode 100644 index 000000000..d0544b814 --- /dev/null +++ b/.github/actions/libnode/action.yml @@ -0,0 +1,64 @@ +name: libnode pack (Linux x64, macOS) +description: >- + Provide the runner's libnode pack for the Node island tests and export + SCRIPTC_LIBNODE. Restores the libnode build that the libnode-packs + workflow cached and assembles @scriptc/libnode- from it (the + bridge compiles in seconds; Node is never built here). Until that cache + exists for this ref, it falls back to the pinned, checksum-verified + development pack (packages/libnode-common/dev-pack.json). Fails the job if + no pack results: the island tests must never skip in CI. +runs: + using: composite + steps: + - name: Name the runner's pack + id: target + shell: bash + run: | + case "$RUNNER_OS-$RUNNER_ARCH" in + Linux-X64) platform=linux-x64-gnu toolchain=manylinux_2_28-gcc-toolset-14 bridge=libscr_node_bridge.so ;; + macOS-ARM64) platform=darwin-arm64 toolchain=macos-15-xcode-clang bridge=libscr_node_bridge.dylib ;; + macOS-X64) platform=darwin-x64 toolchain=macos-15-xcode-clang bridge=libscr_node_bridge.dylib ;; + *) echo "::error::no libnode pack for $RUNNER_OS $RUNNER_ARCH"; exit 1 ;; + esac + printf 'platform=%s\ntoolchain=%s\nbridge=%s\n' "$platform" "$toolchain" "$bridge" >> "$GITHUB_OUTPUT" + - name: Compute the libnode cache key + id: key + shell: bash + env: + SCRIPTC_LIBNODE_TOOLCHAIN: ${{ steps.target.outputs.toolchain }} + run: node packages/libnode-common/scripts/cache-key.mjs --platform ${{ steps.target.outputs.platform }} >> "$GITHUB_OUTPUT" + - uses: actions/cache/restore@v4 + id: libnode + with: + path: .libnode-cache/${{ steps.key.outputs.path }} + key: ${{ steps.key.outputs.key }} + - name: Assemble @scriptc/libnode-${{ steps.target.outputs.platform }} + if: steps.libnode.outputs.cache-hit == 'true' + shell: bash + env: + SCRIPTC_LIBNODE_TOOLCHAIN: ${{ steps.target.outputs.toolchain }} + SCRIPTC_LIBNODE_CACHE: ${{ github.workspace }}/.libnode-cache + run: | + node packages/libnode-${{ steps.target.outputs.platform }}/scripts/build.mjs --from-cache + node packages/libnode-common/scripts/verify.mjs packages/libnode-${{ steps.target.outputs.platform }} + echo "SCRIPTC_LIBNODE=$GITHUB_WORKSPACE/packages/libnode-${{ steps.target.outputs.platform }}" >> "$GITHUB_ENV" + - uses: actions/cache@v4 + if: steps.libnode.outputs.cache-hit != 'true' + id: devpack + with: + path: ${{ runner.temp }}/libnode-dev-pack + key: libnode-dev-pack-${{ steps.target.outputs.platform }}-${{ hashFiles('packages/libnode-common/dev-pack.json', 'packages/libnode-common/build-pack.mjs', 'packages/libnode-common/fetch-dev-pack.mjs', 'packages/libnode-common/bridge/**', 'packages/runtime/src/scr_node_bridge.h') }} + - name: Assemble the development pack + if: steps.libnode.outputs.cache-hit != 'true' + shell: bash + run: | + echo "::notice::no cached libnode-packs build for ${{ steps.key.outputs.key }}; using the development pack" + if [ "${{ steps.devpack.outputs.cache-hit }}" != true ]; then + node packages/libnode-common/fetch-dev-pack.mjs --out "$RUNNER_TEMP/libnode-dev-pack" + fi + echo "SCRIPTC_LIBNODE=$RUNNER_TEMP/libnode-dev-pack" >> "$GITHUB_ENV" + - name: Check the pack + shell: bash + run: | + test -f "$SCRIPTC_LIBNODE/manifest.json" + test -f "$SCRIPTC_LIBNODE/${{ steps.target.outputs.bridge }}" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0eb42486e..16e5e78dd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -215,7 +215,7 @@ jobs: # Node island tests (`// @requires-libnode` corpus programs and # node-island.test.ts) load Node from a libnode pack, and fail rather # than skip in this lane when none is configured. - - uses: ./.github/actions/libnode-dev-pack + - uses: ./.github/actions/libnode # Ubuntu pipes core dumps to apport, and RLIMIT_CORE does not limit a # piped dump. The kernel then streams every page of the runtime's # reserved allocator address space through the pipe, so each expected @@ -286,7 +286,7 @@ jobs: - run: pnpm build # Node island tests load Node from a libnode pack, and fail rather # than skip on macOS CI when none is configured. - - uses: ./.github/actions/libnode-dev-pack + - uses: ./.github/actions/libnode - name: Tests (${{ (github.event_name != 'schedule' && inputs.macos_suite != 'full') && 'darwin suite' || 'corpus and integration suites' }}) run: node scripts/ci-test.mjs @@ -707,7 +707,7 @@ jobs: - run: pnpm build # Node island tests load Node from a libnode pack, and fail rather # than skip on macOS CI when none is configured. - - uses: ./.github/actions/libnode-dev-pack + - uses: ./.github/actions/libnode - name: Node island harness if: matrix.shard == 1 run: pnpm test tests/harness/node-island packages/libnode-common @@ -808,7 +808,7 @@ jobs: - run: pnpm build # Node island tests load Node from a libnode pack, and fail rather # than skip on macOS CI when none is configured. - - uses: ./.github/actions/libnode-dev-pack + - uses: ./.github/actions/libnode - name: Node islands (Intel) run: >- pnpm test tests/harness/node-island packages/libnode-common diff --git a/.github/workflows/libnode-packs.yml b/.github/workflows/libnode-packs.yml new file mode 100644 index 000000000..fab3b630f --- /dev/null +++ b/.github/workflows/libnode-packs.yml @@ -0,0 +1,145 @@ +name: libnode packs + +# Builds the @scriptc/libnode- packs: the pinned Node.js release +# as a shared library plus the scriptc bridge (packages/libnode-common). +# A Node build takes about half an hour on 8 cores, so this workflow is +# never part of the ordinary build or test path. It runs only when the pin, +# the bridge or the pack packages change, or by hand, and it caches each +# platform's libnode build by (Node release, configure flags, platform, +# toolchain): a bridge-only change restores libnode from the cache and +# rebuilds just the bridge in seconds. +# +# Linux packs build in the manylinux_2_28 images (AlmaLinux 8, glibc 2.28, +# gcc-toolset-14), the glibc floor of Node's official Linux builds, on +# native runners: Node disables its startup snapshot when cross-compiling. +# Every pack is smoke-tested by a C program that loads it the way the +# scriptc runtime does (packages/libnode-common/smoke/smoke.c). + +on: + workflow_dispatch: + workflow_call: + push: + branches: + - main + paths: + - "packages/libnode-*/**" + - "packages/runtime/src/scr_node_bridge.h" + - ".github/workflows/libnode-packs.yml" + pull_request: + paths: + - "packages/libnode-*/**" + - "packages/runtime/src/scr_node_bridge.h" + - ".github/workflows/libnode-packs.yml" + +concurrency: + # A fixed name: under workflow_call, github.workflow is the caller's, and + # sharing the caller's group would cancel the run as a deadlock. + group: libnode-packs-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +permissions: + contents: read + +jobs: + build: + name: Build libnode pack (${{ matrix.platform }}) + strategy: + fail-fast: false + matrix: + include: + - platform: linux-x64-gnu + runner: ubuntu-24.04 + image: quay.io/pypa/manylinux_2_28_x86_64 + toolchain: manylinux_2_28-gcc-toolset-14 + - platform: linux-arm64-gnu + runner: ubuntu-24.04-arm + image: quay.io/pypa/manylinux_2_28_aarch64 + toolchain: manylinux_2_28-gcc-toolset-14 + - platform: darwin-arm64 + runner: macos-15 + toolchain: macos-15-xcode-clang + - platform: darwin-x64 + runner: macos-15-intel + toolchain: macos-15-xcode-clang + runs-on: ${{ matrix.runner }} + # A cold Linux build is ~30 minutes on 8 cores; the 3- and 4-core macOS + # runners are expected to take well over an hour. A cache hit is minutes. + timeout-minutes: 240 + env: + SCRIPTC_LIBNODE_TOOLCHAIN: ${{ matrix.toolchain }} + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version: "24" + + - name: Compute the libnode cache key + id: key + run: node packages/libnode-common/scripts/cache-key.mjs --platform ${{ matrix.platform }} >> "$GITHUB_OUTPUT" + + - name: Restore the libnode build + uses: actions/cache@v4 + with: + path: .libnode-cache/${{ steps.key.outputs.path }} + key: ${{ steps.key.outputs.key }} + + - name: Build the pack (Linux, manylinux_2_28) + if: startsWith(matrix.platform, 'linux') + run: | + NODE_ROOT=$(dirname "$(dirname "$(command -v node)")") + docker run --rm --volume "$GITHUB_WORKSPACE:/work" --volume "$NODE_ROOT:/node:ro" \ + --env SCRIPTC_LIBNODE_TOOLCHAIN --env SCRIPTC_LIBNODE_CACHE=/work/.libnode-cache \ + --env CC=gcc --env CXX=g++ \ + --env PATH=/opt/rh/gcc-toolset-14/root/usr/bin:/opt/python/cp312-cp312/bin:/node/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \ + --workdir /work ${{ matrix.image }} sh -c ' + set -eu + dnf install --assumeyes --quiet zstd + pip install --quiet ninja + node packages/libnode-${{ matrix.platform }}/scripts/build.mjs --compress + node packages/libnode-common/scripts/verify.mjs packages/libnode-${{ matrix.platform }} + # The glibc 2.28 floor: load the pack where it was built. + node packages/libnode-common/scripts/smoke.mjs --platform ${{ matrix.platform }} + chown -R "$(stat -c %u:%g /work)" /work/packages /work/.libnode-cache + ' + + - name: Build the pack (macOS) + if: startsWith(matrix.platform, 'darwin') + env: + CC: clang + CXX: clang++ + SCRIPTC_LIBNODE_CACHE: ${{ github.workspace }}/.libnode-cache + run: | + brew install ninja zstd + node packages/libnode-${{ matrix.platform }}/scripts/build.mjs --compress + node packages/libnode-common/scripts/verify.mjs packages/libnode-${{ matrix.platform }} + + - name: Smoke-test the pack on the runner + run: node packages/libnode-common/scripts/smoke.mjs --platform ${{ matrix.platform }} + + - name: Report sizes + run: | + { + echo "### @scriptc/libnode-${{ matrix.platform }} (Node ${{ steps.key.outputs.node }})" + echo + echo '```' + ls -l packages/libnode-${{ matrix.platform }} .libnode-cache/compressed/${{ matrix.platform }} + cat .libnode-cache/${{ steps.key.outputs.path }}/build.json + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + + - name: Upload the pack + uses: actions/upload-artifact@v4 + with: + name: libnode-${{ matrix.platform }} + path: | + packages/libnode-${{ matrix.platform }} + !packages/libnode-${{ matrix.platform }}/scripts + retention-days: 30 + + - name: Upload compressed libraries + uses: actions/upload-artifact@v4 + with: + name: libnode-${{ matrix.platform }}-zst + path: .libnode-cache/compressed/${{ matrix.platform }} + retention-days: 30 diff --git a/.gitignore b/.gitignore index 81b92ddd5..abb3f2558 100644 --- a/.gitignore +++ b/.gitignore @@ -33,6 +33,12 @@ output/ /packages/runtime-*/artifacts/ /packages/runtime-*/runtime-pack.json /packages/runtime-*/.runtime-pack-* +# libnode packs: built by build:native, never committed +/packages/libnode-*/libnode.so.* +/packages/libnode-*/libnode.*.dylib +/packages/libnode-*/libscr_node_bridge.* +/packages/libnode-*/manifest.json +/packages/libnode-*/LICENSE !tests/fixtures/npm/node_modules/*/dist/ !tests/fixtures/npm/node_modules/@*/*/dist/ !tests/fixtures/npm/workspace/*/dist/ diff --git a/RELEASING.md b/RELEASING.md index 09c0ce3b6..a32971f46 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -25,3 +25,19 @@ Each published package must have a GitHub Actions trusted publisher for `release.yml` and the `Release` environment. A missing configuration fails before upload. Re-runs skip package versions already present on npm, so a partially published release can be resumed safely. + +## libnode packs + +The `@scriptc/libnode-` packs (linux-x64-gnu, linux-arm64-gnu, darwin-arm64, darwin-x64) carry the Node.js shared library that programs with Node islands load in process, plus the scriptc bridge compiled against it. They version in lockstep with the other platform packages (`sync-versions.mjs` stamps them); the Node release they contain is pinned separately in `packages/libnode-common/libnode.json`, which must match `.node-version` and `internal/compatibility/node-v24.json`. Each pack's `manifest.json` records the Node version, `NODE_MODULE_VERSION`, the bridge ABI and the size and SHA-256 of both libraries. + +A Node `--shared` build takes about half an hour on 8 cores, so packs never build in `pnpm -r build`, the test suites or the release job. `.github/workflows/libnode-packs.yml` builds them on matching hosts when the pin, the bridge (`packages/libnode-common/bridge/`, `packages/runtime/src/scr_node_bridge.h`) or the pack packages change, or on manual dispatch. Linux packs build in the `manylinux_2_28` images (glibc 2.28, gcc-toolset-14) on native x64 and arm64 runners; Node disables its startup snapshot when cross-compiling, so arm64 is never cross-built. Each platform's libnode build is cached by Node release, configure flags, platform and toolchain; a bridge-only change restores libnode from the cache and rebuilds only the bridge. Every pack is smoke-tested by `packages/libnode-common/smoke/smoke.c`, which loads it the way the runtime does, and uploaded as the `libnode-` artifact, with zstd copies of both libraries as `libnode--zst`. The Linux x64 and macOS test lanes of `ci.yml` restore the cached build for their platform through `.github/actions/libnode`, assemble the pack (`build.mjs --from-cache`, which never builds Node) and set `SCRIPTC_LIBNODE`, so the Node island tests run; without a cached build they fall back to the pinned development pack (`packages/libnode-common/dev-pack.json`). `packages/libnode-common/build-pack.mjs` assembles a pack from any matching libnode and headers, the bridge-only rebuild path. + +To build one pack by hand on its own platform (it needs Python 3.9+, Ninja, a C++20 compiler and zstd; on Linux, build in a manylinux_2_28 container for a distributable pack): + +```bash +pnpm --filter @scriptc/libnode-linux-x64-gnu build:native +node packages/libnode-common/scripts/verify.mjs packages/libnode-linux-x64-gnu +node packages/libnode-common/scripts/smoke.mjs +``` + +The packs are `private` until the in-process Node island runtime ships. Publishing them then means removing `private`, configuring their npm trusted publishers, and having the release job download the `libnode-` artifacts of the workflow run for the release commit (or call the workflow, which restores every platform from its cache) and publish them with the other platform packages. diff --git a/docs/content/docs/experimental.mdx b/docs/content/docs/experimental.mdx index b5570d4e6..6221111ca 100644 --- a/docs/content/docs/experimental.mdx +++ b/docs/content/docs/experimental.mdx @@ -389,9 +389,9 @@ scriptc requires: - `T` to be a `ProgramLifetime` class, and the field to be an instance field of a `ProgramLifetime` class; - the constructor of the class that declares the field to write it once, from one of its parameters. A subclass constructor passes one of its own parameters there through `super(...)`. Nothing else writes the field; - the array passed to the constructor to be one that nothing else keeps: a new array (an array literal, a copy such as `slice()`, `[...items]` or `Array.from(items)`, or the result of a function that returns only such arrays or its own argument), a local array at its last use (calls that only read it or add to it may come first, and a loop may pass it and then assign a new array to the local), a parameter whose callers each pass such an array, an array that an InlineArray field already holds, or a `const` array that is never written, such as a shared empty array; -- the arrays fields hold never to be written, through a read of the field (which the `readonly` type allows only through a cast) or through any other reference. scriptc follows these arrays through locals, parameters, return values, fields, the elements of arrays, Maps and Sets, and closures, and reports one that reaches a write, a dynamic value, a call through a function value, or a library call other than printing and `JSON.stringify` without a replacer. The report names the field read the array may come from; +- the arrays fields hold never to be written, through a read of the field (which the `readonly` type allows only through a cast) or through any other reference. scriptc follows these arrays through locals, parameters, return values, fields, the elements of arrays, Maps and Sets, and closures, and reports one that reaches a write, a dynamic value (also inside the fields of an object that becomes one), a call through a function value (also inside an array, Map or Set passed there), or a library call other than printing and `JSON.stringify` without a replacer. The report names the field read the array may come from; - the arrays fields hold, and their reads, not to be compared by identity: with `===` or `!==`, as an `indexOf` or `includes` argument, or as a Map key or Set element. Two reads of the field need not give the same array. Compare the elements instead, or keep a copy made with `slice()` where identity matters; -- instances of the class not to be constructed through a class value, and not to become dynamic values (for example through `JSON.stringify` or a conversion to `unknown`). +- instances of the class not to be constructed through a class value, and not to become dynamic values (for example through a conversion to `unknown`), directly or inside another value that can hold one: an instance of a class with such a field, a record, array, Map or Set, or a value typed as a base class. A dynamic view reads the field as a new array each time, so a class whose instances can become dynamic values cannot hold an InlineArray field. Printing, `JSON.stringify` without a replacer, `publish` and a worker's `workerData` only read or copy the value and are allowed. The report names the conversion. scriptc reports a field that does not keep these rules (SC7009). diff --git a/packages/cli/src/bootstrap.ts b/packages/cli/src/bootstrap.ts index fbe48414b..cc1ca35ff 100644 --- a/packages/cli/src/bootstrap.ts +++ b/packages/cli/src/bootstrap.ts @@ -70,6 +70,9 @@ async function tryFastPath(): Promise { values["profile-generate"] !== undefined || values["profile-use"] !== undefined || values.sparse || + // A --libnode=beside build places the pack and the island modules next + // to the executable, which a routed hit does not do. + values.libnode === "beside" || // The full compiler resolves `native` against the build host. values["target-cpu"] !== undefined || (values["external-types"] ?? []).length > 0 || @@ -173,7 +176,7 @@ async function tryFastPath(): Promise { : startup.executableNativeEnvironmentFingerprint(), nodeVersion: process.version, // An entry of a program that calls into Node.js serves only the same pack. - libnodeIdentity: () => startup.libnodeCacheKey(values.libnode), + libnodeIdentity: () => startup.libnodeCacheKey(values.libnode, process.env, dirname(input)), }); if (hit === null) return null; if (!values["keep-llvm"]) await rm(hit.llvmPath, { force: true }); diff --git a/packages/cli/test/bootstrap.test.ts b/packages/cli/test/bootstrap.test.ts index 595b52ce5..53d6b63e1 100644 --- a/packages/cli/test/bootstrap.test.ts +++ b/packages/cli/test/bootstrap.test.ts @@ -123,6 +123,61 @@ test("bootstrap exact builds use the routed cache and source edits fall through" } }, 120_000); +test.skipIf(!persistentExecutable)( + "bootstrap leaves --libnode=beside builds to the full compiler", + async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-bootstrap-beside-")); + const entry = join(dir, "main.ts"); + const outPath = join(dir, "program"); + const preload = join(dir, "reject-full-compiler.mjs"); + const env = { ...process.env, SCRIPTC_CACHE_DIR: join(dir, "cache") }; + delete env.SCRIPTC_NO_CACHE; + const build = (args: string[], rejectFullCompiler = false) => + execFileAsync( + process.execPath, + [ + ...(rejectFullCompiler ? ["--import", preload] : []), + bootstrap, + "build", + entry, + "-o", + outPath, + ...args, + ], + { env, maxBuffer: 4 * 1024 * 1024 }, + ); + try { + await Promise.all([ + writeFile(entry, 'console.log("beside");\n'), + writeFile( + preload, + [ + "import { registerHooks } from 'node:module';", + "registerHooks({ load(url, context, nextLoad) {", + " if (url.includes('/packages/compiler/dist/index.js')) throw new Error('compiler graph loaded');", + " return nextLoad(url, context);", + "}});", + "", + ].join("\n"), + ), + ]); + for (const args of [[], ["--libnode=beside"]]) { + await build(args); + await build(args); + } + await expect(build([], true)).resolves.toBeDefined(); + // A routed hit would skip placing the pack and the island modules. + await expect(build(["--libnode=beside"], true)).rejects.toMatchObject({ + stderr: expect.stringContaining("compiler graph loaded"), + }); + expect((await execFileAsync(outPath)).stdout).toBe("beside\n"); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }, + 120_000, +); + test.skipIf(!runtimePackHost)( "bootstrap cache hits reuse LLVM executables", async () => { diff --git a/packages/compiler/native/host.c b/packages/compiler/native/host.c index bffdf933f..62db76b5a 100644 --- a/packages/compiler/native/host.c +++ b/packages/compiler/native/host.c @@ -73,12 +73,14 @@ void scriptc_native_collect_cycles(void) { int32_t scriptc_native_fork(void) { return -1; } void scriptc_native_exit(int32_t code) { ExitProcess((UINT)code); } int32_t scriptc_native_wait(int32_t pid) { (void)pid; return -1; } +int32_t scriptc_native_exited(void) { return -1; } uint8_t scriptc_native_lock_open(const uint8_t *bytes, size_t size) { (void)bytes; (void)size; return 0; } void scriptc_native_lock(void) {} void scriptc_native_unlock(void) {} #else #include #include +#include #include #include @@ -119,6 +121,18 @@ int32_t scriptc_native_wait(int32_t pid) { return -1; } +/* The pid of a child that has ended, waiting for one if none has, or -1 + * when waitid fails. The child is left to scriptc_native_wait: a child + * another part of the process waits for is not reaped here. */ +int32_t scriptc_native_exited(void) { + siginfo_t info; + memset(&info, 0, sizeof info); + while (waitid(P_ALL, 0, &info, WEXITED | WNOWAIT) < 0) { + if (errno != EINTR) return -1; + } + return info.si_pid > 0 ? (int32_t)info.si_pid : -1; +} + static int scriptc_native_lock_fd = -1; uint8_t scriptc_native_lock_open(const uint8_t *bytes, size_t size) { diff --git a/packages/compiler/native/host.ffi.json b/packages/compiler/native/host.ffi.json index e7ecbca60..9215ac834 100644 --- a/packages/compiler/native/host.ffi.json +++ b/packages/compiler/native/host.ffi.json @@ -6,6 +6,7 @@ { "name": "compilerNativeFork", "symbol": "scriptc_native_fork", "params": [], "returns": "i32" }, { "name": "compilerNativeExit", "symbol": "scriptc_native_exit", "params": ["i32"], "returns": "void" }, { "name": "compilerNativeWait", "symbol": "scriptc_native_wait", "params": ["i32"], "returns": "i32" }, + { "name": "compilerNativeExited", "symbol": "scriptc_native_exited", "params": [], "returns": "i32" }, { "name": "compilerNativeLockOpen", "symbol": "scriptc_native_lock_open", "params": ["string"], "returns": "bool" }, { "name": "compilerNativeLock", "symbol": "scriptc_native_lock", "params": [], "returns": "void" }, { "name": "compilerNativeUnlock", "symbol": "scriptc_native_unlock", "params": [], "returns": "void" }, diff --git a/packages/compiler/src/backend/executable-features.ts b/packages/compiler/src/backend/executable-features.ts index 73bce21b6..39756418e 100644 --- a/packages/compiler/src/backend/executable-features.ts +++ b/packages/compiler/src/backend/executable-features.ts @@ -43,7 +43,9 @@ export function executableLinkFeatures(mod: IrModule, largePages = false): Nativ ...(features.dynBuffer ? { dynBuffer: true } : {}), ...(features.builtinValues ? { builtinValues: true } : {}), ...(features.computedRequire ? { computedRequire: true } : {}), - events: features.processEvents, + // Island code's signal and exit listeners join the events unit's + // registries (scope-B NI-3). + events: features.processEvents || features.nodeIsland === true, emitter: features.emitter, symbol: features.symbol, bigint: features.bigint, diff --git a/packages/compiler/src/backend/llvm/emit-module.ts b/packages/compiler/src/backend/llvm/emit-module.ts index be01f4d92..549f5d748 100644 --- a/packages/compiler/src/backend/llvm/emit-module.ts +++ b/packages/compiler/src/backend/llvm/emit-module.ts @@ -115,24 +115,56 @@ export function workerMinFunctions(): number { return Number.isFinite(value) && value > 0 ? value : 2000; } +/** A function's estimated emission cost beyond its IR nodes, in nodes: + * fitted to the worker emission times of a large program's ranges. */ +const FUNCTION_EMIT_COST = 22; + +/** Where each of about `count` contiguous ranges of `functions` starts, + * followed by the end: ranges of about equal estimated emission cost, a + * function's node count plus FUNCTION_EMIT_COST. No range is empty. */ +export function emitRangeStarts( + functions: readonly IrFunction[], + count: number, + nodeCount: (fn: IrFunction) => number | undefined, +): number[] { + const costs = functions.map((fn) => (nodeCount(fn) ?? 0) + FUNCTION_EMIT_COST); + const total = costs.reduce((sum, cost) => sum + cost, 0); + const starts = [0]; + let sum = 0; + for (let i = 0; i < functions.length - 1 && starts.length < count; i++) { + sum += costs[i]!; + if (sum >= (total * starts.length) / count) starts.push(i + 1); + } + starts.push(functions.length); + return starts; +} + /** The function bodies emitted on forked worker processes, `workers` at a - * time: the function list splits into contiguous ranges, each emitted in - * its own process, which inherits this emitter as it is at the fork and - * records a journal (emit-journal.ts). Ranges merge in function order, so - * the text is the serial text. A range whose worker fails is emitted here - * when its turn comes. Null without fork primitives (Node, Windows, inside - * a worker) or when the module is small. */ + * time: the function list splits into contiguous ranges of about equal + * estimated cost (emitRangeStarts), each emitted in its own process, which + * inherits this emitter as it is at the fork and records a journal + * (emit-journal.ts). A new range starts whenever a worker ends, and ranges + * merge in function order, so the text is the serial text. A range whose + * worker fails is emitted here when its turn comes. Null without fork + * primitives (Node, Windows, inside a worker) or when the module is small. */ function emitFunctionsOnWorkers(emitter: LlEmitter, workers: number): string[] | null { const fork = emitter.targetOptions.forkPrimitives; const functions = emitter.mod.functions; if (fork === undefined || workers < 2) return null; if (functions.length < workerMinFunctions()) return null; // More ranges than workers: a range's merge overlaps the later ranges' - // emission, and a slow range holds back less. - const count = Math.min(functions.length, workers * 3); - const size = Math.ceil(functions.length / count); + // emission, and a slow range holds back less. Ranges of equal cost finish + // at about the same time. + const starts = emitRangeStarts(functions, Math.min(functions.length, workers * 3), (fn) => + emitter.nodeIndex.nodeCount(fn), + ); + const count = starts.length - 1; + const rangeFunctions = (range: number): readonly IrFunction[] => + functions.slice(starts[range], starts[range + 1]); const dir = mkdtempSync(join(tmpdir(), "scriptc-emit-")); const pids: number[] = []; + // The workers not yet reaped, by pid, with their ranges. + const running = new Map(); const start = (range: number): void => { const pid = fork.fork(); if (pid === 0) { @@ -141,7 +173,7 @@ function emitFunctionsOnWorkers(emitter: LlEmitter, workers: number): string[] | emitter.journal = new EmitJournal(); emitter.debug?.record(); const texts: string[] = []; - for (const fn of functions.slice(range * size, (range + 1) * size)) { + for (const fn of rangeFunctions(range)) { texts.push(emitter.emitFunction(fn)); relieveHeapPressure(); } @@ -155,17 +187,37 @@ function emitFunctionsOnWorkers(emitter: LlEmitter, workers: number): string[] | fork.exit(code); } pids.push(pid); + if (pid > 0) running.set(pid, range); }; const fnDefs: string[] = []; + // Ended ranges by index, with what their workers wrote (null when a worker + // failed), waiting to merge in function order. + const ended = new Map(); + // Reaps a worker and starts the next range in its place. A worker's journal + // records requests, not table contents, so its range's text doesn't depend + // on what merged before it forked. + const reap = (pid: number, range: number): void => { + const status = pid > 0 ? fork.wait(pid) : -1; + running.delete(pid); + if (pids.length < count) start(pids.length); + ended.set(range, status === 0 ? readRange(dir, range) : null); + }; try { for (let range = 0; range < Math.min(workers, count); range++) start(range); for (let range = 0; range < count; range++) { - const pid = pids[range]!; - const status = pid > 0 ? fork.wait(pid) : -1; - // The next worker starts from this emitter as merged so far, which - // the journal allows: it records requests, not table contents. - const merged = status === 0 ? readRange(dir, range) : null; - const slice = functions.slice(range * size, (range + 1) * size); + // Until this range's worker ends, reap whichever worker ends first, so + // that `workers` keep running while ranges merge in order. A child + // that isn't a worker (or no `exited`) waits for this range's worker. + while (!ended.has(range)) { + const pid = pids[range]!; + const first = pid > 0 && fork.exited !== undefined ? fork.exited() : pid; + const other = running.get(first); + if (first !== pid && other !== undefined) reap(first, other); + else reap(pid, range); + } + const merged = ended.get(range)!; + ended.delete(range); + const slice = rangeFunctions(range); if (merged !== null && merged.texts.length === slice.length) { const rename = emitter.mergeChunk(merged.result); slice.forEach((fn, index) => { @@ -180,7 +232,6 @@ function emitFunctionsOnWorkers(emitter: LlEmitter, workers: number): string[] | fnDefs.push(def); relieveHeapPressure(); } - if (pids.length < count) start(pids.length); } } finally { rmSync(dir, { recursive: true, force: true }); diff --git a/packages/compiler/src/backend/llvm/emitter.ts b/packages/compiler/src/backend/llvm/emitter.ts index 9726c4a92..4e8d6f121 100644 --- a/packages/compiler/src/backend/llvm/emitter.ts +++ b/packages/compiler/src/backend/llvm/emitter.ts @@ -516,6 +516,10 @@ export interface ForkPrimitives { exit(code: number): void; /** The worker's exit status, 128 plus a signal number, or -1. */ wait(pid: number): number; + /** The pid of a child process that has ended, waiting for one if none + * has, or -1, without reaping it. Without it, workers are waited for in + * order. */ + exited?(): number; } export interface LlvmTargetOptions { diff --git a/packages/compiler/src/backend/llvm/lib-shared.ts b/packages/compiler/src/backend/llvm/lib-shared.ts index 185c85638..ceb19a97d 100644 --- a/packages/compiler/src/backend/llvm/lib-shared.ts +++ b/packages/compiler/src/backend/llvm/lib-shared.ts @@ -1045,6 +1045,17 @@ export const LIB_FN_SYMS: Record = { "json.stringifyValue": "scr_json_stringify_value", "nodeIsland.callSync": "scr_node_island_call_sync", "nodeIsland.call": "scr_node_island_call", + "nodeIsland.require": "scr_ni_require", + "nodeIsland.import": "scr_ni_import", + "nodeIsland.get": "scr_node_island_get", + "nodeIsland.getKey": "scr_ni_get", + "nodeIsland.set": "scr_node_island_set", + "nodeIsland.setKey": "scr_node_island_set_key", + "nodeIsland.invoke": "scr_node_island_invoke", + "nodeIsland.apply": "scr_node_island_apply", + "nodeIsland.applyTo": "scr_node_island_apply_to", + "nodeIsland.construct": "scr_node_island_construct", + "nodeIsland.instanceof": "scr_node_island_instanceof", "dyn.keySet": "scr_dyn_key_set", "dyn.keySetComputed": "scr_dyn_key_set_computed", "dyn.keyDelete": "scr_dyn_key_delete", @@ -2022,4 +2033,15 @@ export const USES_TIMERS_LIB_FNS = new Set([ // drives (scr_node_island.c) before the program exits. "nodeIsland.call", "nodeIsland.callSync", + "nodeIsland.require", + "nodeIsland.import", + "nodeIsland.get", + "nodeIsland.getKey", + "nodeIsland.set", + "nodeIsland.setKey", + "nodeIsland.invoke", + "nodeIsland.apply", + "nodeIsland.applyTo", + "nodeIsland.construct", + "nodeIsland.instanceof", ]); diff --git a/packages/compiler/src/backend/native/executable.ts b/packages/compiler/src/backend/native/executable.ts index 1d9f24f03..eca133034 100644 --- a/packages/compiler/src/backend/native/executable.ts +++ b/packages/compiler/src/backend/native/executable.ts @@ -209,7 +209,9 @@ const RUNTIME_UNITS: readonly RuntimeUnit[] = [ u.opts.workers || u.opts.dc || u.opts.fileHandle || - u.nativeFetch, + u.nativeFetch || + // Island promises cross as dyn promise boxes (scr_node_island_handle.c). + u.opts.nodeIsland === true, "scr_async_dyn.c", ), // The performance timeline (every perf_hooks user-timing call selects @@ -369,7 +371,7 @@ const RUNTIME_UNITS: readonly RuntimeUnit[] = [ unitArgs((u) => (u.tlsCa && targetPlatform(u.driver) === "win32" ? ["-lcrypt32"] : [])), // Fetch uses the native network, TLS, and HTTP runtime. unit((u) => u.nativeFetch, "scr_fetch.c"), - unit((u) => u.opts.nodeIsland === true, "scr_node_island.c"), + unit((u) => u.opts.nodeIsland === true, "scr_node_island.c", "scr_node_island_handle.c"), // Readable.fromWeb reads a web ReadableStream through its reader. unit((u) => u.nativeFetch && u.opts.stream, "scr_stream_from_web.c"), ]; diff --git a/packages/compiler/src/cli/command.ts b/packages/compiler/src/cli/command.ts index 41209ab2a..042f09dcb 100644 --- a/packages/compiler/src/cli/command.ts +++ b/packages/compiler/src/cli/command.ts @@ -665,6 +665,7 @@ async function main(args: string[], host: CliHost): Promise { ? {} : { profileUse: resolve(values["profile-use"]) }), ...(npmStatic !== undefined ? { npmStatic } : {}), + ...(nodeIslands !== undefined ? { nodeIslands } : {}), ...(values["shape-overlay"] === false ? { shapeOverlay: false } : {}), ...(ffiProfilePath !== undefined ? { ffiProfilePath } : {}), ...(printNativeLinkInfo ? { nativeLinkInfo: true } : {}), diff --git a/packages/compiler/src/compile-types.ts b/packages/compiler/src/compile-types.ts index 376b2f096..7a632f60c 100644 --- a/packages/compiler/src/compile-types.ts +++ b/packages/compiler/src/compile-types.ts @@ -11,6 +11,9 @@ import type { NodeIslandOptions } from "./node-islands/policy.js"; export type CompileOutputKind = "ir" | "llvm" | "asm" | "obj" | "exe"; export interface CompileBaseOptions { + /** The Node island policy's command-line half (`--node-islands`, + * `--node-island`, `--native`): which npm packages run in Node islands. */ + nodeIslands?: NodeIslandOptions; /** Primary artifact path. The CLI supplies the output-kind default. */ outPath: string; /** Where generated intermediates and compatibility side artifacts land. */ diff --git a/packages/compiler/src/contracts/check.ts b/packages/compiler/src/contracts/check.ts index 76fd76a11..f0f2ccfce 100644 --- a/packages/compiler/src/contracts/check.ts +++ b/packages/compiler/src/contracts/check.ts @@ -373,10 +373,11 @@ class ContractChecker { ); for (const r of plan.rejected) { const note = notes.get(`${r.declaringClass}\0${r.field}`)!; + const at = r.loc === undefined ? "" : ` (at ${this.where(r.loc)})`; this.report( "SC7009", note.loc, - `'${note.name}' cannot keep its InlineArray contract: ${r.reason}`, + `'${note.name}' cannot keep its InlineArray contract: ${r.reason}${at}`, "write the field once in the constructor from one of its parameters (subclasses pass theirs through super(...)), never again, and keep the class's instances out of class values and dynamic values", ); } diff --git a/packages/compiler/src/diagnostics/codes.ts b/packages/compiler/src/diagnostics/codes.ts index 86cc740ab..1727d12e0 100644 --- a/packages/compiler/src/diagnostics/codes.ts +++ b/packages/compiler/src/diagnostics/codes.ts @@ -108,11 +108,13 @@ export const DIAGNOSTIC_CODES = { SC7007: "Acyclic type can lie on a reference cycle", SC7008: "ProgramLifetime class extends another class", SC7009: "InlineArray field contract not kept", + SC8001: "package or module runs in a Node island (note)", SC8002: "Node islands off for a package or module outside the native subset", SC8003: "package marked native is outside the native subset", SC8004: "unknown scriptc.packages or scriptc.nodeIslands configuration", SC8005: "no usable libnode pack for a program that calls into Node.js", SC8006: "Node island modules the build cannot collect", + SC8010: "value crossing a Node island boundary in a form without an exact copy", SC8030: "type patch: no range covers the installed copy", SC8031: "type patch: selector matches nothing", SC8032: "type patch: declaration already has the type", @@ -150,8 +152,6 @@ export const RETIRED_DIAGNOSTIC_CODES = { * that work lands it moves its codes into DIAGNOSTIC_CODES; when it is * abandoned, delete them here. */ export const RESERVED_DIAGNOSTIC_CODES = { - SC8001: "Node islands (ni/* branches)", - SC8010: "Node islands (ni/* branches)", SC8011: "Node islands (ni/* branches)", SC8012: "Node islands (ni/* branches)", SC8020: "Node islands (ni/* branches)", diff --git a/packages/compiler/src/diagnostics/diagnostic.ts b/packages/compiler/src/diagnostics/diagnostic.ts index 7d53b6bbc..8f33deeb1 100644 --- a/packages/compiler/src/diagnostics/diagnostic.ts +++ b/packages/compiler/src/diagnostics/diagnostic.ts @@ -116,6 +116,11 @@ export interface ScrDiagnostic { * alone does not say it (a computed require(), a native addon, `with`). * Report-only: never part of the rendered message. */ reason?: string; + /** Runtime fences only: the fenced site is in a module's top-level code + * (outside every function body), so it runs, and throws, when the module + * evaluates. Island selection treats such a fence like one that needs the + * runtime (scope-B NI-4). */ + moduleInit?: true; /** A runtime fence in a module's top-level code, outside every function * and class member body: it throws when the module initializes, which is * program startup for a module the entry imports. */ @@ -357,6 +362,34 @@ export function nodeIslandsOffModuleDiag( }; } +/** SC8001 — a package runs in a Node island (a note, once per island in + * the build output). */ +export function nodeIslandNoteDiag( + pkg: string, + version: string | null, + klass: string, + reason: string, + loc: SrcLoc, +): ScrDiagnostic { + return { + code: "SC8001", + message: `package '${version === null ? pkg : `${pkg}@${version}`}' runs in a Node island: ${klass}: ${reason}`, + loc, + hint: "it runs in Node.js loaded into the program on first use; --node-islands=off makes this an error", + }; +} + +/** SC8010 — a value crosses an island boundary in a way NI-4 cannot keep + * exact yet (callbacks, native objects, remote promises). */ +export function nodeIslandBoundaryDiag(what: string, loc: SrcLoc): ScrDiagnostic { + return { + code: "SC8010", + message: `${what} cannot cross into or out of a Node island yet`, + loc, + hint: "pass primitives, strings, island objects or fresh object and array literals of those; callbacks, native objects and promises cross from NI-5 on", + }; +} + /** SC8030–SC8033 — a type patch (`scriptc.typePatches`) that did not * apply as written: no range covers the installed copy (SC8030), its * selector matches nothing (SC8031), the declaration already has the type diff --git a/packages/compiler/src/executable/executable-cache.ts b/packages/compiler/src/executable/executable-cache.ts index a7f123860..3771656cf 100644 --- a/packages/compiler/src/executable/executable-cache.ts +++ b/packages/compiler/src/executable/executable-cache.ts @@ -185,6 +185,8 @@ export interface EarlyExecutableCacheOptions { /** Omitted unless SCRIPTC_HEAP_CENSUS is set (backend/llvm/heap-census.ts). */ heapCensus?: true; npmStatic: readonly string[] | "auto" | null; + /** Omitted for the default Node island policy; else nodeIslandOptionsKey. */ + nodeIslands?: string; /** Omitted unless shape inference's overlay compiles the packages * (frontend/shape-inference). */ shapeOverlay?: true; @@ -341,6 +343,7 @@ function cacheKey(options: EarlyExecutableCacheOptions): string { ...(options.functionLayout !== undefined ? [`layout:${options.functionLayout}`] : []), ...(options.checker !== undefined ? [`checker:${options.checker}`] : []), ...(options.youngNursery ? ["young-nursery"] : []), + ...(options.nodeIslands !== undefined ? [`node-islands:${options.nodeIslands}`] : []), ...(options.heapCensus ? ["heap-census"] : []), options.npmStatic === null ? "" @@ -389,6 +392,7 @@ function routeKey(options: Omit {}, + lowerIslands: ReadonlyMap = new Map(), + retryFences = true, ): PreparedExecutableModule | CompileFailure { - const fe = createFrontend(entryPath, opts.npmStatic); + // The island decision reads the detector's scan unless islands are off. + const islandScan = + nodeIslandPolicy(readProjectConfig(entryPath), opts.nodeIslands).mode !== "off"; + let fe = createFrontend(entryPath, opts.npmStatic, undefined, undefined, { islandScan }); timing("frontend-load"); + // Node islands (scope-B NI-4): packages refused natively run in Node, + // unless the policy says otherwise. The frontend reloads without them, + // so their declarations type the program. + const islandDecision = decideNodeIslands(entryPath, opts, fe, lowerIslands); + if (islandDecision !== null) { + fe.dispose(); + fe = createFrontend(entryPath, opts.npmStatic, undefined, undefined, { + islandPackages: new Set(islandDecision.packages.keys()), + islandScan, + }); + timing("frontend-island-reload"); + } let lowered: LowerResult; + let fenceEdges: readonly PackageEdge[] | undefined; let sourceTexts: Map; + let disposed = false; // The frontend (and its tsgo server) is released as soon as lowering // ends — clang and the link never hold it open. try { @@ -65,10 +95,32 @@ export function prepareExecutableModule( ), ); } - if (fe.preflight.length > 0) return fail(fe.preflight); + if (fe.preflight.length > 0) { + // An import of a natively compiled package that preflight refuses + // in the importer (a CommonJS namespace import) is that package's + // refusal: it runs in a Node island instead. + const grown = preflightIslands(entryPath, opts, fe, lowerIslands); + if (grown !== null) { + fe.dispose(); + frontendDisposed(); + disposed = true; + return prepareExecutableModule( + entryPath, + opts, + ffi, + buildPlatform, + createFrontend, + timing, + grown, + retryFences, + ); + } + return fail(fe.preflight); + } try { lowered = fe.lower({ + ...(islandDecision !== null ? { nodeIslands: islandDecision } : {}), targetPlatform: buildPlatform, ...(ffi !== null ? { ffiImports: ffi.functions } : {}), ...(opts.tier !== undefined ? { tierMode: opts.tier } : {}), @@ -83,7 +135,35 @@ export function prepareExecutableModule( checkerPanicDiag(e.message.split("\n", 1)[0]!, { file: entryPath, start: 0, end: 0 }), ]); } - if (lowered.module === null) return fail(lowered.diagnostics); + if (lowered.module === null) { + // Packages whose own modules refused to lower run in Node islands + // too (the authoritative lowering, DESIGN.md §1.4): build again with + // them as islands. + const grown = loweringIslands( + entryPath, + opts, + fe, + lowered.diagnostics, + lowerIslands, + fe.nodeIslandScan?.({ userJs: false }).edges, + ); + if (grown !== null) { + fe.dispose(); + frontendDisposed(); + disposed = true; + return prepareExecutableModule( + entryPath, + opts, + ffi, + buildPlatform, + createFrontend, + timing, + grown, + retryFences, + ); + } + return fail(lowered.diagnostics); + } const validation = validateModule(lowered.module); timing("ir-validate"); @@ -104,6 +184,10 @@ export function prepareExecutableModule( return fail([targetRefusalDiag("wasm32-wasi", unavailable.surface, unavailable.loc)]); } } + // The package import edges island selection reads after lowering, taken + // while the frontend is open. + if (retryFences && lowered.runtimeFences.some(sureFence)) + fenceEdges = fe.nodeIslandScan?.({ userJs: false }).edges; sourceTexts = fe.sourceTexts(); if (lowered.contracts) { const checked = checkContracts(lowered.module, lowered.contracts, sourceTexts); @@ -112,20 +196,277 @@ export function prepareExecutableModule( if (checked.contracts) lowered.module.contracts = checked.contracts; } } finally { - fe.dispose(); - frontendDisposed(); - timing("frontend-dispose"); + if (!disposed) { + fe.dispose(); + frontendDisposed(); + timing("frontend-dispose"); + } } - return { + const prepared: PreparedExecutableModule = { ok: true, mod: lowered.module!, sourceTexts, stats: lowered.stats, warnings: [ + ...(islandDecision?.notes ?? []), ...(fe.warnings ?? []), ...(lowered.divergences ?? []), ...(lowered.performance ?? []), ], }; + // Runtime fences in a compiled package that surely fire: fences that need + // the JavaScript runtime itself (R1: code from strings, computed + // import()/require(), addons, loader hooks), and fences in top-level + // module code, which throw when the module evaluates. Build again with + // those packages as islands, and keep this build when that one cannot + // compile. A gap fence inside a function body may never run, so it stays + // a deferred refusal. + const runtimeFences = lowered.runtimeFences.filter(sureFence); + if (retryFences && runtimeFences.length > 0) { + const grown = loweringIslands(entryPath, opts, fe, runtimeFences, lowerIslands, fenceEdges); + if (grown !== null) { + const islanded = prepareExecutableModule( + entryPath, + opts, + ffi, + buildPlatform, + createFrontend, + timing, + grown, + false, + ); + if (islanded.ok) return islanded; + } + } + return prepared; +} + +/** A run-time fence that surely fires: it needs the JavaScript runtime + * itself, or it is in top-level module code. */ +function sureFence(d: ScrDiagnostic): boolean { + return d.moduleInit === true || classifyDiagnostic(d, "fence").class === "runtime"; +} + +/** The packages a build runs in Node islands, or null for none: under + * `--node-islands=auto` (the default) every npm package the native + * pipeline refused, plus packages marked `node`; under `explicit` only the + * marked ones; under `off`, none (refusals stay SC2013). Each island is + * reported once as an SC8001 note. */ +function decideNodeIslands( + entryPath: string, + opts: CompileRequestOptions, + fe: ReturnType, + lowerIslands: ReadonlyMap, +): (NodeIslandSet & { notes: ScrDiagnostic[] }) | null { + const policy = nodeIslandPolicy(readProjectConfig(entryPath), opts.nodeIslands); + if (policy.mode === "off" || fe.nodeIslandScan === undefined) return null; + // A package marked native stays refused: the build reports it rather + // than running it in Node. + const refused = new Set( + policy.mode === "auto" + ? fe.npmStatic + .filter((e) => e.status === "refused" && !policy.forcedNative.has(e.package)) + .map((e) => e.package) + : [], + ); + for (const name of lowerIslands.keys()) refused.add(name); + if (refused.size === 0 && policy.forcedNode.size === 0) return null; + const report = buildNodeIslandReport({ + entry: entryPath, + scan: fe.nodeIslandScan({ userJs: false }), + npm: fe.npmStatic, + diagnostics: [], + fences: [], + policy, + }); + const packages = new Map(); + const notes: ScrDiagnostic[] = []; + const seen = new Set(); + for (const island of report.islands) { + if (island.path === null) continue; + seen.add(island.package); + if (!refused.has(island.package) && !policy.forcedNode.has(island.package)) continue; + packages.set(island.package, { + name: island.package, + version: island.version, + dir: island.path, + class: island.class, + reason: island.reason, + }); + notes.push( + nodeIslandNoteDiag( + island.package, + island.version, + island.class, + island.reason, + island.importedFrom[0] ?? + fe.npmImportSites.get(island.package) ?? { file: entryPath, start: 0, end: 0 }, + ), + ); + } + // Refused packages the report has no installed directory for (refused + // before they joined the program): their directory from the runtime + // entry Node resolves. + for (const name of refused) { + if (packages.has(name) || lowerIslands.has(name)) continue; + const site = fe.npmImportSites.get(name) ?? { file: entryPath, start: 0, end: 0 }; + const js = + resolveBareModule(site.file, name, "js-only") ?? + resolveBareModule(site.file, name, "js-only", "require"); + const at = js === null ? null : packageDirOf(js.typesFile); + if (at === null) continue; + const status = fe.npmStatic.find((e) => e.package === name); + packages.set(name, { + name, + version: packageVersionOf(at.dir), + dir: at.dir, + class: "packaging", + reason: status?.detail ?? "its static compilation was refused", + }); + notes.push( + nodeIslandNoteDiag(name, packageVersionOf(at.dir), "packaging", status?.detail ?? "", site), + ); + } + // Packages islanded by their lowering refusals keep their first reason + // (the reload no longer compiles them, so the scan has no reason). + for (const [name, island] of lowerIslands) { + if (seen.has(name) && packages.has(name)) { + const kept = packages.get(name)!; + if (kept.reason === "" || kept.class === "") packages.set(name, island); + continue; + } + packages.set(name, island); + notes.push( + nodeIslandNoteDiag( + island.name, + island.version, + island.class, + island.reason, + fe.npmImportSites.get(name) ?? { file: entryPath, start: 0, end: 0 }, + ), + ); + } + return packages.size === 0 ? null : { packages, notes }; +} + +/** The island set grown by the npm packages whose own modules refused to + * lower (`--node-islands=auto` only), or null when there are none. */ +function loweringIslands( + entryPath: string, + opts: CompileRequestOptions, + fe: ReturnType, + diagnostics: readonly ScrDiagnostic[], + known: ReadonlyMap, + edges: readonly PackageEdge[] | undefined, +): Map | null { + const policy = nodeIslandPolicy(readProjectConfig(entryPath), opts.nodeIslands); + if (policy.mode !== "auto" || known.size > 16) return null; + const compiled = new Set(fe.npmStatic.filter((e) => e.status === "static").map((e) => e.package)); + const grown = new Map(known); + for (const d of diagnostics) { + // A use of a package outside the compiled module graph (an import() + // the frontend did not compile): that package runs in an island. + const named = d.code === "SC2013" ? /npm package '([^']+)'/.exec(d.message) : null; + if (named !== null) { + const name = named[1]!; + if (grown.has(name) || policy.forcedNative.has(name) || policy.unloaded.has(name)) continue; + const js = + resolveBareModule(d.loc.file, name, "js-only") ?? + resolveBareModule(d.loc.file, name, "js-only", "require"); + const at = js === null ? null : packageDirOf(js.typesFile); + if (at === null || at.name !== name) continue; + grown.set(name, { + name, + version: packageVersionOf(at.dir), + dir: at.dir, + class: "packaging", + reason: `${d.code}: ${d.message}`, + }); + continue; + } + if (!isIslandReasonCode(d.code)) continue; + const at = packageDirOf(d.loc.file); + if (at === null || !compiled.has(at.name) || grown.has(at.name)) continue; + if (policy.forcedNative.has(at.name) || policy.unloaded.has(at.name)) continue; + const reason = classifyDiagnostic(d, "refusal"); + grown.set(at.name, { + name: at.name, + version: packageVersionOf(at.dir), + dir: at.dir, + class: reason.class, + reason: describe(reason), + }); + } + // A native ES module that imports an island CommonJS module would read + // its bindings from the module's exports object, where Node gives it the + // namespace its CommonJS facade builds (the lexed names, module.exports + // as the default): the importing package runs in the island too. + for (let added = edges !== undefined; added;) { + added = false; + for (const edge of edges!) { + const name = edge.to; + if (!grown.has(name) || edge.fromDir === null || edge.shim) continue; + const at = packageDirOf(edge.fromFile); + if (at === null || grown.has(at.name) || !compiled.has(at.name)) continue; + if (policy.forcedNative.has(at.name) || policy.unloaded.has(at.name)) continue; + if (!isEsmFile(edge.fromFile)) continue; + const target = + resolveBareModule(edge.fromFile, name, "js-only") ?? + resolveBareModule(edge.fromFile, name, "js-only", "require"); + if (target === null || isEsmFile(target.typesFile)) continue; + grown.set(at.name, { + name: at.name, + version: packageVersionOf(at.dir), + dir: at.dir, + class: "packaging", + reason: `an ES module of the package imports the CommonJS island package '${name}'`, + }); + added = true; + } + } + return grown.size === known.size ? null : grown; +} + +/** Preflight refusals at import statements of natively compiled npm + * packages (`--node-islands=auto` only): those packages become islands. */ +function preflightIslands( + entryPath: string, + opts: CompileRequestOptions, + fe: ReturnType, + known: ReadonlyMap, +): Map | null { + const policy = nodeIslandPolicy(readProjectConfig(entryPath), opts.nodeIslands); + if (policy.mode !== "auto" || known.size > 16) return null; + const compiled = new Set(fe.npmStatic.filter((e) => e.status === "static").map((e) => e.package)); + if (compiled.size === 0) return null; + const texts = fe.sourceTexts(); + const grown = new Map(known); + for (const d of fe.preflight) { + if (d.code !== "SC1013") continue; + const text = texts.get(d.loc.file); + if (text === undefined) continue; + // The import statement around the diagnostic: its module specifier. + const start = text.lastIndexOf("import", d.loc.start); + const end = text.indexOf(";", d.loc.end); + const stmt = text.slice(start < 0 ? d.loc.start : start, end < 0 ? text.length : end); + const m = /\bfrom\s*["']([^"']+)["']/.exec(stmt); + if (m === null) continue; + const spec = m[1]!; + const name = spec.startsWith("@") ? spec.split("/").slice(0, 2).join("/") : spec.split("/")[0]!; + if (!compiled.has(name) || grown.has(name)) continue; + if (policy.forcedNative.has(name) || policy.unloaded.has(name)) continue; + const site = fe.npmImportSites.get(name); + const jsEntry = site === undefined ? null : resolveBareModule(site.file, spec, "js-only"); + const at = jsEntry === null ? null : packageDirOf(jsEntry.typesFile); + if (at === null) continue; + grown.set(name, { + name, + version: packageVersionOf(at.dir), + dir: at.dir, + class: "gap", + reason: `${d.code}: ${d.message}`, + }); + } + return grown.size === known.size ? null : grown; } diff --git a/packages/compiler/src/frontend/lowering/builtins/module-resolution.ts b/packages/compiler/src/frontend/lowering/builtins/module-resolution.ts index 60dd8007f..3911188e4 100644 --- a/packages/compiler/src/frontend/lowering/builtins/module-resolution.ts +++ b/packages/compiler/src/frontend/lowering/builtins/module-resolution.ts @@ -194,9 +194,13 @@ function createRequireFence( surface: string, node: ts.Node, hint: string, + reason?: string, ): IrStmt & { kind: "runtimeFence" } { const loc = locOf(node); const d = noLoweringDiag(surface, loc, hint); + // The Node island reason the refusal would carry (a computed require is + // dynamic module loading), so island selection classifies the fence. + if (reason !== undefined) d.reason = reason; lowerer.runtimeFences.push(d); const line = ts.getLineAndCharacterOfPosition(ts.getSourceFileOfNode(node), loc.start).line + 1; return { @@ -299,6 +303,7 @@ function deferredCreateRequireThrow( surface: string, hint: string, loc: SrcLoc, + reason?: string, ): IrExpr { const stmts: IrStmt[] = call.arguments.nodes.map((arg) => ({ kind: "exprStmt", @@ -310,7 +315,7 @@ function deferredCreateRequireThrow( slot.mutable = true; stmts.push( { kind: "varDecl", localId: slot.id, init: null, loc }, - createRequireFence(lowerer, surface, call, hint), + createRequireFence(lowerer, surface, call, hint, reason), ); return { kind: "seqExpr", @@ -339,7 +344,7 @@ export function lowerCreateRequireCall( // Spread arguments have no run-time resolution: they run, then the // call throws the fence. if (args.some((a) => ts.isSpreadElement(a))) - return deferredCreateRequireThrow(lowerer, call, surface, hint, loc); + return deferredCreateRequireThrow(lowerer, call, surface, hint, loc, "dynamic-require"); // Node evaluates every argument, then requires the first. const stmts: IrStmt[] = []; let id = args.length > 0 ? lowerer.lowerExprExpecting(args[0]!, DYN) : dynUndefinedExpr(loc); diff --git a/packages/compiler/src/frontend/lowering/lower-exprs.ts b/packages/compiler/src/frontend/lowering/lower-exprs.ts index 89ea52bc5..b2e452faf 100644 --- a/packages/compiler/src/frontend/lowering/lower-exprs.ts +++ b/packages/compiler/src/frontend/lowering/lower-exprs.ts @@ -12,6 +12,7 @@ import { lowerNewTargetName, } from "./new-target.js"; import { lowerWebAssemblyValue, refuseWebAssemblyUse } from "./builtins/webassembly.js"; +import { lowerIslandExpr } from "./lower-node-island.js"; import { lowerAwaitedUniformPromiseAll } from "./builtins/promises.js"; import { dynUndefinedExpr, nodeThrowExpr, numLit, strLit, varRef } from "../../ir/build.js"; import { InternalCompilerError } from "../../errors.js"; @@ -289,6 +290,13 @@ function lowerExprInner(lowerer: Lowerer, expr: ts.Expression): IrExpr { const chainRecv = lowerer.chainRecvByNode.get(expr); if (chainRecv) return { ...chainRecv, loc }; + // Node islands (scope-B NI-4): member operations on island values and + // island import bindings. Programs without islands skip it. + if (lowerer.nodeIslands !== null) { + const island = lowerIslandExpr(lowerer, expr); + if (island !== null) return island; + } + // --external-types supplies CHECKER truth only. A value rooted in one // of those declarations has no runtime implementation in scriptc, so // every use fences at its expression instead of accidentally lowering diff --git a/packages/compiler/src/frontend/lowering/lower-modules.ts b/packages/compiler/src/frontend/lowering/lower-modules.ts index bffee4ef6..9804aebdf 100644 --- a/packages/compiler/src/frontend/lowering/lower-modules.ts +++ b/packages/compiler/src/frontend/lowering/lower-modules.ts @@ -1,6 +1,7 @@ import { computedGlobalReadOf, globalValuesInstall } from "./global-values.js"; import type { TypeFailureSlot } from "../../diagnostics/diagnostic.js"; import { THREADS_MODULE } from "../threads-import.js"; +import { collectIslandImport } from "./lower-node-island.js"; import { NODE_API_MODULE } from "../node-api-import.js"; import { positionTag } from "./structural-positions.js"; import { EXPERIMENTAL_MODULE } from "../experimental-import.js"; @@ -34,11 +35,13 @@ import { locOf, makeCycleAdmission, orderedImportsOf, + npmStaticDepSf7, requireSpecOf, resolveImport, resolveNpmImport, } from "../program.js"; -import type { CycleEdge } from "../program.js"; +import type { CycleEdge, StartupCrash } from "../program.js"; +import { isUnloadedPackage } from "../package-shims.js"; import { markCycleEarlyBindings, withClassInitFlag, @@ -175,6 +178,8 @@ export interface FileParts { export interface AppendGraph { edges?: ReadonlyMap | undefined; inlineCycleCalls?: Set; + /** LoadResult.dynamicImportRejections: import() roots that never load. */ + rejected?: ReadonlyMap | undefined; } /** An appended module's edges: its import and export declarations, then @@ -440,7 +445,8 @@ export function appendDynamicImportModules( if (scan.length === 0) break; for (const sf of scan) { scanned.add(sf); - for (const dep of dynamicProgramImportsOf(program, sf)) visit(dep); + for (const dep of dynamicProgramImportsOf(program, sf)) + if (graph?.rejected?.has(dep) !== true) visit(dep); } } if (added.length > 0) order.splice(order.length - 1, 0, ...added); @@ -648,7 +654,13 @@ export function collectNpmImports(lowerer: Lowerer, parts: FileParts[]): void { // graph: resolveImport follows its node_modules link to the loaded // TypeScript entry. Its internal `.js` spellings have already been // resolved to `.ts` by that graph. - if (!isRelativeSpecifier(spec) && resolveImport(lowerer.program, fp.sf, spec) !== null) { + const projectDep = isRelativeSpecifier(spec) + ? null + : resolveImport(lowerer.program, fp.sf, spec); + if (projectDep !== null) { + // A Node island package that tsconfig `paths` maps to its installed + // directory resolves to its declarations: it loads in Node. + if (projectDep.isDeclarationFile) collectIslandImport(lowerer, fp.sf, stmt, spec); continue; } const npm = resolveNpmImport(fp.sf.fileName, spec); @@ -700,6 +712,9 @@ export function collectNpmImports(lowerer: Lowerer, parts: FileParts[]): void { // it, %main throws before any init runs, and the bindings never if (npm !== null && probeNodeImportRefusal(fp.sf.fileName, spec) !== null) continue; + // A Node island package (scope-B NI-4): loaded in Node at this + // module's position, its bindings are island values. + if (collectIslandImport(lowerer, fp.sf, stmt, spec)) continue; { lowerer.pushDiag(unsupportedPackageImportDiag(npm?.packageName ?? relPkg!, locOf(stmt))); continue; @@ -722,6 +737,27 @@ export function collectNpmImports(lowerer: Lowerer, parts: FileParts[]): void { * specifiers are skipped — the lowering owns that fence (the module * graph is a build-time artifact; there is nothing to embed for a * runtime-computed name). */ +/** The refusal a literal import() rejects with, or null when it loads: + * Node's ERR_MODULE_NOT_FOUND for a package the project marks never loaded + * (package-shims.ts), or the first refusal in the static graph of a module + * only import() loads (LoadResult.dynamicImportRejections). */ +function dynamicImportRejectionOf( + lowerer: Lowerer, + sf: ts.SourceFile, + spec: string, + call: ts.CallExpression, +): StartupCrash | null { + if (isUnloadedPackage(spec)) { + const refusal = probeNodeImportRefusal(sf.fileName, spec); + if (refusal === null) return null; + return { message: refusal.message, className: "%Error", code: refusal.code, loc: locOf(call) }; + } + if (lowerer.dynamicImportRejections.size === 0) return null; + const root = + resolveImport(lowerer.program, sf, spec) ?? npmStaticDepSf7(lowerer.program, sf, spec); + return root === null ? null : (lowerer.dynamicImportRejections.get(root) ?? null); +} + function collectDynamicImports(lowerer: Lowerer, sf: ts.SourceFile): void { ts.walkPreorder(sf, (node) => { if ( @@ -742,6 +778,13 @@ function collectDynamicImports(lowerer: Lowerer, sf: ts.SourceFile): void { // module has no runtime namespace object — the per-site fence // says so. Only non-declaration source files count; a sibling // .d.mts typing shipped JS is the EMBED case, not this one. + // A package the project marks never loaded, or a module whose + // static graph Node's resolution refuses: the import() rejects. + const rejection = dynamicImportRejectionOf(lowerer, sf, spec, node); + if (rejection !== null) { + lowerer.dynImports.set(mapKey, { kind: "rejected", crash: rejection }); + return undefined; + } const modSym = lowerer.checker.getSymbolAtLocation(node.arguments.nodes[0]); const ownModule = modSym !== undefined && diff --git a/packages/compiler/src/frontend/lowering/lower-namespaces.ts b/packages/compiler/src/frontend/lowering/lower-namespaces.ts index 4ea102769..0a2158872 100644 --- a/packages/compiler/src/frontend/lowering/lower-namespaces.ts +++ b/packages/compiler/src/frontend/lowering/lower-namespaces.ts @@ -41,6 +41,7 @@ import * as ts from "../ts7/adapter.js"; import { isPackageShimFile, isShimIntrinsicName, packageShimFor } from "../package-shims.js"; import type { Lowerer } from "./lowerer.js"; import { boundIdentifiersOf } from "./lowerer.js"; +import { isIslandImportBinding } from "./lower-node-island.js"; import type { FileParts } from "./lower-modules.js"; import { isCjsJsFile, isJsSourceFile, locOf, resolveImport } from "../program.js"; import { F64, type IrExpr, type IrStmt, type IrType, STRING } from "../../ir/ir.js"; @@ -686,6 +687,9 @@ export function ambientUndefinedVarSymbolOf( lowerer: Lowerer, ident: ts.Identifier, ): ts.Symbol | null { + // An island package's `declare const` is defined by its JavaScript, which + // runs in the island. + if (isIslandImportBinding(lowerer, ident)) return null; const sym = lowerer.resolveValueSymbol(ident); const decl = sym ? sym.declarations[0] : undefined; if ( diff --git a/packages/compiler/src/frontend/lowering/lower-node-island.ts b/packages/compiler/src/frontend/lowering/lower-node-island.ts index fae8aa219..7f082fef4 100644 --- a/packages/compiler/src/frontend/lowering/lower-node-island.ts +++ b/packages/compiler/src/frontend/lowering/lower-node-island.ts @@ -16,14 +16,35 @@ * checked against the declared result type, like `JSON.parse(s) as T`; a * thrown Error crosses as its nearest intrinsic class with its message and * string `code`. */ -import { realpathSync } from "node:fs"; -import { isAbsolute } from "node:path"; +import { readFileSync, realpathSync } from "node:fs"; +import { resolveFrom } from "../../node-islands/resolve.js"; +import { dirname, isAbsolute, join } from "node:path"; import * as ts from "../ts7/adapter.js"; import { locOf } from "../program.js"; import type { NodeApiMember } from "../node-api-import.js"; import { PoisonError, type Lowerer } from "./lowerer.js"; -import { nodeIslandModuleDiag } from "../../diagnostics/diagnostic.js"; -import { DYN, STRING, type IrExpr, type IrType } from "../../ir/ir.js"; +import { + BOOL, + DYN, + FOREIGN, + STRING, + type IrExpr, + type IrGlobal, + type IrStmt, + type IrType, + type SrcLoc, +} from "../../ir/ir.js"; +import { varRef } from "../../ir/build.js"; +import { resolveBareModule } from "../resolve.js"; +import { nodeIslandBoundaryDiag, nodeIslandModuleDiag } from "../../diagnostics/diagnostic.js"; +import { + crossingOf, + isForeignType, + islandPackageOfFile, + type NodeIslandSet, +} from "../../node-islands/boundary.js"; +import { stripTypeWrappers } from "./typed-arguments.js"; +import { thenReader } from "./checked-promise-all.js"; const JSON_HINT = "only JSON data (numbers, strings, booleans, null, arrays, plain objects of those, and unknown) crosses into Node.js in this version"; @@ -88,6 +109,7 @@ export function lowerNodeApiCall( const loc = locOf(call); const surface = `${member} (@scriptc/node)`; const argNodes = call.arguments.nodes; + if (member === "requireModule") return lowerRequireModule(lowerer, call, surface); if (call.questionDotToken || argNodes.length < 2 || argNodes.some(ts.isSpreadElement)) { lowerer.noLowering( `${surface} without a specifier and an export name, or with spread arguments`, @@ -155,3 +177,766 @@ export function lowerNodeApiCall( // `JSON.parse(s) as T`: types don't lie, a mismatch throws. return lowerer.coerceToExpected(raw, declared); } + +/** requireModule(specifier): Node's require() from the calling module, run + * in the island; the exports cross by reference (scope-B NI-4: an island + * handle the program uses as an `any` value). The specifier must be a + * string literal: it resolves at build time exactly as require.resolve + * resolves it from the caller, and the island loads that file, shipped + * with the program (islandLoadPath). */ +function lowerRequireModule(lowerer: Lowerer, call: ts.CallExpression, surface: string): IrExpr { + const loc = locOf(call); + const argNodes = call.arguments.nodes; + const arg = argNodes[0]; + if ( + call.questionDotToken || + argNodes.length !== 1 || + arg === undefined || + !ts.isStringLiteral(arg) + ) { + lowerer.noLowering( + `${surface} without exactly one string-literal specifier`, + call, + "pass the module specifier as a string literal", + ); + } + const from = callerPath(call); + let path: string; + try { + path = resolveFrom(from, arg.text); + } catch { + lowerer.noLowering( + `${surface} of '${arg.text}', which does not resolve from ${from}`, + call, + "install the module or fix the specifier", + ); + } + lowerer.usesNodeIsland = true; + const raw: IrExpr = { + kind: "libCall", + fn: "nodeIsland.require", + args: [islandLoadPath(lowerer, path, loc)], + type: DYN, + loc, + }; + const declared = lowerer.mapTypeOf(lowerer.typeOf(call)); + if (declared === null || declared === undefined || declared.kind === "dyn") return raw; + return lowerer.coerceToExpected(raw, declared); +} + +/* ── compiler-managed Node islands (scope-B NI-4) ───────────────────────── + * + * A package the build runs in a Node island is never compiled. Its static + * imports load the island module at the importing module's position + * (eager evaluation: `nodeIsland.require` in the module's %init prelude, + * before the body), and their bindings read island values: a named import + * of an ES module reads the live namespace member at each use; a CommonJS + * module's named imports are read once after it evaluates, as Node's + * namespace for CommonJS snapshots them; a default import of CommonJS is + * module.exports. `require()` of an island package anywhere and `import()` + * load the same way at the call. + * + * Island values are typed by the package's declarations: values of + * island-declared object types are island handles (FOREIGN, see + * node-islands/boundary.ts), and member reads, writes, calls and `new` on + * them become bridge calls whose results meet the declared type through + * the checked dyn conversion (types don't lie: a mismatch throws). */ + +/** One island module load: the module global that holds what the import + * form sees (module.exports for CommonJS, the namespace for ESM). */ +export interface IslandModuleBinding { + global: IrGlobal; + /** The JavaScript file Node loads. */ + path: string; + esm: boolean; + /** CommonJS named imports, snapshotted after evaluation: name -> global. */ + snapshots: Map; +} + +function boundaryFail(lowerer: Lowerer, what: string, node: ts.Node): never { + lowerer.pushDiag(nodeIslandBoundaryDiag(what, locOf(node))); + throw new PoisonError(); +} + +/** Whether the JavaScript file is an ES module by Node's rules. */ +export function isEsmFile(file: string): boolean { + if (file.endsWith(".mjs")) return true; + if (file.endsWith(".cjs")) return false; + for (let dir = dirname(file); ;) { + try { + const pkg = JSON.parse(readFileSync(join(dir, "package.json"), "utf8")) as { + type?: unknown; + }; + return pkg.type === "module"; + } catch { + /* no package.json here */ + } + const parent = dirname(dir); + if (parent === dir) return false; + dir = parent; + } +} + +/** The island module file `spec` names from `fromFile`, or null when the + * specifier does not resolve into an island package. */ +export function islandModuleFile( + islands: NodeIslandSet, + fromFile: string, + spec: string, + kind: "import" | "require", +): string | null { + const js = resolveBareModule(fromFile, spec, "js-only", kind); + if (js === null) return null; + if (islandPackageOfFile(islands, js.typesFile) === null) return null; + try { + return realpathSync(js.typesFile); + } catch { + return js.typesFile; + } +} + +/** The module global of the island module at `path`, created on first use. */ +function islandModuleGlobal(lowerer: Lowerer, path: string): IslandModuleBinding { + let binding = lowerer.islandModules.get(path); + if (binding === undefined) { + const global: IrGlobal = { + id: `%g.island.${lowerer.globalsList.length}`, + name: path.split("/").at(-1) ?? "island", + type: DYN, + mutable: true, + }; + lowerer.globalsList.push(global); + binding = { global, path, esm: isEsmFile(path), snapshots: new Map() }; + lowerer.islandModules.set(path, binding); + } + return binding; +} + +/** The path literal of a load of the file at `path`. The build ships the + * file with the program (node-islands/modules.ts): the literal names it by + * its path in the shipped tree once lowering ends. A built-in module keeps + * its name. */ +function islandLoadPath(lowerer: Lowerer, path: string, loc: SrcLoc): IrExpr { + const literal: IrExpr & { kind: "strLit" } = { kind: "strLit", value: path, type: STRING, loc }; + if (!isAbsolute(path)) return literal; + let real = path; + try { + real = realpathSync(path); + } catch { + /* the resolved path */ + } + const problem = lowerer.nodeIslandModules.addLoaded(real); + if (problem !== null) { + lowerer.pushDiag(nodeIslandModuleDiag(problem, loc)); + throw new PoisonError(); + } + lowerer.nodeIslandLoads.push({ literal, path: real }); + return literal; +} + +function loadIslandModule(lowerer: Lowerer, path: string, loc: SrcLoc): IrExpr { + return { + kind: "libCall", + fn: "nodeIsland.require", + args: [islandLoadPath(lowerer, path, loc)], + type: DYN, + loc, + }; +} + +/** A static import (or re-export) of an island package: loads the module + * in the importer's %init prelude. Returns false when `spec` is not an + * island module. */ +export function collectIslandImport( + lowerer: Lowerer, + sf: ts.SourceFile, + stmt: ts.Statement, + spec: string, +): boolean { + const islands = lowerer.nodeIslands; + if (islands === null) return false; + const path = islandModuleFile(islands, sf.fileName, spec, "import"); + if (path === null) return false; + if (ts.isExportDeclaration(stmt)) { + lowerer.pushDiag( + nodeIslandBoundaryDiag(`re-exporting the Node island module '${spec}'`, locOf(stmt)), + ); + return true; + } + lowerer.usesNodeIsland = true; + const loc = locOf(stmt); + const binding = islandModuleGlobal(lowerer, path); + const actions = lowerer.jsonInitActions.get(sf) ?? []; + lowerer.jsonInitActions.set(sf, actions); + actions.push({ + kind: "assign", + localId: binding.global.id, + value: loadIslandModule(lowerer, path, loc), + loc, + }); + // CommonJS named imports: Node's namespace snapshots them once the + // module evaluated. + const clause = ts.isImportDeclaration(stmt) ? stmt.importClause : undefined; + const named = clause?.namedBindings; + if (!binding.esm && named !== undefined && ts.isNamedImports(named)) { + for (const el of named.elements.nodes) { + if (el.isTypeOnly) continue; + const name = el.propertyName?.text ?? el.name.text; + if (name === "default") continue; + let g = binding.snapshots.get(name); + if (g === undefined) { + g = { + id: `%g.island.${lowerer.globalsList.length}`, + name, + type: DYN, + mutable: true, + }; + lowerer.globalsList.push(g); + binding.snapshots.set(name, g); + } + actions.push({ + kind: "assign", + localId: g.id, + value: islandGet(varRef(binding.global.id, DYN, loc), name, loc), + loc, + }); + } + } + return true; +} + +function islandGet(target: IrExpr, key: string, loc: SrcLoc): IrExpr { + return { + kind: "libCall", + fn: "nodeIsland.get", + args: [target, { kind: "strLit", value: key, type: STRING, loc }], + type: DYN, + loc, + }; +} + +type IslandBinding = + | { kind: "namespace"; module: IslandModuleBinding } + | { kind: "default"; module: IslandModuleBinding } + | { kind: "named"; module: IslandModuleBinding; name: string }; + +function moduleSpecifierOf(decl: ts.Node | undefined): string | null { + return ts.isImportDeclaration(decl) && ts.isStringLiteral(decl.moduleSpecifier) + ? decl.moduleSpecifier.text + : null; +} + +/** What an identifier binds from an island module, or null. */ +/** Whether `ident` names an import of a Node island module: its value + * comes from the island, whatever the package's declarations say. */ +export function isIslandImportBinding(lowerer: Lowerer, ident: ts.Identifier): boolean { + return islandBindingOf(lowerer, ident) !== null; +} + +function islandBindingOf(lowerer: Lowerer, ident: ts.Identifier): IslandBinding | null { + const islands = lowerer.nodeIslands; + if (islands === null) return null; + const symbol = lowerer.checker.getSymbolAtLocation(ident); + const decl = symbol ? symbol.declarations[0] : undefined; + if (decl === undefined) return null; + let importDecl: ts.Node | undefined; + let binding: "namespace" | "default" | { name: string }; + if (ts.isNamespaceImport(decl)) { + importDecl = decl.parent?.parent; + binding = "namespace"; + } else if (ts.isImportClause(decl)) { + importDecl = decl.parent; + binding = "default"; + } else if (ts.isImportSpecifier(decl)) { + importDecl = decl.parent?.parent?.parent; + const name = decl.propertyName?.text ?? decl.name.text; + binding = name === "default" ? "default" : { name }; + } else { + return null; + } + const spec = moduleSpecifierOf(importDecl); + if (spec === null || importDecl === undefined) return null; + const path = islandModuleFile( + islands, + ts.getSourceFileOfNode(importDecl).fileName, + spec, + "import", + ); + if (path === null) return null; + const module = lowerer.islandModules.get(path); + if (module === undefined) return null; + if (binding === "namespace") return { kind: "namespace", module }; + if (binding === "default") return { kind: "default", module }; + return { kind: "named", module, name: binding.name }; +} + +/** The raw (dyn) value an island import binding reads. */ +function islandBindingValue( + lowerer: Lowerer, + b: IslandBinding, + node: ts.Node, + loc: SrcLoc, +): IrExpr { + const ns = varRef(b.module.global.id, DYN, loc); + switch (b.kind) { + case "namespace": + if (!b.module.esm) + boundaryFail(lowerer, "the namespace object of a CommonJS island module as a value", node); + return ns; + case "default": + return b.module.esm ? islandGet(ns, "default", loc) : ns; + case "named": { + if (b.module.esm) return islandGet(ns, b.name, loc); + const g = b.module.snapshots.get(b.name); + return g === undefined ? islandGet(ns, b.name, loc) : varRef(g.id, DYN, loc); + } + } +} + +/** An island value (dyn) at the declared type of `node`: island handles + * stay dyn (FOREIGN), primitives and strings meet their declared type + * through the checked conversion. */ +function fromIsland(lowerer: Lowerer, raw: IrExpr, node: ts.Expression): IrExpr { + const tsType = lowerer.typeOf(node); + const islands = lowerer.nodeIslands!; + if (isForeignType(islands, tsType, lowerer.checker)) return raw; + if ((tsType.flags & (ts.TypeFlags.Any | ts.TypeFlags.Unknown)) !== 0) return raw; + if ((tsType.flags & (ts.TypeFlags.Void | ts.TypeFlags.Undefined)) !== 0 && isDiscarded(node)) + return raw; + const mapped = lowerer.mapTypeOf(tsType); + if (mapped === null) + boundaryFail( + lowerer, + `an island value of type '${lowerer.checker.typeToString(tsType)}'`, + node, + ); + if (mapped.kind === "dyn") return raw; + if (mapped.kind === "void") return raw; + // An island promise is the same promise on both sides (the runtime keeps + // one ScrPromise per V8 promise): PromiseResolve of the dyn value, whose + // settlement value meets the declared type when it is awaited. + if ( + mapped.kind === "promise" && + (mapped.inner.kind === "dyn" || copiedType(lowerer, mapped.inner)) + ) { + const promise: IrExpr = { + kind: "libCall", + fn: "dyn.promiseResolve", + args: [raw, thenReader(lowerer, raw.loc)], + type: { kind: "promise", inner: DYN }, + loc: raw.loc, + }; + return mapped.inner.kind === "dyn" ? promise : lowerer.coerceToExpected(promise, mapped); + } + if (!copiedType(lowerer, mapped)) + boundaryFail( + lowerer, + `an island value of type '${lowerer.checker.typeToString(tsType)}' (only primitives and strings are copied; objects of types the island does not declare stay handles from NI-5 on)`, + node, + ); + return lowerer.coerceToExpected(raw, mapped); +} + +/** Whether a value of IR type `t` is copied at the boundary: primitives, + * strings, null/undefined, and unions of those. */ +function copiedType(lowerer: Lowerer, t: IrType): boolean { + switch (t.kind) { + case "f64": + case "string": + case "bool": + case "bigint": + case "nullT": + case "undefinedT": + return true; + case "union": { + const def = lowerer.unions.get(t.unionId); + return def !== undefined && def.arms.every((a) => copiedType(lowerer, a)); + } + default: + return false; + } +} + +function isDiscarded(node: ts.Node): boolean { + let n: ts.Node = node; + while (n.parent && ts.isParenthesizedExpression(n.parent)) n = n.parent; + return ( + n.parent !== undefined && (ts.isExpressionStatement(n.parent) || ts.isVoidExpression(n.parent)) + ); +} + +/** Whether `node` evaluates to an island handle (by its declared type) or + * is an island namespace binding. */ +function isForeignExpr(lowerer: Lowerer, node: ts.Expression): boolean { + const islands = lowerer.nodeIslands; + if (islands === null) return false; + const inner = stripTypeWrappers(node); + if (ts.isIdentifier(inner) && islandBindingOf(lowerer, inner) !== null) return true; + return isForeignType(islands, lowerer.typeOf(inner), lowerer.checker); +} + +/** A value crossing INTO the island (an argument or an assigned value), as + * dyn. */ +function toIsland(lowerer: Lowerer, node: ts.Expression): IrExpr { + const islands = lowerer.nodeIslands!; + const inner = stripTypeWrappers(node); + if (ts.isObjectLiteralExpression(inner) || ts.isArrayLiteralExpression(inner)) { + return freshLiteral(lowerer, inner); + } + const crossing = crossingOf(islands, lowerer.typeOf(node), lowerer.checker); + switch (crossing) { + case "copy": + case "handle": + return lowerer.coerceToExpected(lowerer.lowerExpr(node), DYN); + case "dyn": + case "refused": + return boundaryFail( + lowerer, + `a '${lowerer.checker.typeToString(lowerer.typeOf(node))}' value (a native object would cross as a copy and lose its identity)`, + node, + ); + } +} + +/** A fresh object or array literal crosses as a copy (a plain dyn object + * or array the runtime copies into V8): nothing native can alias it. Its + * members cross by the same rules (toIsland). */ +function freshLiteral( + lowerer: Lowerer, + node: ts.ObjectLiteralExpression | ts.ArrayLiteralExpression, +): IrExpr { + const loc = locOf(node); + if (ts.isArrayLiteralExpression(node)) { + const elems = node.elements.nodes.map((el) => { + if (ts.isSpreadElement(el) || ts.isOmittedExpression(el)) + boundaryFail(lowerer, "a spread or hole in an array literal", el); + return toIsland(lowerer, el); + }); + return { kind: "dynArrLit", elems, type: DYN, loc }; + } + const fields = node.properties.nodes.map((p) => { + let key: string; + let value: IrExpr; + if (ts.isPropertyAssignment(p) && (ts.isIdentifier(p.name) || ts.isStringLiteral(p.name))) { + key = p.name.text; + value = toIsland(lowerer, p.initializer); + } else if (ts.isShorthandPropertyAssignment(p)) { + const name = p.name as ts.Identifier; + key = name.text; + value = toIsland(lowerer, name); + } else { + return boundaryFail( + lowerer, + "a computed key, method, accessor or spread in an object literal", + p, + ); + } + return { key: { kind: "strLit", value: key, type: STRING, loc } as IrExpr, value }; + }); + return { kind: "dynObjLit", fields, type: DYN, loc }; +} + +/** Whether evaluating an argument runs no code that could change the + * receiver: literals, names, `this`, functions it creates, object and + * array literals of those, and operators over those. A unary operator + * (`+`, `-`, `~`) converts its operand with ToNumber, which calls a + * user-defined `valueOf`: only a numeric literal operand is inert. */ +function inertArgument(node: ts.Expression): boolean { + const e = stripTypeWrappers(node); + if ( + ts.isIdentifier(e) || + ts.isLiteralExpression(e) || + ts.isNoSubstitutionTemplateLiteral(e) || + ts.isArrowFunction(e) || + ts.isFunctionExpression(e) || + e.kind === ts.SyntaxKind.ThisKeyword || + e.kind === ts.SyntaxKind.TrueKeyword || + e.kind === ts.SyntaxKind.FalseKeyword || + e.kind === ts.SyntaxKind.NullKeyword + ) + return true; + if (ts.isParenthesizedExpression(e)) return inertArgument(e.expression); + if (ts.isArrayLiteralExpression(e)) + return e.elements.nodes.every((el) => !ts.isSpreadElement(el) && inertArgument(el)); + if (ts.isObjectLiteralExpression(e)) + return e.properties.nodes.every( + (p) => + ts.isShorthandPropertyAssignment(p) || + ts.isMethodDeclaration(p) || + (ts.isPropertyAssignment(p) && + !ts.isComputedPropertyName(p.name) && + inertArgument(p.initializer)), + ); + if (ts.isPrefixUnaryExpression(e)) { + // `!x` runs no conversion on its operand. `+`, `-` and `~` convert it + // with ToNumber, which calls a user-defined `valueOf` on an object: + // that can replace the method being called, so only a numeric literal + // is inert. + if (e.operator === ts.SyntaxKind.ExclamationToken) return inertArgument(e.operand); + return e.operand.kind === ts.SyntaxKind.NumericLiteral; + } + return false; +} + +function argsOf(lowerer: Lowerer, args: readonly ts.Expression[], loc: SrcLoc): IrExpr { + for (const a of args) if (ts.isSpreadElement(a)) boundaryFail(lowerer, "spread arguments", a); + return { kind: "dynArrLit", elems: args.map((a) => toIsland(lowerer, a)), type: DYN, loc }; +} + +/** Expressions on island values (the lowerExpr hook; null: not an island + * expression, the ordinary lowering applies). */ +export function lowerIslandExpr(lowerer: Lowerer, expr: ts.Expression): IrExpr | null { + if (lowerer.nodeIslands === null) return null; + const loc = locOf(expr); + if (ts.isIdentifier(expr)) { + const b = islandBindingOf(lowerer, expr); + if (b === null) return null; + lowerer.usesNodeIsland = true; + return fromIsland(lowerer, islandBindingValue(lowerer, b, expr, loc), expr); + } + if (ts.isPropertyAccessExpression(expr) && ts.isIdentifier(expr.name)) { + if (!isForeignExpr(lowerer, expr.expression)) return null; + if (expr.questionDotToken) + boundaryFail(lowerer, "an optional member read on an island value", expr); + lowerer.usesNodeIsland = true; + const target = islandReceiver(lowerer, expr.expression, expr.name.text, loc); + if (target.direct !== undefined) return fromIsland(lowerer, target.direct, expr); + return fromIsland(lowerer, islandGet(target.value, expr.name.text, loc), expr); + } + if (ts.isElementAccessExpression(expr)) { + if (!isForeignExpr(lowerer, expr.expression)) return null; + if (expr.questionDotToken) + boundaryFail(lowerer, "an optional member read on an island value", expr); + lowerer.usesNodeIsland = true; + const target = lowerer.lowerExpr(expr.expression); + const key = toIsland(lowerer, expr.argumentExpression); + return fromIsland( + lowerer, + { kind: "libCall", fn: "nodeIsland.getKey", args: [target, key], type: DYN, loc }, + expr, + ); + } + if (ts.isCallExpression(expr)) { + const dynamicLoad = lowerIslandLoadCall(lowerer, expr, loc); + if (dynamicLoad !== undefined) return dynamicLoad; + const callee = stripTypeWrappers(expr.expression); + if (expr.questionDotToken) return null; + const args = expr.arguments.nodes; + if (ts.isPropertyAccessExpression(callee) && ts.isIdentifier(callee.name)) { + if (!isForeignExpr(lowerer, callee.expression)) return null; + if (callee.questionDotToken) + boundaryFail(lowerer, "an optional call on an island value", expr); + lowerer.usesNodeIsland = true; + const target = islandReceiver(lowerer, callee.expression, callee.name.text, loc); + if (target.direct !== undefined) { + return fromIsland( + lowerer, + { + kind: "libCall", + fn: "nodeIsland.apply", + args: [target.direct, argsOf(lowerer, args, loc)], + type: DYN, + loc, + }, + expr, + ); + } + const name: IrExpr = { kind: "strLit", value: callee.name.text, type: STRING, loc }; + if (args.every((a) => inertArgument(a))) { + return fromIsland( + lowerer, + { + kind: "libCall", + fn: "nodeIsland.invoke", + args: [target.value, name, argsOf(lowerer, args, loc)], + type: DYN, + loc, + }, + expr, + ); + } + // An argument that runs code could replace the method: JavaScript + // reads the method first, then evaluates the arguments. + const stmts: IrStmt[] = []; + const receiver = lowerer.declareHiddenLocal("%islandRecv", target.value.type); + stmts.push({ kind: "varDecl", localId: receiver.id, init: target.value, loc }); + const recv = varRef(receiver.id, target.value.type, loc); + const method = lowerer.declareHiddenLocal("%islandMethod", DYN); + stmts.push({ + kind: "varDecl", + localId: method.id, + init: { kind: "libCall", fn: "nodeIsland.get", args: [recv, name], type: DYN, loc }, + loc, + }); + const call: IrExpr = { + kind: "libCall", + fn: "nodeIsland.applyTo", + args: [varRef(method.id, DYN, loc), recv, argsOf(lowerer, args, loc)], + type: DYN, + loc, + }; + const result = fromIsland(lowerer, call, expr); + return { kind: "seqExpr", stmts, result, type: result.type, loc }; + } + if (!isForeignExpr(lowerer, callee)) return null; + lowerer.usesNodeIsland = true; + const fn = lowerer.lowerExpr(callee); + return fromIsland( + lowerer, + { + kind: "libCall", + fn: "nodeIsland.apply", + args: [fn, argsOf(lowerer, args, loc)], + type: DYN, + loc, + }, + expr, + ); + } + if (ts.isNewExpression(expr)) { + if (!isForeignExpr(lowerer, expr.expression)) return null; + lowerer.usesNodeIsland = true; + const ctor = lowerer.lowerExpr(expr.expression); + return fromIsland( + lowerer, + { + kind: "libCall", + fn: "nodeIsland.construct", + args: [ctor, argsOf(lowerer, expr.arguments?.nodes ?? [], loc)], + type: DYN, + loc, + }, + expr, + ); + } + if (ts.isBinaryExpression(expr)) { + const op = expr.operatorToken.kind; + if (op === ts.SyntaxKind.InstanceOfKeyword && isForeignExpr(lowerer, expr.right)) { + lowerer.usesNodeIsland = true; + const value = lowerer.coerceToExpected(lowerer.lowerExpr(expr.left), DYN); + const ctor = lowerer.lowerExpr(expr.right); + return { kind: "libCall", fn: "nodeIsland.instanceof", args: [value, ctor], type: BOOL, loc }; + } + if (op === ts.SyntaxKind.EqualsToken) { + const left = stripTypeWrappers(expr.left); + if ( + (ts.isPropertyAccessExpression(left) && ts.isIdentifier(left.name)) || + ts.isElementAccessExpression(left) + ) { + if (!isForeignExpr(lowerer, left.expression)) return null; + lowerer.usesNodeIsland = true; + // JavaScript's order: the receiver, then the key, then the value. + const stmts: IrStmt[] = []; + const hold = (name: string, init: IrExpr, type: IrType): IrExpr => { + const local = lowerer.declareHiddenLocal(name, type); + stmts.push({ kind: "varDecl", localId: local.id, init, loc }); + return varRef(local.id, type, loc); + }; + const target = lowerer.lowerExpr(left.expression); + const receiver = hold("%islandSetTarget", target, target.type); + const key = ts.isPropertyAccessExpression(left) + ? null + : hold("%islandSetKey", toIsland(lowerer, left.argumentExpression), DYN); + const v = hold("%islandSet", toIsland(lowerer, expr.right), DYN); + stmts.push({ + kind: "exprStmt", + expr: + key === null + ? { + kind: "libCall", + fn: "nodeIsland.set", + args: [ + receiver, + { + kind: "strLit", + value: (left as ts.PropertyAccessExpression).name.text, + type: STRING, + loc, + }, + v, + ], + type: { kind: "void" }, + loc, + } + : { + kind: "libCall", + fn: "nodeIsland.setKey", + args: [receiver, key, v], + type: { kind: "void" }, + loc, + }, + loc, + }); + const result = fromIsland(lowerer, v, expr); + return { kind: "seqExpr", stmts, result, type: result.type, loc }; + } + return null; + } + const left = stripTypeWrappers(expr.left); + if ( + op >= ts.SyntaxKind.FirstCompoundAssignment && + op <= ts.SyntaxKind.LastCompoundAssignment && + (ts.isPropertyAccessExpression(left) || ts.isElementAccessExpression(left)) && + isForeignExpr(lowerer, left.expression) + ) { + boundaryFail(lowerer, "a compound assignment to an island object's member", expr); + } + } + return null; +} + +/** The receiver of a member operation on an island value. A CommonJS + * namespace binding's members are module.exports' (and `default` is + * module.exports itself), answered directly. */ +function islandReceiver( + lowerer: Lowerer, + receiver: ts.Expression, + member: string, + loc: SrcLoc, +): { value: IrExpr; direct?: IrExpr } { + const inner = stripTypeWrappers(receiver); + if (ts.isIdentifier(inner)) { + const b = islandBindingOf(lowerer, inner); + if (b !== null && b.kind === "namespace" && !b.module.esm) { + const exports = varRef(b.module.global.id, DYN, loc); + if (member === "default") return { value: exports, direct: exports }; + return { value: exports }; + } + } + return { value: lowerer.coerceToExpected(lowerer.lowerExpr(receiver), FOREIGN) }; +} + +/** `require("pkg")` (the global require, anywhere) and `import("pkg")` of + * an island module: load it at the call. Undefined: not such a call. */ +function lowerIslandLoadCall( + lowerer: Lowerer, + expr: ts.CallExpression, + loc: SrcLoc, +): IrExpr | undefined { + const islands = lowerer.nodeIslands!; + const arg = expr.arguments.nodes[0]; + if (expr.arguments.nodes.length !== 1 || arg === undefined || !ts.isStringLiteralLike(arg)) + return undefined; + const isImport = expr.expression.kind === ts.SyntaxKind.ImportKeyword; + const isRequire = + ts.isIdentifier(expr.expression) && + expr.expression.text === "require" && + !lowerer.checker + .getSymbolAtLocation(expr.expression) + ?.declarations.some((d) => !ts.getSourceFileOfNode(d).isDeclarationFile); + if (!isImport && !isRequire) return undefined; + const file = ts.getSourceFileOfNode(expr).fileName; + const path = islandModuleFile(islands, file, arg.text, isImport ? "import" : "require"); + if (path === null) return undefined; + lowerer.usesNodeIsland = true; + if (isImport) { + return { + kind: "libCall", + fn: "nodeIsland.import", + args: [islandLoadPath(lowerer, path, loc)], + type: { kind: "promise", inner: DYN }, + loc, + }; + } + return fromIsland(lowerer, loadIslandModule(lowerer, path, loc), expr); +} diff --git a/packages/compiler/src/frontend/lowering/lower-stdlib.ts b/packages/compiler/src/frontend/lowering/lower-stdlib.ts index 26200efbb..44f2b7370 100644 --- a/packages/compiler/src/frontend/lowering/lower-stdlib.ts +++ b/packages/compiler/src/frontend/lowering/lower-stdlib.ts @@ -38,6 +38,7 @@ import { isJsSourceFile, locOf, npmPackageNameOf, + type StartupCrash, } from "../program.js"; import { foldedStringKeyOf, lowerDynObjectLiteral } from "./expressions/object-literals.js"; import { noteEventListeners } from "./lower-event-target.js"; @@ -2243,6 +2244,7 @@ export function lowerDynamicImportCall(lowerer: Lowerer, call: ts.CallExpression } const res = lowerer.dynImports.get(`${ts.getSourceFileOfNode(call).fileName}\u0000${arg.text}`); if (res === undefined) lowerer.refuseUnsupportedApi("'import()'", call); + if (res.kind === "rejected") return lowerRejectedImport(lowerer, call, res.crash); if (res.kind === "program-module") { return lowerOwnStaticModuleImport(lowerer, call, arg); } @@ -2252,6 +2254,69 @@ export function lowerDynamicImportCall(lowerer: Lowerer, call: ts.CallExpression }); } +/** An import() Node rejects before any module of its graph evaluates + * (a package marked never loaded, a refused edge in the module's static + * graph): the promise rejects on a microtask with Node's error, and the + * module never joins the compiled graph. Its namespace is never produced, + * so the result is typed dynamically. */ +function lowerRejectedImport( + lowerer: Lowerer, + call: ts.CallExpression, + crash: StartupCrash, +): IrExpr { + const loc = locOf(call); + const resultT: IrType & { kind: "promise" } = { kind: "promise", inner: DYN }; + const fnName = lowerer.closureName(loc, "_rejectedImport"); + const fnType: IrType & { kind: "func" } = { kind: "func", params: [], ret: resultT }; + const fnCtx = newFnCtx(true, null, fnType, DYN); + fnCtx.isAsync = true; + lowerer.fnStack.push(fnCtx); + try { + const hop: IrExpr = { + kind: "awaitExpr", + value: { + kind: "intrinsic", + name: "promise.resolve", + args: [], + type: { kind: "promise", inner: VOID }, + loc, + }, + type: VOID, + loc, + }; + const errorKind = crash.className === "%TypeError" ? 1 : 0; + const body: IrStmt[] = [ + { kind: "exprStmt", expr: hop, loc }, + { + kind: "exprStmt", + expr: nodeThrowExpr(errorKind, crash.code ?? "", crash.message, VOID, loc), + loc, + }, + { kind: "return", value: dynUndefinedExpr(loc), loc }, + ]; + const ctx = lowerer.ctx; + lowerer.liftedFns.push({ + name: fnName, + params: [], + returnType: DYN, + locals: ctx.locals, + captures: ctx.captures ?? [], + body, + async: true, + loc, + }); + } finally { + lowerer.fnStack.pop(); + } + return { + kind: "callValue", + callee: { kind: "closure", fnName, captures: [], type: fnType, loc }, + args: [], + type: resultT, + loc, + }; +} + /** The checker-resolved program source behind one literal import(). */ function programImportTarget(lowerer: Lowerer, arg: ts.StringLiteralLike): ts.SourceFile | null { const modSym = lowerer.checker.getSymbolAtLocation(arg); diff --git a/packages/compiler/src/frontend/lowering/lower-stmts.ts b/packages/compiler/src/frontend/lowering/lower-stmts.ts index 08f468850..937f10379 100644 --- a/packages/compiler/src/frontend/lowering/lower-stmts.ts +++ b/packages/compiler/src/frontend/lowering/lower-stmts.ts @@ -2,6 +2,7 @@ import type { DiagnosticCode } from "../../diagnostics/codes.js"; import { keepsInitializerLayout, notePreservedLayout } from "./structural-instances.js"; import { jsGlobalValueByName } from "./global-values.js"; import { lowerTupleAliasDecl, stripTypeWrappers } from "./typed-arguments.js"; +import { lowerIslandExpr } from "./lower-node-island.js"; import { allBytesArms, indexedUnionArms, @@ -8032,6 +8033,11 @@ function lowerDereferencedElementWrite(lowerer: Lowerer, expr: ts.BinaryExpressi export function lowerExprStatement(lowerer: Lowerer, expr: ts.Expression): IrStmt { const stmtNode = expr.parent; while (ts.isParenthesizedExpression(expr)) expr = expr.expression; + // Node islands (scope-B NI-4): writes and calls on island values. + if (lowerer.nodeIslands !== null) { + const island = lowerIslandExpr(lowerer, expr); + if (island !== null) return { kind: "exprStmt", expr: island, loc: locOf(expr) }; + } // A discarded conditional has no value to merge: represent its lazy // branch selection directly as statement control flow. In particular, // both arms may be void calls (`flag ? emitA() : emitB()`), for which a diff --git a/packages/compiler/src/frontend/lowering/lowerer.ts b/packages/compiler/src/frontend/lowering/lowerer.ts index 7ae27aefd..2c3864964 100644 --- a/packages/compiler/src/frontend/lowering/lowerer.ts +++ b/packages/compiler/src/frontend/lowering/lowerer.ts @@ -44,6 +44,13 @@ import { import { moduleNamespaceObject, moduleNamespaceValue } from "./module-namespace-elements.js"; import { keyedHelperName, moduleTagNumber, portableRelativePath } from "./stable-names.js"; import { positionTag, StructuralPositions } from "./structural-positions.js"; +import { + activeNodeIslands, + isForeignType, + setActiveNodeIslands, + type NodeIslandSet, +} from "../../node-islands/boundary.js"; +import type { IslandModuleBinding } from "./lower-node-island.js"; import { bindingSymbolOfRead, onlyForwarded, stripTypeWrappers } from "./typed-arguments.js"; import { needsGeneralNodeStreams, @@ -938,6 +945,9 @@ export interface LowerResult { } export interface LowerOptions { + /** Packages that run in Node islands (scope-B NI-4): their imports lower + * to island loads and their values to island handles. */ + nodeIslands?: NodeIslandSet | null; /** Parser/checker services owned by the load; required for npm graph analysis. */ frontendServices?: FrontendServices | undefined; /** The project's tsconfig sets `checkJs` (see isCheckedJsSourceFile). */ @@ -965,6 +975,8 @@ export interface LowerOptions { inlineRequireCycleCalls?: ReadonlySet | undefined; /** LoadResult.moduleEdges. */ moduleEdges?: ReadonlyMap | undefined; + /** LoadResult.dynamicImportRejections. */ + dynamicImportRejections?: ReadonlyMap | undefined; /** LIBRARY mode's reachability roots: the profile-mapped exports of the * entry module. Executable builds root at the entry's top level alone * (an unreferenced export dead-strips); a library's exports are called from @@ -1086,6 +1098,8 @@ export interface LowererMode { dynNodeStreams?: boolean; /** LoadResult.inlineRequireCycleCalls. */ inlineRequireCycleCalls?: ReadonlySet | undefined; + /** LoadResult.dynamicImportRejections. */ + dynamicImportRejections?: ReadonlyMap | undefined; /** LowerOptions.libraryCallbacks (see there). */ libraryCallbacks?: boolean; projectCheckJs?: boolean; @@ -1175,6 +1189,60 @@ export function lowerToIr( entry: ts.SourceFile, moduleOrder: ts.SourceFile[], options: LowerOptions = {}, +): LowerResult { + setActiveNodeIslands(options.nodeIslands ?? null); + try { + const result = lowerToIrWithIslands(program, entry, moduleOrder, options); + markModuleInitFences(program, result.runtimeFences); + return result; + } finally { + setActiveNodeIslands(null); + } +} + +/** Marks the runtime fences in npm package code whose site is top-level + * module code (outside every function body): they throw when the module + * evaluates (ScrDiagnostic.moduleInit). */ +function markModuleInitFences(program: ts.Program, fences: readonly ScrDiagnostic[]): void { + for (const d of fences) { + if (!d.loc.file.includes("/node_modules/")) continue; + // A fence without a source position belongs to synthesized code, such + // as the global object's lookup (global-values.ts), which runs when the + // program calls it, not when the module evaluates. + if (d.loc.start < 0) continue; + const sf = program.getSourceFile(d.loc.file); + if (sf === undefined) continue; + let inFunction = false; + let node: ts.Node | undefined = sf; + while (node !== undefined && !inFunction) { + let next: ts.Node | undefined; + ts.forEachChild(node, (child) => { + if ( + next === undefined && + ts.tokenStartOfNode(child) <= d.loc.start && + d.loc.start < child.end + ) + next = child; + return undefined; + }); + if (next === undefined) break; + if ( + ts.isFunctionLike(next) || + (ts.isPropertyDeclaration(next) && + !next.modifiers?.nodes.some((m) => m.kind === ts.SyntaxKind.StaticKeyword)) + ) + inFunction = true; + node = next; + } + if (!inFunction) d.moduleInit = true; + } +} + +function lowerToIrWithIslands( + program: ts.Program, + entry: ts.SourceFile, + moduleOrder: ts.SourceFile[], + options: LowerOptions, ): LowerResult { const phaseTiming = process.env["SCRIPTC_TIMING"] === "1"; const phaseStarted = performance.now(); @@ -1219,7 +1287,11 @@ export function lowerToIr( // Inline require() calls closing a cycle inside the appended subgraphs // join preflight's set (the same run-time refusal by name). const inlineRequireCycleCalls = new Set(options.inlineRequireCycleCalls ?? []); - const graph = { edges: options.moduleEdges, inlineCycleCalls: inlineRequireCycleCalls }; + const graph = { + edges: options.moduleEdges, + inlineCycleCalls: inlineRequireCycleCalls, + rejected: options.dynamicImportRejections, + }; for (let pass = 0; pass < 32; pass++) { const before = moduleOrder.length; appendDynamicImportModules(program, moduleOrder, onExtraCycle, graph); @@ -1240,6 +1312,7 @@ export function lowerToIr( targetPlatform, startupCrash, inlineRequireCycleCalls, + dynamicImportRejections: options.dynamicImportRejections, ffiImports, libraryCallbacks, forkTargets, @@ -1264,6 +1337,7 @@ export function lowerToIr( targetPlatform, startupCrash, inlineRequireCycleCalls, + dynamicImportRejections: options.dynamicImportRejections, ffiImports, libraryCallbacks, forkTargets, @@ -1329,6 +1403,7 @@ export function lowerToIr( targetPlatform, startupCrash, inlineRequireCycleCalls, + dynamicImportRejections: options.dynamicImportRejections, ffiImports, libraryCallbacks, forkTargets, @@ -1374,6 +1449,7 @@ export function lowerToIr( targetPlatform, startupCrash, inlineRequireCycleCalls, + dynamicImportRejections: options.dynamicImportRejections, ffiImports, libraryCallbacks, forkTargets, @@ -1515,6 +1591,7 @@ export function lowerToIr( degradedTypes: resultLowerer.degradedTypes, retriedSymbols: resultLowerer.retriedSymbols, inlineRequireCycleCalls, + dynamicImportRejections: options.dynamicImportRejections, targetPlatform, ffiImports, libraryCallbacks, @@ -2135,6 +2212,8 @@ export class Lowerer { /** Inline require() calls whose module edge closes a require cycle * (LoadResult.inlineRequireCycleCalls). */ readonly inlineRequireCycleCalls: ReadonlySet; + /** LoadResult.dynamicImportRejections. */ + readonly dynamicImportRejections: ReadonlyMap; /** Immutable callable aliases and compiler-generated adapters. The map * is symbol-keyed, so shadowing and cross-module imports keep the same * exact binding discipline as globals/functions. */ @@ -3472,7 +3551,9 @@ export class Lowerer { * collection; lowerDynamicImportCall reads it per site). */ readonly dynImports = new Map< string, - { kind: "program-module" } | { kind: "static-builtin"; module: string } + | { kind: "program-module" } + | { kind: "static-builtin"; module: string } + | { kind: "rejected"; crash: StartupCrash } >(); readonly createRequireImports = new Map< @@ -3533,6 +3614,11 @@ export class Lowerer { /** The program calls into Node.js (@scriptc/node): the module carries * the nodeIsland record, which the executable build fills in. */ usesNodeIsland = false; + /** The program's Node island packages (scope-B NI-4), or null. */ + readonly nodeIslands: NodeIslandSet | null = activeNodeIslands(); + /** Island module loads, by the importing file and specifier: the module + * global holding what the import form sees (lower-node-island.ts). */ + readonly islandModules = new Map(); /** The modules Node island calls load (lower-node-island.ts). */ readonly nodeIslandModules = new NodeIslandModuleCollector(); /** Each call's caller literal, set to its tree path when lowering ends. */ @@ -3543,6 +3629,8 @@ export class Lowerer { caller: string; target: string; }[] = []; + /** Each island module load's path literal, set to its tree path when lowering ends. */ + readonly nodeIslandLoads: { literal: IrExpr & { kind: "strLit" }; path: string }[] = []; /** Per-directory answers of experimental-import.ts's declaration-file test. */ readonly experimentalFiles = new Map(); /** Interned LifetimeRegion.run helpers, by callback type key. */ @@ -4027,9 +4115,15 @@ export class Lowerer { // one lowers again rather than replaying a record. checkedTable("nodeIslandCallers", () => this.nodeIslandCallers.length), checkedTable("nodeIslandSpecifiers", () => this.nodeIslandSpecifiers.length), + // Island module loads hold literals of the unit's IR too, and their + // module globals (with each CommonJS named import's snapshot global) + // are created by the first load: a unit that adds either lowers again. + checkedTable("islandModules", () => this.islandModules.size), + checkedTable("nodeIslandLoads", () => this.nodeIslandLoads.length), checkedTable("fnSigsBySymbol", () => this.fnSigsBySymbol.size), checkedTable("cjsExportsAssignments", () => this.cjsExportsAssignments.size), checkedTable("inlineRequireCycleCalls", () => this.inlineRequireCycleCalls.size), + checkedTable("dynamicImportRejections", () => this.dynamicImportRejections.size), checkedTable("staticCallables", () => this.staticCallables.size), checkedTable("genericFnsBySymbol", () => this.genericFnsBySymbol.size), checkedTable("objLitGenericFns", () => this.objLitGenericFns.size), @@ -4249,6 +4343,7 @@ export class Lowerer { static readonly unjournaled: readonly string[] = [ // settings "frontendServices", + "nodeIslands", "ambient", "overridesAmbient", "remainder", @@ -4781,6 +4876,7 @@ export class Lowerer { this.forkTargets = mode.forkTargets ?? []; this.frontendServices = mode.frontendServices; this.inlineRequireCycleCalls = mode.inlineRequireCycleCalls ?? new Set(); + this.dynamicImportRejections = mode.dynamicImportRejections ?? new Map(); this.libraryCallbacks = mode.libraryCallbacks ?? false; this.projectCheckJs = mode.projectCheckJs ?? false; this.reachable = mode.reachable ?? null; @@ -4801,8 +4897,12 @@ export class Lowerer { this.externalTypeSpecifiersByFile = mode.externalTypeSpecifiersByFile ?? directExternalTypeSpecifiersByFile(this.externalTypes); this.checker = program.getTypeChecker(); + const islands = this.nodeIslands; const shadow = canonShadowHook(this.checker); this.typeCtx = { + ...(islands !== null + ? { foreignType: (t: ts.Type) => isForeignType(islands, t, this.checker) } + : {}), checker: this.checker, positionOf: (node) => this.positions.of(node), shapes: this.shapes, @@ -10550,14 +10650,16 @@ export class Lowerer { } /** The module's Node island record: the collected module tree, with each - * call's caller literal set to its path in that tree. The executable - * build fills in the libnode pack (index.ts). */ + * call's caller literal and each load's path literal set to its path in + * that tree. The executable build fills in the libnode pack (index.ts). */ private nodeIslandRecord(): NonNullable { const modules = this.nodeIslandModules.tree(); for (const { literal, caller } of this.nodeIslandCallers) literal.value = NodeIslandModuleCollector.callerPath(modules, caller); for (const { literal, caller, target } of this.nodeIslandSpecifiers) literal.value = NodeIslandModuleCollector.relativeSpecifier(caller, target); + for (const { literal, path } of this.nodeIslandLoads) + literal.value = NodeIslandModuleCollector.loadPath(modules, path); return { libnode: "", nodeVersion: "", ...(modules === null ? {} : { modules }) }; } diff --git a/packages/compiler/src/frontend/node-api-import.ts b/packages/compiler/src/frontend/node-api-import.ts index 04db2489b..798697bde 100644 --- a/packages/compiler/src/frontend/node-api-import.ts +++ b/packages/compiler/src/frontend/node-api-import.ts @@ -6,7 +6,11 @@ import * as ts from "./ts7/adapter.js"; export const NODE_API_MODULE = "@scriptc/node"; -export type NodeApiMember = "call" | "callSync"; +export type NodeApiMember = "call" | "callSync" | "requireModule"; + +function isNodeApiMember(name: string): name is NodeApiMember { + return name === "call" || name === "callSync" || name === "requireModule"; +} function moduleSpecifierText(decl: ts.Node | undefined): string | null { return ts.isImportDeclaration(decl) && ts.isStringLiteral(decl.moduleSpecifier) @@ -28,7 +32,7 @@ export function nodeApiBindingOf( if (!ts.isImportSpecifier(decl)) return null; if (moduleSpecifierText(decl.parent?.parent?.parent) !== NODE_API_MODULE) return null; const member = decl.propertyName?.text ?? decl.name.text; - return member === "call" || member === "callSync" ? member : null; + return isNodeApiMember(member) ? member : null; } /** The @scriptc/node export an identifier is a named-import binding of. */ @@ -55,6 +59,6 @@ export function nodeApiCalleeOf(program: ts.Program, callee: ts.Expression): Nod ) return null; const member = callee.name.text; - if (member !== "call" && member !== "callSync") return null; + if (!isNodeApiMember(member)) return null; return isNodeApiNamespace(program, callee.expression) ? member : null; } diff --git a/packages/compiler/src/frontend/pipeline.ts b/packages/compiler/src/frontend/pipeline.ts index 13407f499..c88077f59 100644 --- a/packages/compiler/src/frontend/pipeline.ts +++ b/packages/compiler/src/frontend/pipeline.ts @@ -49,6 +49,7 @@ import { NODE_API_MODULE } from "./node-api-import.js"; import { erasedTypeOnlyImport } from "./type-only-imports.js"; import { EXPERIMENTAL_MODULE } from "./experimental-import.js"; import { readProjectConfig } from "../project-config.js"; +import { setFrontendIslands } from "../node-islands/boundary.js"; import { typePatchDiagnostics } from "./type-patches/layer.js"; import { packageDirOf } from "../node-islands/origin.js"; import { @@ -128,12 +129,18 @@ export interface Frontend { } /** Hosts own transport lifetimes; shared compiler stages own the frontend. */ -/** What a frontend computes beyond the build's own needs. */ +/** How a frontend loads the program, beyond its npm package selection. */ export interface FrontendOptions { + /** Packages that run in Node islands (scope-B NI-4): never compiled, + * typed by their declarations. */ + islandPackages?: ReadonlySet; /** The Node island report (coverage): the project's scriptc settings and * the detector's scan of the program and its npm packages. Builds do not * read either. */ nodeIslands?: boolean; + /** The detector's scan alone, for a build that may run packages in Node + * islands (executable/prepare.ts decides which). */ + islandScan?: boolean; } export type FrontendFactory = ( @@ -196,6 +203,8 @@ function addDynamicImportGraph(load: LoadResult, scanned: Set): void npmStaticDepSf7(load.program, sf, spec, kind); if (dep === null || scanned.has(dep)) continue; if (dep.fileName.includes("/node_modules/") && !isJsSourceFileName(dep.fileName)) continue; + // An import() whose graph Node refuses rejects: nothing in it loads. + if (kind === "import" && load.dynamicImportRejections?.has(dep)) continue; scanned.add(dep); added.add(dep); queue.push(dep); @@ -480,15 +489,21 @@ export function runFrontend( transitive.add(base); return base; }); - return loadFrontend( - entryPath, - trackedLoader, - "lib", - externalTypes, - [...new Set(names)], - transitive, - options, - ); + const islands = options.islandPackages ?? new Set(); + setFrontendIslands(islands); + try { + return loadFrontend( + entryPath, + trackedLoader, + "lib", + externalTypes, + [...new Set(names)].filter((name) => !islands.has(name)), + transitive, + options, + ); + } finally { + setFrontendIslands(new Set()); + } } catch (error) { for (const load of active) { // Preserve the original failure and attempt every remaining cleanup. @@ -518,8 +533,10 @@ function loadFrontend( clearResolveCaches(); const statuses: NpmStaticStatus[] = []; const npmSites = new Map(); - // Node island detection (report-only): how each package was refused, and - // the modules of packages removed from the program. + // Node island detection (the report, or a build that may run islands): + // how each package was refused, and the modules of packages removed from + // the program. + const scanIslands = options.nodeIslands === true || options.islandScan === true; const refusals: RefusalRecord[] = []; const droppedScan: { modules: NodeIslandScan["modules"]; edges: NodeIslandScan["edges"] } = { modules: [], @@ -530,7 +547,11 @@ function loadFrontend( }; // Explicit library attempts bypass only the auto-selection heuristics. // The shared preflight/lowering refusal and dependency-closure checks remain. + // Node island packages are judged already: they stay out of the + // program, typed by their declarations (scope-B NI-4). const judged = new Set(libraryNpmStatic); + if (options.islandPackages !== undefined) + for (const name of options.islandPackages) judged.add(name); let requested: string[] = []; let reusableScout: ReturnType | null = null; let reusablePreflight: ScrDiagnostic[] | null = null; @@ -656,7 +677,7 @@ function loadFrontend( } const dropping = [...reasons.keys()].filter((p) => effective.has(p)); if (dropping.length === 0) break; - if (options.nodeIslands === true) scanDroppedPackages(load, dropping, droppedScan); + if (scanIslands) scanDroppedPackages(load, dropping, droppedScan); for (const p of dropping) { effective.delete(p); statuses.push({ package: p, status: "refused", detail: reasons.get(p)! }); @@ -708,15 +729,13 @@ function loadFrontend( probe.dispose(); if (probeDiags.some((d) => d.code === "SC0001")) dropWithNote(p); } - if (options.nodeIslands === true) - scanDroppedPackages(load, lastResortDropped.splice(0), droppedScan); + if (scanIslands) scanDroppedPackages(load, lastResortDropped.splice(0), droppedScan); load.dispose(); load = loadProgram(entryPath, { npmStatic: [...effective], externalTypes }); preflight = checkPreflight(load); if (preflight.some((d) => d.code === "SC0001") && effective.size > 0) { for (const p of [...effective]) dropWithNote(p); - if (options.nodeIslands === true) - scanDroppedPackages(load, lastResortDropped.splice(0), droppedScan); + if (scanIslands) scanDroppedPackages(load, lastResortDropped.splice(0), droppedScan); load.dispose(); load = loadProgram(entryPath, { npmStatic: [...effective], externalTypes }); preflight = checkPreflight(load); @@ -802,6 +821,7 @@ function loadFrontend( startupCrash: finalLoad.startupCrash ?? null, inlineRequireCycleCalls: finalLoad.inlineRequireCycleCalls, moduleEdges: finalLoad.moduleEdges, + dynamicImportRejections: finalLoad.dynamicImportRejections, projectCheckJs: finalLoad.typeWorld?.projectCheckJs ?? false, externalTypes: finalLoad.externalTypes, externalTypeSpecifiersByFile: finalLoad.externalTypeSpecifiersByFile, @@ -817,29 +837,28 @@ function loadFrontend( }, npmStatic: statuses, npmImportSites: npmSites, - nodeIslandScan: - options.nodeIslands !== true - ? undefined - : ({ userJs }) => { - const files = new Set([finalLoad.entry, ...finalLoad.moduleOrder]); - addDynamicImportGraph(finalLoad, files); - // Package edges matter only when the program has npm packages: one - // that loads, one that was refused, or one judged and dropped. - const npm = - statuses.length > 0 || - droppedScan.modules.length > 0 || - [...files].some((sf) => sf.fileName.includes("/node_modules/")); - const scanned = scanFiles(finalLoad.program, [...files], { - userJs, - dropped: false, - edges: npm, - }); - return { - modules: [...droppedScan.modules, ...scanned.modules], - edges: [...droppedScan.edges, ...scanned.edges], - refusals, - }; - }, + nodeIslandScan: !scanIslands + ? undefined + : ({ userJs }) => { + const files = new Set([finalLoad.entry, ...finalLoad.moduleOrder]); + addDynamicImportGraph(finalLoad, files); + // Package edges matter only when the program has npm packages: one + // that loads, one that was refused, or one judged and dropped. + const npm = + statuses.length > 0 || + droppedScan.modules.length > 0 || + [...files].some((sf) => sf.fileName.includes("/node_modules/")); + const scanned = scanFiles(finalLoad.program, [...files], { + userJs, + dropped: false, + edges: npm, + }); + return { + modules: [...droppedScan.modules, ...scanned.modules], + edges: [...droppedScan.edges, ...scanned.edges], + refusals, + }; + }, dispose: finalLoad.dispose, }; } diff --git a/packages/compiler/src/frontend/process-fork.ts b/packages/compiler/src/frontend/process-fork.ts index 744d57aa5..60637b44d 100644 --- a/packages/compiler/src/frontend/process-fork.ts +++ b/packages/compiler/src/frontend/process-fork.ts @@ -8,6 +8,9 @@ export interface ProcessFork { exit(code: number): void; /** The worker's exit status, 128 plus a signal number, or -1. */ wait(pid: number): number; + /** The pid of a child process that has ended, waiting for one if none + * has, or -1. It stays unreaped: `wait` reaps it. */ + exited(): number; /** Opens the lock file checker requests from workers take. */ openLock(path: string): boolean; lock(): void; diff --git a/packages/compiler/src/frontend/program.ts b/packages/compiler/src/frontend/program.ts index 58cdf219a..6438c50da 100644 --- a/packages/compiler/src/frontend/program.ts +++ b/packages/compiler/src/frontend/program.ts @@ -39,6 +39,7 @@ * JSON.parse's wording rather than tsc's, the one message-text delta on * that path (no snapshot pins it). */ +import { isFrontendIsland } from "../node-islands/boundary.js"; import { builtinModules } from "node:module"; import { dirname, isAbsolute, resolve } from "node:path"; import * as ts from "./ts7/adapter.js"; @@ -97,6 +98,7 @@ import { configurePackageShims, isPackageShimFile, isShimmedImport, + isUnloadedPackage, npmStaticOmitsPackage, } from "./package-shims.js"; import { provenanceEntryFor, provenancePaths } from "./provenance-registry.js"; @@ -123,6 +125,7 @@ import { isRelativeSpecifier, isWorkspacePackageName, npmPackageNameOf, + packageNameOfSpecifier, registerWorkspacePackage, workspacePackageOfPath, } from "./workspace-registry.js"; @@ -387,6 +390,11 @@ export interface LoadResult { * literal import(), fork() or Worker reaches with them. Filled by * checkPreflight. */ moduleEdges?: ReadonlyMap | undefined; + /** Program modules only a literal import() loads whose static graph + * Node's resolution refuses: each such import() rejects with the refusal + * instead of failing the build, and the module never joins the compiled + * graph. Filled by checkPreflight. */ + dynamicImportRejections?: ReadonlyMap | undefined; /** Node's startup refusal, when the graph carries an import edge Node * rejects before ANY module evaluates — a RESOLUTION refusal (a bare * specifier nothing installed resolves, an exports/imports map that @@ -539,6 +547,8 @@ export interface StartupCrash { message: string; className: "%Error" | "%TypeError" | "%SyntaxError"; loc: { file: string; start: number; end: number }; + /** Node's error code, for the refusals that carry one. */ + code?: string; } /** True when a coverage mapping names one exact bare host module. TypeScript @@ -1408,12 +1418,19 @@ function declBoundaryDroppedDiag(file: string): ScrDiagnostic { * preflight diagnostics. Preserve the caller's complete load shape when * mutating it, including its owned services and disposal callback. */ export function checkPreflight(load: T): ScrDiagnostic[] { - const { diags, moduleOrder, startupCrash, inlineRequireCycleCalls, moduleEdges } = - preflight7(load); + const { + diags, + moduleOrder, + startupCrash, + inlineRequireCycleCalls, + moduleEdges, + dynamicImportRejections, + } = preflight7(load); load.moduleOrder = moduleOrder; load.startupCrash = startupCrash; load.inlineRequireCycleCalls = inlineRequireCycleCalls; load.moduleEdges = moduleEdges; + load.dynamicImportRejections = dynamicImportRejections; return diags; } @@ -5352,6 +5369,7 @@ function preflight7(load: LoadResult): { startupCrash: StartupCrash | null; inlineRequireCycleCalls: ReadonlySet; moduleEdges?: ReadonlyMap; + dynamicImportRejections?: ReadonlyMap; } { const { program, entry } = load; const diags: ScrDiagnostic[] = [...load.configDiags]; @@ -5844,6 +5862,16 @@ function preflight7(load: LoadResult): { ); continue; } + // A re-export from a Node island package that tsconfig `paths` maps + // to its installed directory: its declarations are no program + // module (the lowering reports the island boundary). + if ( + projectReexport !== null && + projectReexport.isDeclarationFile && + !isRelativeSpecifier(fromSpec) && + isFrontendIsland(packageNameOfSpecifier(fromSpec)) + ) + continue; const reDep = projectReexport; // `export * as ns from "./m"` re-exports the module NAMESPACE // object under a name: importers' `x.ns.member` reads resolve @@ -5909,9 +5937,10 @@ function preflight7(load: LoadResult): { message: string, className: StartupCrash["className"], fallbackFeature: string, + code?: string, ): void => { refusals.get(sf)!.push({ - crash: { message, className, loc: locOf7(stmt) }, + crash: { message, className, loc: locOf7(stmt), ...(code !== undefined ? { code } : {}) }, fallback: unsupportedDiag("SC1010", locOf7(stmt), fallbackFeature), pos: ts.tokenStartOfNode(stmt), }); @@ -5954,6 +5983,7 @@ function preflight7(load: LoadResult): { refusal.message, "%Error", `the '${spec}' package (its runtime resolution fails: ${refusal.message})`, + refusal.code, ); } continue; @@ -5966,6 +5996,16 @@ function preflight7(load: LoadResult): { // Node's startup crash with Node's exact message; what keeps a compile // fence is unsupported builtin or types-only surface. if (isBare && npmStaticDep === null) { + // A Node island package that tsconfig `paths` maps to its + // installed directory resolves to its declarations here: it loads + // in Node, typed by them, like an island package resolved through + // node_modules (scope-B NI-4). + if ( + projDep !== null && + projDep.isDeclarationFile && + isFrontendIsland(packageNameOfSpecifier(spec)) + ) + continue; if (projDep !== null && projDep.isDeclarationFile) { diags.push( unsupportedDiag( @@ -5991,6 +6031,7 @@ function preflight7(load: LoadResult): { `Invalid module "#" is not a valid internal imports specifier name imported from ${sf.fileName}`, "%TypeError", `the '#' import (Node rejects the bare '#' specifier as invalid)`, + "ERR_INVALID_MODULE_SPECIFIER", ); continue; } @@ -6011,6 +6052,7 @@ function preflight7(load: LoadResult): { `Package import specifier "${spec}" is not defined${pkgJson !== null ? ` in package ${pkgJson}` : ""} imported from ${sf.fileName}`, "%TypeError", `the '${spec}' import (no package.json imports-field entry maps it)`, + "ERR_PACKAGE_IMPORT_NOT_DEFINED", ); continue; } @@ -6019,7 +6061,7 @@ function preflight7(load: LoadResult): { const ambientNote = ambientDeclared(spec) ? `the '${spec}' module (it exists only as an ambient 'declare module' type surface — there is no runtime module to import)` : `the '${spec}' package (nothing installed resolves it)`; - refuse(refusal.message, "%Error", ambientNote); + refuse(refusal.message, "%Error", ambientNote, refusal.code); continue; } // Runtime-resolvable (or a shape the probe stays conservative @@ -6125,7 +6167,12 @@ function preflight7(load: LoadResult): { ), ); } - } else if (dep === null && !(!isRelative && ambientModules.has(spec))) { + } else if ( + dep === null && + !(!isRelative && ambientModules.has(spec)) && + // A Node island module's namespace (scope-B NI-4). + !(!isRelative && isFrontendIsland(packageNameOfSpecifier(spec))) + ) { diags.push(unsupportedDiag("SC1013", locOf7(clause.namedBindings))); } } @@ -6485,6 +6532,7 @@ function preflight7(load: LoadResult): { // them). A found resolution crash wins over the CJS link check below: // Node's fetch/resolve phase runs before instantiate. let resolveCrash: StartupCrash | null = null; + const dynamicImportRejections = new Map(); { const seen = new Set(); const walk = (sf: ts.SourceFile): StartupCrash | null => { @@ -6504,6 +6552,45 @@ function preflight7(load: LoadResult): { return null; }; resolveCrash = walk(entry); + // A module only a literal import() loads resolves its static graph when + // that import runs: Node rejects the import's promise with the first + // refusal it meets, before any module of that graph evaluates. Such a + // module never joins the compiled graph, and the refusals inside its + // graph are that rejection, not build failures. + const reach = (from: ts.SourceFile, into: Set): void => { + const stack = [from]; + while (stack.length > 0) { + const sf = stack.pop()!; + if (into.has(sf)) continue; + into.add(sf); + for (const { dep } of depPositions.get(sf) ?? []) stack.push(dep); + } + }; + const startup = new Set(); + reach(entry, startup); + const covered = new Set(); + for (const sf of userFiles) { + if (!sf.text.includes("import")) continue; + for (const call of moduleSourceCandidates(program, sf).calls) { + const arg = call.arguments.nodes[0]; + if (call.expression.kind !== ts.SyntaxKind.ImportKeyword || arg === undefined) continue; + if (!ts.isStringLiteralLike(arg) || isUnloadedPackage(arg.text)) continue; + const root = + resolveImport7(program, sf, arg.text) ?? npmStaticDepSf7(program, sf, arg.text); + if (root === null || root.isDeclarationFile || startup.has(root)) continue; + // A module inside an already-rejected graph is `covered`, but a + // literal import() of it needs its OWN rejection entry: its graph + // rejects when that import runs, so it must not join the compiled + // graph (where lowering would fail) merely because another rejected + // import also reaches it. Walk each direct root independently. + if (dynamicImportRejections.has(root)) continue; + seen.clear(); + const crash = walk(root); + if (crash === null) continue; + dynamicImportRejections.set(root, crash); + reach(root, covered); + } + } // The crash channel engages only for programs that OTHERWISE pass // preflight: when any other diagnostic already fails the build, every // refusal keeps its compile-fence fallback instead — the report names @@ -6511,7 +6598,8 @@ function preflight7(load: LoadResult): { // get a diagnostic. (No reachable refusal → same: fallbacks only.) if (resolveCrash === null || diags.length > 0) { resolveCrash = null; - for (const cands of refusals.values()) { + for (const [sf, cands] of refusals) { + if (covered.has(sf) && !startup.has(sf)) continue; for (const c of cands) diags.push(c.fallback); } } @@ -6532,6 +6620,7 @@ function preflight7(load: LoadResult): { startupCrash: resolveCrash ?? linkCrash, inlineRequireCycleCalls, moduleEdges: edges, + dynamicImportRejections, }; } diff --git a/packages/compiler/src/frontend/resolve.ts b/packages/compiler/src/frontend/resolve.ts index 86252499f..dca3e52a8 100644 --- a/packages/compiler/src/frontend/resolve.ts +++ b/packages/compiler/src/frontend/resolve.ts @@ -24,6 +24,7 @@ import { } from "./npm-static.js"; import { clearPackageShimCaches, + isUnloadedPackage, packageShimFor, throughBundledShimForwarder, } from "./package-shims.js"; @@ -1026,6 +1027,9 @@ export function resolveWorkspaceSourceModule( ) { return null; } + // A workspace package the project marks never loaded (package-shims.ts) + // resolves like a missing one. + if (isUnloadedPackage(specifier)) return null; const source = resolveBareModule(fromFile, specifier, "runtime-source"); if ( source?.workspaceDir === undefined || diff --git a/packages/compiler/src/frontend/type-mapper.ts b/packages/compiler/src/frontend/type-mapper.ts index 8a75d5660..55a7631a4 100644 --- a/packages/compiler/src/frontend/type-mapper.ts +++ b/packages/compiler/src/frontend/type-mapper.ts @@ -19,6 +19,7 @@ import { CRYPTOHMAC_T, DATE_T, DYN, + FOREIGN, F64, funcOf, isSupportedArrayElem, @@ -1366,6 +1367,10 @@ export type TypeParamTsResolver = (t: ts.Type) => ts.Type | null; * once per Lowerer and threaded through the recursion — adding a hook means * one new field here, not a new parameter at every call site. */ export interface TypeMapperCtx { + /** Node islands (scope-B NI-4): whether a type's values are island + * handles, which map to the `foreign` type (a dyn value). Absent in + * programs without islands. */ + foreignType?: (type: ts.Type) => boolean; /** Set only while a union maps an arm that is the type inferred from an * empty object literal, beside another non-nullish arm: that arm holds * only the literal's objects, so it maps to the empty record (see the @@ -2006,6 +2011,7 @@ function mapTypeEntry(type: ts.Type, ctx: TypeMapperCtx): IrType | null { return STRING; if (flags & (ts.TypeFlags.Boolean | ts.TypeFlags.BooleanLiteral)) return BOOL; } + if (ctx.foreignType !== undefined && ctx.foreignType(type)) return FOREIGN; const localValue = ctx.localClassValue?.(type); if (localValue) { contextResolutions++; @@ -2290,6 +2296,11 @@ function mapTypeInner(type: ts.Type, ctx: TypeMapperCtx): IrType | null { return viaConditional; } } + const viaDistribution = mapDistributedConditional(type, ctx); + if (viaDistribution !== null) { + contextResolutions++; + return viaDistribution; + } } } const widened = checker.getBaseTypeOfLiteralType(type); @@ -5302,11 +5313,19 @@ function mapTypeInner(type: ts.Type, ctx: TypeMapperCtx): IrType | null { !checker.isNeverType(part) && !isEmptyObjectLiteralType(part, ctx), ); + let neverConditionalPart = false; + let undefinedPart: ts.Type | undefined; for (const part of parts) { if (hasArrayPart && emptyTuple(part)) continue; // TypeScript's client can retain impossible intersections in a // distributed union. They have no inhabitants and no runtime tag. if (checker.isNeverType(part)) continue; + // A conditional over a bound parameter whose every member selects + // `never` contributes no arm either. + if (distributedConditionalParts(part, ctx)?.length === 0) { + neverConditionalPart = true; + continue; + } // A `void` PART is inhabited only by undefined (`Promise | // void` return types, `string | void`): it becomes the undefinedT // unit arm, exactly like an undefined part — the value either @@ -5314,6 +5333,7 @@ function mapTypeInner(type: ts.Type, ctx: TypeMapperCtx): IrType | null { // void stays VOID (the return-position mapping above). if (part.flags & (ts.TypeFlags.Undefined | ts.TypeFlags.Void)) { byKey.set(typeKey(UNDEFINED_T), UNDEFINED_T); + undefinedPart ??= part; continue; } if (part.flags & ts.TypeFlags.Null) { @@ -5356,6 +5376,11 @@ function mapTypeInner(type: ts.Type, ctx: TypeMapperCtx): IrType | null { } const arms = [...byKey.values()]; if (arms.length === 0) return F64; // same unreachable placeholder as standalone never + // `Cond | undefined` whose conditional selects never for the + // instance's binding is the plain `undefined` the concrete + // instantiation's optional member has. + if (neverConditionalPart && arms.length === 1 && undefinedPart !== undefined) + return mapType(undefinedPart, ctx); // A single surviving UNIT arm cannot stand alone (degenerate — the // checker collapsed everything else away); anything else single is // just that type. A single arm UNDER A MINTED PLACEHOLDER cannot @@ -6442,6 +6467,104 @@ function mapGenericIndexedAccess(type: ts.Type, ctx: TypeMapperCtx): IrType | nu return { kind: "union", unionId: ctx.unions.intern(arms) }; } +/** A distributive conditional (`T extends "a" ? A : T extends "b" ? B : + * never`, its check type a naked type parameter) over a parameter the + * instance binds: evaluated once per member of the binding (`"a" | "b"`, + * or the open `"a" | (string & {})` spelling), as the checker distributes + * it. Answers the non-never results (empty when every member selects + * never, so an optional member of that type holds only undefined, as in + * the concrete instantiation), or null when the conditional is another + * shape or a member does not evaluate. */ +function distributedConditionalParts(type: ts.Type, ctx: TypeMapperCtx): ts.Type[] | null { + const { checker, resolveTypeParamTs } = ctx; + if (!resolveTypeParamTs || !type.isConditionalType()) return null; + const checkType = type.getCheckType(); + if (!checkType || !(checkType.flags & ts.TypeFlags.TypeParameter)) return null; + const symbol = checkType.getSymbol(); + const bound = resolveTypeParamTs(checkType); + if (!symbol || !bound) return null; + // The answer depends on the instance's binding: a mapping that consults + // it, even one that drops an all-never part, is not context-free. + contextResolutions++; + const parts: ts.Type[] = []; + for (const member of bound.isUnionType() ? ts.constituentTypes(bound) : [bound]) { + // `string & {}` (the open-union spelling) extends exactly what `string` + // extends: the empty object adds nothing to a primitive. + const emptyObject = (part: ts.Type): boolean => + (part.flags & ts.TypeFlags.Object) !== 0 && + checker.getPropertiesOfType(part).length === 0 && + checker.getIndexInfosOfType(part).length === 0 && + checker.getCallSignatures(part).length === 0 && + checker.getConstructSignatures(part).length === 0; + const primitive = member.isIntersectionType() + ? ts.constituentTypes(member).filter((part) => !emptyObject(part)) + : [member]; + if (primitive.length !== 1) return null; + const evaluated = evalMappedTemplate(type, new Map(), new Map([[symbol, primitive[0]!]]), ctx); + if (!evaluated) return null; + if (checker.isNeverType(evaluated)) continue; + if (!parts.includes(evaluated)) parts.push(evaluated); + } + return parts; +} + +/** A unit-only type once its conditionals over bound parameters are + * distributed: `Cond | undefined` where every member of T's binding + * selects never (the optional member's type in a generic body). */ +function neverConditionalUnitOnly(t: ts.Type, ctx: TypeMapperCtx): boolean { + const UNIT = ts.TypeFlags.Undefined | ts.TypeFlags.Void | ts.TypeFlags.Null; + if (!t.isUnionType()) return false; + let conditional = false; + for (const part of ts.constituentTypes(t)) { + if ((part.flags & UNIT) !== 0) continue; + if (distributedConditionalParts(part, ctx)?.length !== 0) return false; + conditional = true; + } + return conditional; +} + +/** mapType of a distributed conditional (distributedConditionalParts): the + * union of its members' results, as the concrete conditional's union maps + * them. An `unknown` or `any` result absorbs the rest (the checker's union + * is that type), as does a result that maps to the checked-dynamic value + * (`{}`, `object`, `Object`: the checker's union of one with a concrete + * result is the top type), and `undefined` and `null` results are the unit + * arms. Null when it does not distribute, when a result does not map, when + * every member selects never (the enclosing union drops such a part), or + * when only units remain. */ +function mapDistributedConditional(type: ts.Type, ctx: TypeMapperCtx): IrType | null { + const parts = distributedConditionalParts(type, ctx); + if (!parts || parts.length === 0) return null; + const top = parts.find((part) => (part.flags & (ts.TypeFlags.Any | ts.TypeFlags.Unknown)) !== 0); + if (top !== undefined) return mapType(top, ctx); + const arms: IrType[] = []; + for (const part of parts) { + if (part.flags & (ts.TypeFlags.Undefined | ts.TypeFlags.Void)) { + if (!arms.some((a) => typeEquals(a, UNDEFINED_T))) arms.push(UNDEFINED_T); + continue; + } + if (part.flags & ts.TypeFlags.Null) { + if (!arms.some((a) => typeEquals(a, NULL_T))) arms.push(NULL_T); + continue; + } + const mapped = mapType(part, ctx); + if (!mapped) return null; + // A top-type result (`{}`, `object`, `Object`): the checker's union with + // a concrete result is the top type itself, so the member is + // checked-dynamic. A `dyn` arm would be invalid union IR (SC9001). + if (mapped.kind === "dyn") return DYN; + const members = + mapped.kind === "union" ? (ctx.unions.get(mapped.unionId)?.arms ?? null) : [mapped]; + if (!members) return null; + for (const arm of members) if (!arms.some((a) => typeEquals(a, arm))) arms.push(arm); + } + if (arms.every(isUnitType)) return null; + if (arms.length === 1) return arms[0]!; + if (!unionContainerArmsOk(arms)) return null; + arms.sort((a, b) => (typeKey(a) < typeKey(b) ? -1 : 1)); + return { kind: "union", unionId: ctx.unions.intern(arms) }; +} + /** `keyof T extends never ? A : B` over a bound record T (the lib's * Object.entries element spelling): decided by whether the binding has any * key. */ @@ -8577,7 +8700,10 @@ function mapRecordTypeInner( // absent-field idiom — and `{ p: undefined }` spellings): the // unit-only union. The runtime value is the interned unit, JSON // omits the undefined arm exactly like an omitted optional. - if (pt?.kind === "void" && isUnitOnlyTsType(fieldTs, ctx.resolveTypeParam)) + if ( + pt?.kind === "void" && + (isUnitOnlyTsType(fieldTs, ctx.resolveTypeParam) || neverConditionalUnitOnly(fieldTs, ctx)) + ) pt = unitOnlyUnion(ctx.unions); if (pt !== null && isUnitOnlyIr(pt, ctx) && jsObjectLiteralProperty(p)) pt = DYN; if (pt?.kind === "array" && jsNeverArrayProp(p, fieldTs, checker)) pt = DYN; diff --git a/packages/compiler/src/index.ts b/packages/compiler/src/index.ts index 8764fe9ee..c9500e97d 100644 --- a/packages/compiler/src/index.ts +++ b/packages/compiler/src/index.ts @@ -1,3 +1,4 @@ +import { createRequire } from "node:module"; import { shapeOverlayEnabled } from "./frontend/shape-inference/index.js"; import { nodeFrontendServices } from "./frontend/services-node.js"; import { @@ -9,6 +10,8 @@ import { optimizationField, type NativeOptimization } from "./backend/optimizati import { targetCpuField, targetCpuTargetRefusal } from "./backend/target-cpu.js"; import { compilationTiming, type CompilationTiming, type TimingClock } from "./timing.js"; import { prepareExecutableModule } from "./executable/prepare.js"; +import { nodeIslandOptionsKey } from "./node-islands/policy.js"; +import { setIslandResolver } from "./node-islands/resolve.js"; import { prepareLibrary, libraryLocalizeSymbols, @@ -458,6 +461,10 @@ export async function analyzeCached( return result; } +// Island code resolves through Node (node-islands/resolve.ts; the +// self-hosted compiler builds no islands). +setIslandResolver((fromFile, spec) => createRequire(fromFile).resolve(spec)); + const nodeFrontend: FrontendFactory = ( entry, npmStatic, @@ -926,7 +933,7 @@ async function prepareExecutableInput( if (!moduleRuntimeFeatures(mod).nodeIsland) { delete mod.nodeIsland; } else if (outputKind === "exe") { - const recorded = libnodeRecord(opts.libnode, opts.outPath); + const recorded = libnodeRecord(opts.libnode, opts.outPath, process.env, dirname(entryPath)); if (typeof recorded === "string") return { ok: false, diagnostics: [nodeIslandNodeDiag(recorded, entryPath)], sourceTexts }; mod.nodeIsland = { ...mod.nodeIsland, ...recorded }; @@ -1762,7 +1769,7 @@ async function compileTracked( ...(opts.detachedExit === true ? { detachedExit: true as const } : {}), // Calls into Node record the libnode pack; their entries record its // identity (node-islands/libnode.ts) and a hit requires it unchanged. - libnodeIdentity: () => libnodeCacheKey(opts.libnode), + libnodeIdentity: () => libnodeCacheKey(opts.libnode, process.env, dirname(entryPath)), ...(opts.largePages === true ? { largePages: true as const } : {}), ...targetCpuField(opts.targetCpu), ...(opts.cpuDispatch === true ? { cpuDispatch: true as const } : {}), @@ -1770,6 +1777,9 @@ async function compileTracked( ...(functionLayoutDigest === undefined ? {} : { functionLayout: functionLayoutDigest }), ...checkerField(), ...(youngNurseryRequested() ? { youngNursery: true as const } : {}), + ...(nodeIslandOptionsKey(opts.nodeIslands) !== "" + ? { nodeIslands: nodeIslandOptionsKey(opts.nodeIslands) } + : {}), ...(heapCensusRequested() ? { heapCensus: true as const } : {}), npmStatic: opts.npmStatic ?? null, ...(shapeOverlayEnabled() ? { shapeOverlay: true as const } : {}), diff --git a/packages/compiler/src/ir/builtin-effects.ts b/packages/compiler/src/ir/builtin-effects.ts index c8dbc5a61..94ca32eb3 100644 --- a/packages/compiler/src/ir/builtin-effects.ts +++ b/packages/compiler/src/ir/builtin-effects.ts @@ -10,6 +10,17 @@ import type { IrLibFn } from "./builtin-signatures.js"; export const MAY_THROW_LIB_FNS: ReadonlySet = new Set([ // The copied error of a synchronous call into Node (@scriptc/node). "nodeIsland.callSync", + // Island module evaluation and member operations run JavaScript. + "nodeIsland.require", + "nodeIsland.get", + "nodeIsland.getKey", + "nodeIsland.set", + "nodeIsland.setKey", + "nodeIsland.invoke", + "nodeIsland.apply", + "nodeIsland.applyTo", + "nodeIsland.construct", + "nodeIsland.instanceof", // Node's argument, cipher-name, IV, tag-length and key-length errors. "crypto.cipherNew", // Node's curve, key, point, encoding and format errors. diff --git a/packages/compiler/src/ir/builtin-signatures.ts b/packages/compiler/src/ir/builtin-signatures.ts index 06ddb5007..65b560a22 100644 --- a/packages/compiler/src/ir/builtin-signatures.ts +++ b/packages/compiler/src/ir/builtin-signatures.ts @@ -185,6 +185,23 @@ function buildLibFnSignatures() { argTypes: [STRING, STRING, STRING, DYN], result: { kind: "promise", inner: DYN }, }, + /** Node islands (scope-B NI-4, scr_node_island_handle.c): island values + * are dyn values (`foreign` lowers as dyn), objects are handle boxes + * whose identity is the V8 object's. require/import load an island + * module by its absolute (or `$ORIGIN/`) path (also @scriptc/node + * requireModule); the rest are member reads, + * writes and calls on island values, argument lists as one dyn array. */ + "nodeIsland.require": { argTypes: [STRING], result: DYN }, + "nodeIsland.import": { argTypes: [STRING], result: { kind: "promise", inner: DYN } }, + "nodeIsland.get": { argTypes: [DYN, STRING], result: DYN }, + "nodeIsland.getKey": { argTypes: [DYN, DYN], result: DYN }, + "nodeIsland.set": { argTypes: [DYN, STRING, DYN], result: VOID }, + "nodeIsland.setKey": { argTypes: [DYN, DYN, DYN], result: VOID }, + "nodeIsland.invoke": { argTypes: [DYN, STRING, DYN], result: DYN }, + "nodeIsland.apply": { argTypes: [DYN, DYN], result: DYN }, + "nodeIsland.applyTo": { argTypes: [DYN, DYN, DYN], result: DYN }, + "nodeIsland.construct": { argTypes: [DYN, DYN], result: DYN }, + "nodeIsland.instanceof": { argTypes: [DYN, DYN], result: BOOL }, /** Keyed WRITE on a dyn value — `h.onDone = cb` / `h["k"] = v` on a * checked-dynamic object (args: receiver, key string, value — all * borrowed; the runtime copies the key and retains the value in). An diff --git a/packages/compiler/src/ir/inline-array-sources.ts b/packages/compiler/src/ir/inline-array-sources.ts index 5457b48af..54774e8c9 100644 --- a/packages/compiler/src/ir/inline-array-sources.ts +++ b/packages/compiler/src/ir/inline-array-sources.ts @@ -24,9 +24,18 @@ * Arrays are followed shallowly: only the array objects matter, not the * elements, which stay ordinary mutable instances. */ -import type { IrExpr, IrFunction, IrGlobal, IrModule, IrStmt, IrType, SrcLoc } from "./ir.js"; +import type { + IrClassDef, + IrExpr, + IrFunction, + IrGlobal, + IrModule, + IrStmt, + IrType, + SrcLoc, +} from "./ir.js"; import { typeKey } from "./ir.js"; -import type { InlineArrayPlan } from "./inline-arrays.js"; +import { readsArguments, type InlineArrayPlan } from "./inline-arrays.js"; import { everyExprChild, everyStmtChild } from "./traverse.js"; /** A use that breaks an InlineArray field's contract. */ @@ -88,25 +97,6 @@ const RETURNS_RECEIVER: ReadonlySet = new Set([ "copyWithin", ]); -/** Library calls that only read their arguments: printing, inspection, - * and JSON.stringify without a replacer function. */ -function readsArguments(e: IrExpr & { kind: "libCall" }): boolean { - if (/^(console|insp|inspect)\./.test(e.fn)) return true; - switch (e.fn) { - case "array.hasProperties": - case "array.hasNamedKey": - case "array.namedKeys": - case "json.stringifyValue": - return true; - case "json.stringifyReplacer": { - const replacer = e.args[1]; - return replacer?.kind === "dynFrom" && replacer.value.kind === "unitLit"; - } - default: - return false; - } -} - /** Expressions whose value is one of their children's. */ const PASS_THROUGH: ReadonlySet = new Set([ "upcast", @@ -510,6 +500,8 @@ class SourceAnalysis { * one). */ private readonly elementRoots = new Set(); private readonly bases = new Map(); + private readonly classDefs = new Map(); + private readonly subclasses = new Map(); private readonly globals = new Map(); /** The slot graph (see "Arrays fields hold"). */ private readonly edges = new Map>(); @@ -557,7 +549,15 @@ class SourceAnalysis { if (!list) this.methodImpls.set(m, (list = [])); list.push(`%${cls.name}.${m}`); } - for (const cls of mod.classes ?? []) this.bases.set(cls.name, cls.base); + for (const cls of mod.classes ?? []) { + this.bases.set(cls.name, cls.base); + this.classDefs.set(cls.name, cls); + if (cls.base !== undefined) { + const list = this.subclasses.get(cls.base); + if (list === undefined) this.subclasses.set(cls.base, [cls.name]); + else list.push(cls.name); + } + } for (const g of mod.globals ?? []) this.globals.set(g.id, g); for (const cls of plan.classes.values()) this.elementRoots.add(this.rootOf(cls.elemClass)); } @@ -1219,7 +1219,12 @@ class SourceAnalysis { /** The field read whose array values of type `t` in `slots`, or the * arrays, Maps, Sets and records inside them, may hold. */ - private originWithin(slots: readonly string[], t: IrType, depth = 0): SrcLoc | undefined { + private originWithin( + slots: readonly string[], + t: IrType, + objects: boolean, + depth = 0, + ): SrcLoc | undefined { if (depth > 4) return undefined; if (depth > 0 && this.holdsFieldArray(t)) for (const slot of slots) { @@ -1230,29 +1235,67 @@ class SourceAnalysis { switch (t.kind) { case "array": case "set": - return this.originWithin(elements, t.elem, depth + 1); + return this.originWithin(elements, t.elem, objects, depth + 1); case "map": return ( - this.originWithin(elements, t.key, depth + 1) ?? - this.originWithin(elements, t.value, depth + 1) + this.originWithin(elements, t.key, objects, depth + 1) ?? + this.originWithin(elements, t.value, objects, depth + 1) ); case "union": for (const arm of this.arms(t)) { - const origin = this.originWithin(slots, arm, depth + 1); + const origin = this.originWithin(slots, arm, objects, depth + 1); if (origin !== undefined) return origin; } return undefined; case "record": for (const f of (this.mod.records ?? []).find((r) => r.id === t.shapeId)?.fields ?? []) { - const origin = this.originWithin([`s:${t.shapeId}\0${f.name}`], f.type, depth + 1); + const origin = this.originWithin( + [`s:${t.shapeId}\0${f.name}`], + f.type, + objects, + depth + 1, + ); if (origin !== undefined) return origin; } return undefined; + // Code that reads an instance's fields by name (a dynamic view) can + // reach the arrays they hold. Other code reads them through field + // reads, which the graph follows. + case "object": + return objects ? this.objectOrigin(t.className, depth) : undefined; default: return undefined; } } + /** `originWithin` through objects for an instance of `className` or of a + * subclass: the arrays its fields may hold. An inline field is left out: its reads are + * followed where they are. By `className\0depth`. */ + private readonly objectOrigins = new Map(); + + private objectOrigin(className: string, depth: number): SrcLoc | undefined { + const key = `${className}\0${depth}`; + const known = this.objectOrigins.get(key); + if (known !== undefined) return known ?? undefined; + // A cycle through fields finds nothing new. + this.objectOrigins.set(key, null); + let found: SrcLoc | undefined; + const visit = (name: string): void => { + const cls = this.classDefs.get(name); + if (cls === undefined) return; + const inline = this.plan.classes.get(name)?.field; + for (const f of cls.fields) { + if (found !== undefined) return; + if (f.name === inline) continue; + found = this.originWithin([this.fieldSlot(name, f.name)], f.type, true, depth + 1); + } + for (const sub of this.subclasses.get(name) ?? []) if (found === undefined) visit(sub); + }; + visit(className); + this.objectOrigins.set(key, found ?? null); + return found; + } + /** The slots of the parameters of `fn` that only ever hold the caller's * arguments (never assigned or captured), by index. */ private ownParams(fn: IrFunction): Map { @@ -1505,11 +1548,19 @@ class SourceAnalysis { }; // `deep`: arrays inside the value escape too (dynamic and library code // reads them other than through the reads this pass follows). - const escape = (value: IrExpr, loc: SrcLoc, what: string, deep = true): void => { + // `objects`: through the fields of class instances too (a dynamic view + // reads them by name). + const escape = ( + value: IrExpr, + loc: SrcLoc, + what: string, + deep = true, + objects = false, + ): void => { if (check === null) return; const origin = this.origin(fn, value) ?? - (deep ? this.originWithin(this.slotsOf(fn, value), value.type) : undefined); + (deep ? this.originWithin(this.slotsOf(fn, value), value.type, objects) : undefined); if (origin !== undefined) check({ loc, @@ -1648,10 +1699,10 @@ class SourceAnalysis { } case "dynFrom": case "caughtToDyn": - if (!readSinks.has(e)) escape(e.value, e.loc, "a dynamic value"); + if (!readSinks.has(e)) escape(e.value, e.loc, "a dynamic value", true, true); break; case "callValue": - for (const a of e.args) escape(a, e.loc, "a call through a function value", false); + for (const a of e.args) escape(a, e.loc, "a call through a function value"); break; case "intrinsic": // Printing reads its arguments; publishing freezes them. diff --git a/packages/compiler/src/ir/inline-arrays.ts b/packages/compiler/src/ir/inline-arrays.ts index 50e5a00c2..7522a33da 100644 --- a/packages/compiler/src/ir/inline-arrays.ts +++ b/packages/compiler/src/ir/inline-arrays.ts @@ -60,6 +60,8 @@ export interface InlineArrayRejection { readonly declaringClass: string; readonly field: string; readonly reason: string; + /** The use that rules the layout out, when it is one site. */ + readonly loc?: SrcLoc; } export interface InlineArrayPlan { @@ -97,8 +99,8 @@ export function planInlineArrays( const at = entry.indexOf("\0"); const declaringClass = entry.slice(0, at); const field = entry.slice(at + 1); - const reject = (reason: string): void => { - rejected.push({ declaringClass, field, reason }); + const reject = (reason: string, loc?: SrcLoc): void => { + rejected.push({ declaringClass, field, reason, ...(loc !== undefined ? { loc } : {}) }); }; const decl = classes.get(declaringClass); if (decl === undefined) { @@ -147,9 +149,9 @@ export function planInlineArrays( } lengthParams.set(name, param); } - reason ??= otherUses(mod, members, field, declaringClass); - if (reason !== null) { - reject(reason); + const other = reason === null ? otherUses(mod, members, field, declaringClass) : null; + if (reason !== null || other !== null) { + reject(reason ?? other!.reason, other?.loc); continue; } for (const name of subtree) @@ -258,17 +260,13 @@ function otherUses( members: ReadonlySet, field: string, declaringClass: string, -): string | null { - const reaches = (type: IrType): boolean => { - let found = false; - const visit = (t: IrType): boolean => { - if (t.kind === "object" && members.has(t.className)) found = true; - return !found && everyTypeChild(t, visit); - }; - visit(type); - return found; - }; +): { reason: string; loc?: SrcLoc } | null { + const reaches = holdersOf(mod, members, declaringClass); let reason: string | null = null; + let loc: SrcLoc | undefined; + // Dynamic values that printing, JSON.stringify without a replacer and + // publishing only read, and the options a worker copies its data from. + const readOnly = new Set(); for (const fn of mod.functions) { const isDeclaringCtor = fn.name === `%${declaringClass}.constructor`; everyStmtList(fn.body, { @@ -276,9 +274,21 @@ function otherUses( if (reason !== null) return false; if (e.kind === "classRef" && members.has(e.className)) reason = `${e.className} is used as a value`; - else if ((e.kind === "dynFrom" || e.kind === "caughtToDyn") && reaches(e.value.type)) - reason = "an instance becomes a dynamic value"; - else if (e.kind === "fieldIncDec" && e.field === field && members.has(e.className)) + else if ( + (e.kind === "libCall" && (readsArguments(e) || e.fn === "worker.new")) || + (e.kind === "intrinsic" && + (e.name.startsWith("console.") || e.name === "threads.publish")) + ) + for (const a of e.args) readOnly.add(a); + else if ( + (e.kind === "dynFrom" || e.kind === "caughtToDyn") && + !readOnly.has(e) && + reaches(e.value.type) + ) { + reason = + "an instance, or a value that can hold one, becomes a dynamic value (a dynamic view reads the field as a new array each time)"; + loc = e.loc; + } else if (e.kind === "fieldIncDec" && e.field === field && members.has(e.className)) reason = "the field is written outside the constructor"; else if ( e.kind === "bin" && @@ -301,11 +311,84 @@ function otherUses( return reason === null; }, }); - if (reason !== null) return reason; + if (reason !== null) return loc === undefined ? { reason } : { reason, loc }; } return null; } +/** Whether values of a type may be, or hold, an instance of `members` + * (the subtree of `declaringClass`): a value typed as a member or as a base + * of the declaring class, or a class instance, record, union, array, Map or + * Set whose fields or elements may hold one. A dynamic view of any of them + * reads the instance's fields by name. */ +function holdersOf( + mod: IrModule, + members: ReadonlySet, + declaringClass: string, +): (type: IrType) => boolean { + const classes = new Map((mod.classes ?? []).map((c) => [c.name, c])); + const arms = new Map((mod.unions ?? []).map((u) => [u.id, u.arms])); + const records = new Map((mod.records ?? []).map((r) => [r.id, r.fields])); + const holderClasses = new Set(members); + const holderShapes = new Set(); + const addWithBases = (name: string): boolean => { + let added = false; + for (let at: string | undefined = name; at !== undefined; at = classes.get(at)?.base) + if (!holderClasses.has(at)) { + holderClasses.add(at); + added = true; + } + return added; + }; + addWithBases(declaringClass); + const holds = (type: IrType): boolean => { + const seen = new Set(); + let found = false; + const visit = (t: IrType): boolean => { + if (t.kind === "object") found = holderClasses.has(t.className); + else if (t.kind === "record") found = holderShapes.has(t.shapeId); + else if (t.kind === "union" && !seen.has(t.unionId)) { + seen.add(t.unionId); + for (const arm of arms.get(t.unionId) ?? []) if (!visit(arm)) break; + } + return !found && everyTypeChild(t, visit); + }; + visit(type); + return found; + }; + for (let changed = true; changed;) { + changed = false; + for (const cls of mod.classes ?? []) + if (!holderClasses.has(cls.name) && cls.fields.some((f) => holds(f.type))) + changed = addWithBases(cls.name) || changed; + for (const [shapeId, fields] of records) + if (!holderShapes.has(shapeId) && fields.some((f) => holds(f.type))) { + holderShapes.add(shapeId); + changed = true; + } + } + return holds; +} + +/** Library calls that only read their arguments: printing, inspection, + * and JSON.stringify without a replacer function. */ +export function readsArguments(e: IrExpr & { kind: "libCall" }): boolean { + if (/^(console|insp|inspect)\./.test(e.fn)) return true; + switch (e.fn) { + case "array.hasProperties": + case "array.hasNamedKey": + case "array.namedKeys": + case "json.stringifyValue": + return true; + case "json.stringifyReplacer": { + const replacer = e.args[1]; + return replacer?.kind === "dynFrom" && replacer.value.kind === "unitLit"; + } + default: + return false; + } +} + function readsField(e: IrExpr, members: ReadonlySet, field: string): boolean { return e.kind === "fieldGet" && e.field === field && members.has(e.className); } diff --git a/packages/compiler/src/ir/ir.ts b/packages/compiler/src/ir/ir.ts index 8f74d7493..ce220275b 100644 --- a/packages/compiler/src/ir/ir.ts +++ b/packages/compiler/src/ir/ir.ts @@ -618,6 +618,14 @@ export const CHILDWRITER_T: IrType = { kind: "childWriter" }; export const PROCSTREAM_T: IrType = { kind: "procStream" }; export const VOID: IrType = { kind: "void" }; export const DYN: IrType = { kind: "dyn" }; +/** The `foreign` type (scope-B NI-4): a value owned by a Node island, + * typed by the island package's declarations. Objects stay in V8 and cross + * as handles (identity is the V8 object's); member operations compile from + * the declarations into `nodeIsland.*` bridge calls. NI-4 represents it + * exactly as `dyn` (a handle box with the island tag, primitives as plain + * dyn values), so foreign -> any is the identity and the generic dyn paths + * (equality, inspection, any slots) apply unchanged. */ +export const FOREIGN: IrType = DYN; export const JSVAL: IrType = { kind: "jsval" }; export const CAUGHT: IrType = { kind: "caught" }; export const UNDEFINED_T: IrType = { kind: "undefinedT" }; diff --git a/packages/compiler/src/ir/node-index.ts b/packages/compiler/src/ir/node-index.ts index be1fa506a..2775ebb0c 100644 --- a/packages/compiler/src/ir/node-index.ts +++ b/packages/compiler/src/ir/node-index.ts @@ -196,6 +196,13 @@ export class IrNodeIndex { return scan; } + /** The number of nodes recorded for `fn`, or undefined when `fn` is not + * recorded. */ + nodeCount(fn: IrFunction): number | undefined { + const f = this.functionIndex.get(fn); + return f === undefined ? undefined : this.starts[f + 1]! - this.starts[f]!; + } + /** Calls `note` with each recorded node of `fn` whose kind `scan` names, * in walk order, until it answers false. False (nothing called) when * `fn` is not recorded. */ diff --git a/packages/compiler/src/native/cli.ts b/packages/compiler/src/native/cli.ts index 961fd02a9..bc33e683a 100644 --- a/packages/compiler/src/native/cli.ts +++ b/packages/compiler/src/native/cli.ts @@ -36,6 +36,7 @@ declare function compilerNativeCollectCycles(): void; declare function compilerNativeFork(): number; declare function compilerNativeExit(code: number): void; declare function compilerNativeWait(pid: number): number; +declare function compilerNativeExited(): number; declare function compilerNativeLockOpen(path: string): boolean; declare function compilerNativeLock(): void; declare function compilerNativeUnlock(): void; @@ -73,6 +74,7 @@ async function main(): Promise { fork: () => compilerNativeFork(), exit: (code) => compilerNativeExit(code), wait: (pid) => compilerNativeWait(pid), + exited: () => compilerNativeExited(), openLock: (path) => compilerNativeLockOpen(path), lock: () => compilerNativeLock(), unlock: () => compilerNativeUnlock(), diff --git a/packages/compiler/src/native/prepare.ts b/packages/compiler/src/native/prepare.ts index db40c220f..703c00559 100644 --- a/packages/compiler/src/native/prepare.ts +++ b/packages/compiler/src/native/prepare.ts @@ -603,5 +603,6 @@ function emitForkPrimitives(): ForkPrimitives | undefined { fork: () => primitives.fork(), exit: (code) => primitives.exit(code), wait: (pid) => primitives.wait(pid), + exited: () => primitives.exited(), }; } diff --git a/packages/compiler/src/node-islands/boundary.ts b/packages/compiler/src/node-islands/boundary.ts new file mode 100644 index 000000000..b7ca82354 --- /dev/null +++ b/packages/compiler/src/node-islands/boundary.ts @@ -0,0 +1,157 @@ +/* The Node island boundary (DESIGN.md §2.4–§2.5, scope-B NI-4): which + * values of a compiled program are island values, and what crosses by + * copy. Rule (decision 2): a value crosses by reference unless copying it + * is unobservable. Primitives and strings are copied; every object stays a + * handle whose identity is the V8 object's. + * + * Island values are typed by the island package's own declarations. A + * checker type is FOREIGN when it is an object type declared in an island + * package (its .d.ts, or its @types twin): a class, interface, type alias + * or type literal there, the type of an exported function or class value, + * or an island module's namespace. Values of foreign types are held as + * island handles (the `foreign` IR type, represented as a dyn value), and + * member operations on them compile from the declarations into bridge + * calls. A union is foreign when every arm that is not null or undefined + * is foreign (the dyn representation holds null and undefined). */ +import * as ts from "../frontend/ts7/adapter.js"; +import { packageDirOf } from "./origin.js"; + +/** One package the build runs in a Node island. */ +export interface IslandPackage { + name: string; + version: string | null; + /** The package directory (…/node_modules/). */ + dir: string; + /** The island class (runtime, object-model, gap, packaging) and the first + * reason, for the SC8001 note. */ + class: string; + reason: string; +} + +/** The program's island packages, by package name. */ +export interface NodeIslandSet { + packages: ReadonlyMap; +} + +/** The island package that owns a declaration or JavaScript file, or null. + * `@types/` declarations type the package ``. */ +export function islandPackageOfFile(islands: NodeIslandSet, file: string): IslandPackage | null { + const at = packageDirOf(file); + if (at === null) return null; + let name = at.name; + if (name.startsWith("@types/")) { + const bare = name.slice("@types/".length); + name = bare.includes("__") ? `@${bare.replace("__", "/")}` : bare; + } + return islands.packages.get(name) ?? null; +} + +function declaredInIsland(islands: NodeIslandSet, symbol: ts.Symbol | null | undefined): boolean { + if (!symbol) return false; + const decls = symbol.declarations; + if (decls.length === 0) return false; + return decls.every( + (d) => islandPackageOfFile(islands, ts.getSourceFileOfNode(d).fileName) !== null, + ); +} + +const NULLISH = ts.TypeFlags.Null | ts.TypeFlags.Undefined | ts.TypeFlags.Void; + +/** Whether values of `type` are island handles (see the header). An + * anonymous object type every property of which the island declares (the + * namespace TypeScript synthesizes for `import()` of a CommonJS island + * module) is foreign too. */ +export function isForeignType( + islands: NodeIslandSet, + type: ts.Type, + checker?: ts.TypeChecker, +): boolean { + if (type.isUnionType()) { + const arms = ts.constituentTypes(type).filter((t) => (t.flags & NULLISH) === 0); + return arms.length > 0 && arms.every((t) => isForeignType(islands, t, checker)); + } + // An intersection the island declares (execa's `ChildProcess & + // ExecaChildPromise & Promise<…>`), or one with an island-declared arm, + // is made by island code: its values are island objects. + if ((type.flags & ts.TypeFlags.Intersection) !== 0) { + if (declaredInIsland(islands, type.getAliasSymbol())) return true; + return ts.constituentTypes(type).some((t) => isForeignType(islands, t, checker)); + } + if ((type.flags & ts.TypeFlags.Object) === 0) return false; + if ( + declaredInIsland(islands, type.getAliasSymbol()) || + declaredInIsland(islands, type.getSymbol()) + ) + return true; + if (checker === undefined || type.getAliasSymbol()) return false; + const symbol = type.getSymbol(); + if (symbol && symbol.declarations.length > 0) return false; + // Synthetic members (the interop `default`) have no declarations. + let declared = 0; + for (const p of checker.getPropertiesOfType(type)) { + if (p.declarations.length === 0) continue; + if (!declaredInIsland(islands, p)) return false; + declared++; + } + return declared > 0; +} + +/** How a value of a checker type crosses INTO an island as an argument or + * an assigned value: copied (primitives, strings, null/undefined), passed + * as its handle (foreign), passed as a dyn value the runtime converts + * (any/unknown: a handle crosses as its object, a primitive is copied, a + * fresh literal is copied), or not representable yet. */ +export type BoundaryCrossing = "copy" | "handle" | "dyn" | "refused"; + +const COPIED = + ts.TypeFlags.NumberLike | + ts.TypeFlags.StringLike | + ts.TypeFlags.BooleanLike | + ts.TypeFlags.BigIntLike | + ts.TypeFlags.Null | + ts.TypeFlags.Undefined | + ts.TypeFlags.Void; + +export function crossingOf( + islands: NodeIslandSet, + type: ts.Type, + checker?: ts.TypeChecker, +): BoundaryCrossing { + if (type.isUnionType()) { + const arms = ts.constituentTypes(type); + if (arms.every((t) => (t.flags & COPIED) !== 0)) return "copy"; + if (isForeignType(islands, type, checker)) return "handle"; + return "refused"; + } + if ((type.flags & COPIED) !== 0) return "copy"; + if ((type.flags & (ts.TypeFlags.Any | ts.TypeFlags.Unknown)) !== 0) return "dyn"; + if (isForeignType(islands, type, checker)) return "handle"; + return "refused"; +} + +/* The island set of the lowering in progress. lowerToIr sets it from its + * options for the duration of one synchronous lowering; every Lowerer it + * creates reads it at construction. Null in programs without islands, so + * their lowering is unchanged. */ +let active: NodeIslandSet | null = null; + +export function setActiveNodeIslands(islands: NodeIslandSet | null): void { + active = islands !== null && islands.packages.size > 0 ? islands : null; +} + +export function activeNodeIslands(): NodeIslandSet | null { + return active; +} + +/* The island package names of the frontend load in progress (runFrontend + * sets them for one synchronous load): preflight lets their imports + * through, typed by their declarations. */ +let frontendIslands: ReadonlySet = new Set(); + +export function setFrontendIslands(names: ReadonlySet): void { + frontendIslands = names; +} + +export function isFrontendIsland(pkg: string): boolean { + return frontendIslands.size > 0 && frontendIslands.has(pkg); +} diff --git a/packages/compiler/src/node-islands/classify.ts b/packages/compiler/src/node-islands/classify.ts index c41b9437c..b08ac63aa 100644 --- a/packages/compiler/src/node-islands/classify.ts +++ b/packages/compiler/src/node-islands/classify.ts @@ -25,7 +25,7 @@ const MESSAGE_REASONS: { code: string; pattern: RegExp; reason: string }[] = [ pattern: /createRequire's require with this argument shape/, reason: "dynamic-require", }, - { code: "SC2020", pattern: /native addon/, reason: "native-addon" }, + { code: "SC2020", pattern: /^native addon /, reason: "native-addon" }, { code: "SC2020", pattern: /through require\.cache|require\.extensions|Module\._|module\.register/, diff --git a/packages/compiler/src/node-islands/libnode-pack.test.ts b/packages/compiler/src/node-islands/libnode-pack.test.ts new file mode 100644 index 000000000..06ddcfabd --- /dev/null +++ b/packages/compiler/src/node-islands/libnode-pack.test.ts @@ -0,0 +1,338 @@ +import { createHash } from "node:crypto"; +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + readdirSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { afterEach, describe, expect, test } from "vitest"; +import { compilerReleaseVersion } from "../library/sidecar.js"; +import { NODE_BRIDGE_ABI, libnodeRecord } from "./libnode.js"; +import { + LIBNODE_PACK_SCHEMA, + LibnodePackError, + findInstalledLibnodePack, + hostLibnodeTarget, + libnodePackPackage, + parseLibnodeSetting, + readLibnodePack, + writeLibnodeNotices, +} from "./libnode-pack.js"; + +const repo = fileURLToPath(new URL("../../../..", import.meta.url)); +const dirs: string[] = []; +afterEach(() => { + for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true }); +}); + +function temp(): string { + const dir = mkdtempSync(join(tmpdir(), "scriptc-libnode-pack-")); + dirs.push(dir); + return dir; +} + +function entry(bytes: string) { + return { + size: Buffer.byteLength(bytes), + sha256: createHash("sha256").update(bytes).digest("hex"), + }; +} + +/** A published-style pack with stand-in libraries; `edit` adjusts the + * manifest build-pack.mjs would write. */ +function fakePack( + dir: string, + edit: (manifest: Record) => void = () => {}, + name = "@scriptc/libnode-linux-x64-gnu", +): string { + mkdirSync(dir, { recursive: true }); + const libnode = "stand-in libnode"; + const bridge = "stand-in bridge"; + writeFileSync(join(dir, "libnode.so.137"), libnode); + writeFileSync(join(dir, "libscr_node_bridge.so"), bridge); + writeFileSync(join(dir, "LICENSE"), "Node.js is licensed for use as follows:\n\nMIT\n"); + writeFileSync( + join(dir, "package.json"), + JSON.stringify({ name, version: compilerReleaseVersion() }), + ); + const manifest: Record = { + schema: LIBNODE_PACK_SCHEMA, + nodeVersion: "24.22.0", + moduleVersion: 137, + bridgeAbi: NODE_BRIDGE_ABI, + platform: "linux", + arch: "x64", + libnode: "libnode.so.137", + bridge: "libscr_node_bridge.so", + package: name, + version: compilerReleaseVersion(), + target: "linux-x64-gnu", + license: "LICENSE", + files: { "libnode.so.137": entry(libnode), "libscr_node_bridge.so": entry(bridge) }, + }; + edit(manifest); + writeFileSync(join(dir, "manifest.json"), JSON.stringify(manifest)); + return dir; +} + +function thrown(run: () => unknown): LibnodePackError { + try { + run(); + } catch (error) { + expect(error).toBeInstanceOf(LibnodePackError); + return error as LibnodePackError; + } + throw new Error("expected a LibnodePackError"); +} + +describe("the pack packages", () => { + test("every libnode platform package is the pack of one native target", () => { + const packages = readdirSync(join(repo, "packages")) + .filter((name) => name.startsWith("libnode-") && name !== "libnode-common") + .map( + (name) => + JSON.parse(readFileSync(join(repo, "packages", name, "package.json"), "utf8")).name, + ) + .sort(); + const targets = ["linux-x64-gnu", "linux-arm64-gnu", "macos-arm64", "macos-x64"]; + expect(targets.map((target) => libnodePackPackage(target)).sort()).toEqual(packages); + expect(libnodePackPackage("linux-x64-musl")).toBeNull(); + expect(libnodePackPackage("windows-x64-msvc")).toBeNull(); + expect(libnodePackPackage("wasm32-wasi")).toBeNull(); + }); + + test("host targets map to native target names", () => { + expect(hostLibnodeTarget("linux", "x64")).toBe("linux-x64-gnu"); + expect(hostLibnodeTarget("linux", "arm64")).toBe("linux-arm64-gnu"); + expect(hostLibnodeTarget("darwin", "arm64")).toBe("macos-arm64"); + expect(hostLibnodeTarget("darwin", "x64")).toBe("macos-x64"); + }); +}); + +describe("parseLibnodeSetting", () => { + test("accepts embedded, beside and absolute directories", () => { + expect(parseLibnodeSetting("embedded")).toEqual({ kind: "embedded" }); + expect(parseLibnodeSetting("beside")).toEqual({ kind: "beside" }); + expect(parseLibnodeSetting("/opt/scriptc/libnode-24.22.0")).toEqual({ + kind: "path", + path: "/opt/scriptc/libnode-24.22.0", + }); + }); + + test("rejects relative paths and other values", () => { + expect(parseLibnodeSetting("libnode")).toBe( + 'must be "embedded", "beside" or an absolute directory, not the relative path "libnode"', + ); + expect(parseLibnodeSetting(true)).toBe('must be "embedded", "beside" or an absolute directory'); + }); +}); + +describe("finding the installed pack", () => { + test("in the nearest node_modules above the project", () => { + const root = temp(); + const pack = fakePack(join(root, "node_modules/@scriptc/libnode-linux-x64-gnu")); + const project = join(root, "apps/web/src"); + mkdirSync(project, { recursive: true }); + expect( + findInstalledLibnodePack({ + targetName: "linux-x64-gnu", + projectDir: project, + resolveInstalled: () => null, + }), + ).toBe(pack); + }); + + test("next to the compiler when the project has none", () => { + const pack = fakePack(join(temp(), "libnode-darwin-arm64")); + expect( + findInstalledLibnodePack({ + targetName: "macos-arm64", + projectDir: temp(), + resolveInstalled: (name) => (name === "@scriptc/libnode-darwin-arm64" ? pack : null), + }), + ).toBe(pack); + expect( + findInstalledLibnodePack({ + targetName: "macos-arm64", + projectDir: temp(), + resolveInstalled: () => null, + }), + ).toBeNull(); + }); + + test("targets without packs have none", () => { + expect(findInstalledLibnodePack({ targetName: "linux-x64-musl", projectDir: temp() })).toBe( + null, + ); + }); + + test("libnodeRecord uses the installed pack when nothing names one", () => { + const root = temp(); + const name = libnodePackPackage(hostLibnodeTarget()); + if (name === null) return; + const pack = fakePack(join(root, "node_modules", name), () => {}, name); + expect(libnodeRecord(undefined, join(root, "app"), {}, root)).toEqual({ + libnode: pack, + nodeVersion: "24.22.0", + }); + // SCRIPTC_LIBNODE still wins. + const other = fakePack(join(temp(), "other"), (m) => { + m["nodeVersion"] = "24.13.1"; + }); + expect(libnodeRecord(undefined, join(root, "app"), { SCRIPTC_LIBNODE: other }, root)).toEqual({ + libnode: other, + nodeVersion: "24.13.1", + }); + }); +}); + +test("an installed pack of another scriptc version is refused", () => { + const root = temp(); + const name = libnodePackPackage(hostLibnodeTarget()); + if (name === null) return; + const pack = fakePack( + join(root, "node_modules", name), + (m) => { + m["version"] = "0.0.1"; + }, + name, + ); + expect(libnodeRecord(undefined, join(root, "app"), {}, root)).toBe( + `the installed libnode pack ${name} at ${pack} is version 0.0.1; this scriptc is ${compilerReleaseVersion()}: install ${name}@${compilerReleaseVersion()}`, + ); +}); + +test("an installed pack whose files do not match their digests is refused", () => { + const root = temp(); + const name = libnodePackPackage(hostLibnodeTarget()); + if (name === null) return; + const pack = fakePack(join(root, "node_modules", name), () => {}, name); + // Same size, other bytes. + writeFileSync(join(pack, "libnode.so.137"), "stand-in libnodX"); + expect(libnodeRecord(undefined, join(root, "app"), {}, root)).toBe( + `libnode.so.137 in the libnode pack at ${pack} does not match its manifest`, + ); +}); + +describe("readLibnodePack", () => { + test("reads a pack in build-pack.mjs's format and checks its package", () => { + const pack = fakePack(join(temp(), "pack")); + expect( + readLibnodePack(pack, { packageName: "@scriptc/libnode-linux-x64-gnu", verifyHashes: true }) + .manifest.nodeVersion, + ).toBe("24.22.0"); + expect( + thrown(() => readLibnodePack(pack, { packageName: "@scriptc/libnode-linux-arm64-gnu" })) + .message, + ).toBe( + `the libnode pack at ${pack} is @scriptc/libnode-linux-x64-gnu; this target needs @scriptc/libnode-linux-arm64-gnu`, + ); + }); + + test("refuses other manifest schemas and file names outside the pack", () => { + const schema = fakePack(join(temp(), "pack"), (m) => { + m["schema"] = "scriptc.libnode-pack.v0"; + }); + expect(thrown(() => readLibnodePack(schema)).kind).toBe("invalid"); + for (const name of [ + "lib/libnode.so.137", + "../libnode.so.137", + "..", + "/etc/libnode.so", + "a\\b", + "a\0b", + ]) { + const pack = fakePack(join(temp(), "pack"), (m) => { + m["libnode"] = name; + m["files"] = { ...(m["files"] as object), [name]: entry("stand-in libnode") }; + }); + expect(thrown(() => readLibnodePack(pack)).message).toBe( + `the libnode pack at ${pack} names a file outside the pack (${JSON.stringify(name)})`, + ); + } + }); + + test("checks sizes always and hashes on request", () => { + const pack = fakePack(join(temp(), "pack")); + writeFileSync(join(pack, "libnode.so.137"), "stand-in libnodE"); + expect(readLibnodePack(pack).root).toBe(pack); + expect(thrown(() => readLibnodePack(pack, { verifyHashes: true })).message).toBe( + `libnode.so.137 in the libnode pack at ${pack} does not match its manifest`, + ); + writeFileSync(join(pack, "libnode.so.137"), "short"); + expect(thrown(() => readLibnodePack(pack)).kind).toBe("invalid"); + }); + + test("a directory without a manifest is missing", () => { + const dir = temp(); + expect(thrown(() => readLibnodePack(dir)).message).toBe( + `no libnode pack at ${dir} (manifest.json is missing or unreadable)`, + ); + }); +}); + +test("the license notices carry Node's LICENSE", () => { + const pack = readLibnodePack(fakePack(join(temp(), "pack"))); + const out = join(temp(), "server"); + const path = writeLibnodeNotices(pack, out); + expect(path).toBe(`${out}.licenses.txt`); + expect(readFileSync(path, "utf8")).toBe( + [ + `server runs part of its code in Node.js 24.22.0, loaded from libnode.so.137 (@scriptc/libnode-linux-x64-gnu@${compilerReleaseVersion()}).`, + "Node.js and the components it bundles are licensed as follows. The scriptc bridge,", + "libscr_node_bridge.so, is licensed under Apache-2.0.", + "", + "Node.js is licensed for use as follows:", + "", + "MIT", + "", + ].join("\n"), + ); +}); + +test("beside refuses an installed pack whose manifest names a file outside it", () => { + const root = temp(); + const name = libnodePackPackage(hostLibnodeTarget()); + if (name === null) return; + const pack = fakePack( + join(root, "node_modules", name), + (m) => { + m["files"] = { ...(m["files"] as object), "../escaped": entry("payload") }; + }, + name, + ); + // The pack's "../escaped" and the file it would replace next to the output. + writeFileSync(join(pack, "..", "escaped"), "payload"); + writeFileSync(join(root, "escaped"), "victim"); + expect(libnodeRecord("beside", join(root, "app"), {}, root)).toBe( + `the libnode pack at ${pack} names a file outside the pack ("../escaped")`, + ); + expect(readFileSync(join(root, "escaped"), "utf8")).toBe("victim"); + expect(existsSync(join(root, "app.libnode"))).toBe(false); +}); + +test("readLibnodePack refuses any listed file or license outside the pack", () => { + for (const edit of [ + (m: Record) => { + m["files"] = { ...(m["files"] as object), "../escaped": entry("payload") }; + }, + (m: Record) => { + m["files"] = { ...(m["files"] as object), "/etc/passwd": entry("payload") }; + }, + (m: Record) => { + m["license"] = "../../secret"; + }, + (m: Record) => { + m["license"] = 7; + }, + ]) { + const pack = fakePack(join(temp(), "pack"), edit); + expect(thrown(() => readLibnodePack(pack)).kind).toBe("invalid"); + } +}); diff --git a/packages/compiler/src/node-islands/libnode-pack.ts b/packages/compiler/src/node-islands/libnode-pack.ts new file mode 100644 index 000000000..4de4a2af3 --- /dev/null +++ b/packages/compiler/src/node-islands/libnode-pack.ts @@ -0,0 +1,267 @@ +/* The installed libnode packs (scope-B design §2.11, §2.13, §2.16.3): + * `@scriptc/libnode-`, built only by the libnode-packs workflow + * from the pinned Node release (packages/libnode-common) and installed like + * the runtime packs. A scriptc build never compiles Node. + * + * libnode.ts decides which pack a build records: --libnode, else + * SCRIPTC_LIBNODE, else the installed pack this module finds: the nearest + * `node_modules/@scriptc/libnode-` walking up from the project + * directory, else the one next to the compiler. The manifest format is the + * one packages/libnode-common/build-pack.mjs writes (schema, nodeVersion, + * moduleVersion, bridgeAbi, platform, arch, libnode, bridge, files); the + * published packs add package, version, target, license and build inputs. + * + * Also here: the `nodeIslands.libnode` setting ("embedded" | "beside" | + * ) for the project configuration, and the license + * notices a program that redistributes Node.js carries. */ +import { createHash } from "node:crypto"; +import { existsSync, readFileSync, statSync, writeFileSync } from "node:fs"; +import { createRequire } from "node:module"; +import { basename, dirname, isAbsolute, join, resolve } from "node:path"; + +export const LIBNODE_PACK_SCHEMA = "scriptc.libnode-pack.v1"; + +/** The pack package for each native target that can host Node islands. */ +const PACK_PACKAGES: Readonly> = { + "linux-x64-gnu": "@scriptc/libnode-linux-x64-gnu", + "linux-arm64-gnu": "@scriptc/libnode-linux-arm64-gnu", + "macos-arm64": "@scriptc/libnode-darwin-arm64", + "macos-x64": "@scriptc/libnode-darwin-x64", +}; + +export type LibnodePlacement = + | { kind: "embedded" } + | { kind: "beside" } + | { kind: "path"; path: string }; + +export interface LibnodeFile { + size: number; + sha256: string; +} + +export interface LibnodePackManifest { + schema: string; + nodeVersion: string; + moduleVersion: number; + bridgeAbi: number; + /** process.platform and process.arch of the pack. */ + platform: string; + arch: string; + /** File names inside the pack directory. */ + libnode: string; + bridge: string; + files: Record; + /** Published packs only. */ + package?: string; + version?: string; + target?: string; + license?: string; +} + +export interface LibnodePack { + root: string; + manifest: LibnodePackManifest; +} + +export class LibnodePackError extends Error { + readonly kind: "missing" | "invalid" | "unsupported"; + + constructor(message: string, kind: "missing" | "invalid" | "unsupported") { + super(message); + this.name = "LibnodePackError"; + this.kind = kind; + } +} + +/** The libnode pack package for a native target, or null when Node islands + * have no pack there (musl, Windows, WASI and mobile targets). */ +export function libnodePackPackage(targetName: string): string | null { + return PACK_PACKAGES[targetName] ?? null; +} + +/** The native target name of the host, as libnodePackPackage takes it. */ +export function hostLibnodeTarget( + platform: string = process.platform, + arch: string = process.arch, +): string { + return platform === "darwin" ? `macos-${arch}` : `${platform}-${arch}-gnu`; +} + +/** Validates a `nodeIslands.libnode` value; a string result is the reason + * it is invalid. */ +export function parseLibnodeSetting(value: unknown): LibnodePlacement | string { + if (value === "embedded") return { kind: "embedded" }; + if (value === "beside") return { kind: "beside" }; + if (typeof value === "string" && isAbsolute(value)) return { kind: "path", path: value }; + if (typeof value === "string") { + return `must be "embedded", "beside" or an absolute directory, not the relative path "${value}"`; + } + return `must be "embedded", "beside" or an absolute directory`; +} + +/** Whether `name` names a file directly in the pack directory. */ +function packFileName(name: string): boolean { + return name !== "" && name !== "." && name !== ".." && !/[/\\\0]/.test(name) && !isAbsolute(name); +} + +function isRecord(value: unknown): value is Record { + return value !== null && typeof value === "object" && !Array.isArray(value); +} + +function isFileEntry(value: unknown): value is LibnodeFile { + return ( + isRecord(value) && + typeof value["size"] === "number" && + typeof value["sha256"] === "string" && + /^[0-9a-f]{64}$/.test(value["sha256"]) + ); +} + +/** Parses a pack's manifest.json; throws LibnodePackError when it is not a + * pack manifest. Bridge ABI and Node version policy stay in libnode.ts. */ +export function parseLibnodePackManifest(json: unknown, where: string): LibnodePackManifest { + const invalid = (detail: string): never => { + throw new LibnodePackError(`the libnode pack at ${where} ${detail}`, "invalid"); + }; + if (!isRecord(json)) return invalid("has a malformed manifest.json"); + if (json["schema"] !== LIBNODE_PACK_SCHEMA) { + return invalid( + `has manifest schema ${String(json["schema"])}, expected ${LIBNODE_PACK_SCHEMA}`, + ); + } + for (const field of ["nodeVersion", "platform", "arch", "libnode", "bridge"]) { + if (typeof json[field] !== "string" || json[field] === "") { + return invalid(`has no ${field} in manifest.json`); + } + } + if (typeof json["moduleVersion"] !== "number" || typeof json["bridgeAbi"] !== "number") { + return invalid("has no moduleVersion or bridgeAbi in manifest.json"); + } + const files = json["files"]; + if (!isRecord(files)) return invalid("lists no files in manifest.json"); + // Every name the pack lists is a file directly in the pack directory. + const license = json["license"]; + if (license !== undefined && typeof license !== "string") { + return invalid("has a malformed license in manifest.json"); + } + for (const name of [...Object.keys(files), json["license"] ?? "LICENSE"] as string[]) { + if (!packFileName(name)) { + return invalid(`names a file outside the pack (${JSON.stringify(name)})`); + } + } + for (const name of [json["libnode"], json["bridge"]] as string[]) { + if (!packFileName(name)) { + return invalid(`names a file outside the pack (${JSON.stringify(name)})`); + } + if (!isFileEntry(files[name])) return invalid(`lists no size and SHA-256 for ${name}`); + } + return json as unknown as LibnodePackManifest; +} + +/** Reads and checks the pack in `root`: its manifest, its package when + * `packageName` is given, and that each library exists with the recorded + * size (and SHA-256 when `verifyHashes`). */ +export function readLibnodePack( + root: string, + options: { packageName?: string | undefined; verifyHashes?: boolean | undefined } = {}, +): LibnodePack { + let json: unknown; + try { + json = JSON.parse(readFileSync(join(root, "manifest.json"), "utf8")); + } catch { + throw new LibnodePackError( + `no libnode pack at ${root} (manifest.json is missing or unreadable)`, + "missing", + ); + } + const manifest = parseLibnodePackManifest(json, root); + if (options.packageName !== undefined && manifest.package !== options.packageName) { + throw new LibnodePackError( + `the libnode pack at ${root} is ${manifest.package ?? "unnamed"}; this target needs ${options.packageName}`, + "invalid", + ); + } + for (const name of [manifest.libnode, manifest.bridge]) { + const path = join(root, name); + const entry = manifest.files[name]!; + let size: number; + try { + size = statSync(path).size; + } catch { + throw new LibnodePackError(`the libnode pack at ${root} is missing ${name}`, "invalid"); + } + const hashMismatch = + options.verifyHashes === true && + createHash("sha256").update(readFileSync(path)).digest("hex") !== entry.sha256; + if (size !== entry.size || hashMismatch) { + throw new LibnodePackError( + `${name} in the libnode pack at ${root} does not match its manifest`, + "invalid", + ); + } + } + return { root, manifest }; +} + +/** The nearest `node_modules/` directory with a built pack (a + * manifest.json) walking up from `from`. */ +function findInstalled(name: string, from: string): string | null { + let dir = resolve(from); + for (;;) { + const candidate = join(dir, "node_modules", name); + if (existsSync(join(candidate, "manifest.json"))) return candidate; + const parent = dirname(dir); + if (parent === dir) return null; + dir = parent; + } +} + +function resolveNextToCompiler(name: string): string | null { + try { + const dir = dirname(createRequire(import.meta.url).resolve(`${name}/package.json`)); + // An unbuilt workspace package is not a pack. + return existsSync(join(dir, "manifest.json")) ? dir : null; + } catch { + return null; + } +} + +/** The installed pack directory for `targetName`, or null when none is + * installed or the target has no packs. */ +export function findInstalledLibnodePack(options: { + targetName: string; + projectDir: string; + /** Test hook for the compiler-relative lookup. */ + resolveInstalled?: ((name: string) => string | null) | undefined; +}): string | null { + const packageName = libnodePackPackage(options.targetName); + if (packageName === null) return null; + return ( + findInstalled(packageName, options.projectDir) ?? + (options.resolveInstalled ?? resolveNextToCompiler)(packageName) + ); +} + +/** Writes the license notices a program that redistributes Node.js must + * carry, `.licenses.txt` (design §2.13), and returns its path. */ +export function writeLibnodeNotices(pack: LibnodePack, outPath: string): string { + const { manifest } = pack; + const nodeLicense = readFileSync(join(pack.root, manifest.license ?? "LICENSE"), "utf8"); + const origin = + manifest.package !== undefined && manifest.version !== undefined + ? ` (${manifest.package}@${manifest.version})` + : ""; + const path = `${outPath}.licenses.txt`; + writeFileSync( + path, + [ + `${basename(outPath)} runs part of its code in Node.js ${manifest.nodeVersion}, loaded from ${manifest.libnode}${origin}.`, + `Node.js and the components it bundles are licensed as follows. The scriptc bridge,`, + `${manifest.bridge}, is licensed under Apache-2.0.`, + "", + nodeLicense.trimEnd(), + "", + ].join("\n"), + ); + return path; +} diff --git a/packages/compiler/src/node-islands/libnode.test.ts b/packages/compiler/src/node-islands/libnode.test.ts index 94add4f1c..0f713ab72 100644 --- a/packages/compiler/src/node-islands/libnode.test.ts +++ b/packages/compiler/src/node-islands/libnode.test.ts @@ -11,7 +11,9 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { describe, expect, test } from "vitest"; import { USES_TIMERS_LIB_FNS } from "../backend/llvm/lib-shared.js"; -import { libnodeCacheKey, libnodeRecord, NODE_BRIDGE_ABI } from "./libnode.js"; +import { libnodeCacheKey, libnodeRecord, libnodeTarget, NODE_BRIDGE_ABI } from "./libnode.js"; +import { libnodePackPackage } from "./libnode-pack.js"; +import { findInstalledLibnodePack, hostLibnodeTarget } from "./libnode-pack.js"; function pack(nodeVersion: string, bridgeAbi = NODE_BRIDGE_ABI): string { const dir = mkdtempSync(join(tmpdir(), "scriptc-libnode-")); @@ -39,7 +41,21 @@ describe("the recorded libnode pack", () => { }); test("refuses a missing, too old or incompatible pack", () => { - expect(libnodeRecord(undefined, "/out/app", {})).toMatch(/no libnode pack is installed/); + // Unless a workspace build of the host's @scriptc/libnode- + // resolves next to the compiler, which libnodeRecord then uses. + const installed = findInstalledLibnodePack({ + targetName: hostLibnodeTarget(), + projectDir: "/nonexistent", + }); + if (installed === null) { + expect(libnodeRecord(undefined, "/out/app", {}, "/nonexistent")).toMatch( + /no libnode pack is installed/, + ); + } else { + expect(libnodeRecord(undefined, "/out/app", {}, "/nonexistent")).toMatchObject({ + libnode: installed, + }); + } expect(libnodeRecord("/nonexistent", "/out/app", {})).toBe( "no libnode pack at /nonexistent (manifest.json is missing or unreadable)", ); @@ -143,9 +159,36 @@ describe("the recorded libnode pack", () => { }); }); -describe("the libnode pack in an executable's cache key", () => { - test("is absent without --libnode or SCRIPTC_LIBNODE", () => { - expect(libnodeCacheKey(undefined, {})).toBeUndefined(); +describe("the libnode pack identity a cached build keeps", () => { + // A workspace build of the target's @scriptc/libnode- that + // resolves next to the compiler (CI assembles one) is installed for every + // project, so without a pack in the project the identity names it. + const nextToCompiler = findInstalledLibnodePack({ + targetName: libnodeTarget({}), + projectDir: "/nonexistent", + }); + const withoutProjectPack = (key: string | undefined): void => { + if (nextToCompiler === null) expect(key).toBeUndefined(); + else expect(key).toContain(`|${nextToCompiler}|`); + }; + + test("is absent when no pack is named or installed", () => { + withoutProjectPack(libnodeCacheKey(undefined, {}, "/nonexistent")); + }); + + test("names the installed pack, and changes with it", () => { + const project = mkdtempSync(join(tmpdir(), "scriptc-libnode-project-")); + const name = libnodePackPackage(libnodeTarget({})); + if (name === null) return; + const installed = join(project, "node_modules", name); + mkdirSync(installed, { recursive: true }); + writeFileSync(join(installed, "manifest.json"), JSON.stringify({ nodeVersion: "24.22.0" })); + const before = libnodeCacheKey(undefined, {}, project); + expect(before).toContain(installed); + writeFileSync(join(installed, "manifest.json"), JSON.stringify({ nodeVersion: "24.23.0" })); + expect(libnodeCacheKey(undefined, {}, project)).not.toBe(before); + rmSync(installed, { recursive: true, force: true }); + withoutProjectPack(libnodeCacheKey(undefined, {}, project)); }); test("changes when the pack in the same directory is upgraded", () => { diff --git a/packages/compiler/src/node-islands/libnode.ts b/packages/compiler/src/node-islands/libnode.ts index dc6498924..30ff638ee 100644 --- a/packages/compiler/src/node-islands/libnode.ts +++ b/packages/compiler/src/node-islands/libnode.ts @@ -6,8 +6,8 @@ * SCRIPTC_LIBNODE overrides the recorded directory at run time. * * Where the build finds the pack: `--libnode=`, else - * SCRIPTC_LIBNODE in the build's environment (until NI-13 publishes - * @scriptc/libnode- packages that an installation carries). + * SCRIPTC_LIBNODE in the build's environment, else the installed + * @scriptc/libnode- pack (libnode-pack.ts). * `--libnode=beside` takes that pack and copies it next to the executable * as `.libnode/`, recorded as `$ORIGIN/.libnode`, so the * executable and its Node travel together; a recorded `$ORIGIN/` path is @@ -24,6 +24,16 @@ import { rmSync, } from "node:fs"; import { basename, isAbsolute, join, resolve, sep } from "node:path"; +import { compilerReleaseVersion } from "../library/sidecar.js"; +import { nativeCodegenTarget } from "../backend/targets.js"; +import { + findInstalledLibnodePack, + hostLibnodeTarget, + LibnodePackError, + libnodePackPackage, + readLibnodePack, + type LibnodePack, +} from "./libnode-pack.js"; /** The oldest Node.js release Node islands support: synchronous * module.registerHooks, which shared modules need, is a release candidate @@ -31,7 +41,7 @@ import { basename, isAbsolute, join, resolve, sep } from "node:path"; export const NODE_ISLAND_MIN_NODE = "24.13.1"; /** The bridge ABI this compiler's runtime speaks (scr_node_bridge.h). */ -export const NODE_BRIDGE_ABI = 3; +export const NODE_BRIDGE_ABI = 5; /** The prefix of a recorded path relative to the executable's directory. */ export const ORIGIN = "$ORIGIN/"; @@ -131,18 +141,56 @@ function realpathOr(path: string): string { } } +/** The native target whose libnode pack a build records: the codegen + * target (SCRIPTC_TARGET, and the host's libc), else the host. */ +export function libnodeTarget(env: NodeJS.ProcessEnv = process.env): string { + return nativeCodegenTarget(env)?.name ?? hostLibnodeTarget(); +} + +/** Why the installed pack in `dir` cannot serve this compiler, or null: it + * must be the target's package at the compiler's own version. */ +function installedPackProblem(dir: string, packageName: string): string | null { + let pack: LibnodePack; + try { + // An installed pack is found, not named: check its files' digests too. + pack = readLibnodePack(dir, { packageName, verifyHashes: true }); + } catch (error) { + return error instanceof LibnodePackError ? error.message : String(error); + } + const version = compilerReleaseVersion(); + if (pack.manifest.version !== version) + return `the installed libnode pack ${packageName} at ${dir} is version ${pack.manifest.version ?? "unknown"}; this scriptc is ${version}: install ${packageName}@${version}`; + return null; +} + /** What to record for a program that calls into Node, or the reason the * build cannot (an SC8005 message). `option` is --libnode. */ export function libnodeRecord( option: string | undefined, outPath: string, env: NodeJS.ProcessEnv = process.env, + projectDir: string = process.cwd(), ): LibnodeRecord | string { const fromEnv = env["SCRIPTC_LIBNODE"]; const beside = option === "beside"; - const source = option !== undefined && !beside ? option : fromEnv; - if (source === undefined || source === "") - return "this program calls into Node.js (@scriptc/node), but no libnode pack is installed: set SCRIPTC_LIBNODE or pass --libnode="; + const explicit = option !== undefined && !beside ? option : fromEnv; + const target = libnodeTarget(env); + // Without an explicit pack, the installed @scriptc/libnode-. + const installed = + explicit !== undefined && explicit !== "" + ? null + : findInstalledLibnodePack({ targetName: target, projectDir }); + const source = installed ?? explicit; + if (source === undefined || source === null || source === "") { + const pack = libnodePackPackage(target); + return pack === null + ? `this program calls into Node.js (@scriptc/node), but there is no libnode pack for ${target}: set SCRIPTC_LIBNODE or pass --libnode=` + : `this program calls into Node.js (@scriptc/node), but no libnode pack is installed: install ${pack}, set SCRIPTC_LIBNODE or pass --libnode=`; + } + if (installed !== null) { + const problem = installedPackProblem(installed, libnodePackPackage(target)!); + if (problem !== null) return problem; + } if (source.startsWith(ORIGIN)) { // A pack the deployment places next to the executable: nothing to read. return { libnode: source, nodeVersion: "" }; @@ -159,18 +207,23 @@ export function libnodeRecord( /** The identity of the libnode pack a build that calls into Node.js * records, which the program's executable cache entry keeps - * (EarlyExecutableCacheOptions.libnodeIdentity), or undefined when neither - * --libnode nor SCRIPTC_LIBNODE is set. It holds the option, the pack - * directory and a digest of the pack's manifest.json, so that a pack - * upgraded in place (another Node version, bridge ABI or file list) - * invalidates the entry instead of restoring its old record. */ + * (EarlyExecutableCacheOptions.libnodeIdentity): the option, the pack + * directory (named by --libnode or SCRIPTC_LIBNODE, or the installed pack + * found from `projectDir`), and a digest of the pack's manifest.json. A pack + * upgraded in place, removed, or replaced by another installed pack changes + * it. Undefined when no pack is named or installed. */ export function libnodeCacheKey( option: string | undefined, env: NodeJS.ProcessEnv = process.env, + projectDir: string = process.cwd(), ): string | undefined { const fromEnv = env["SCRIPTC_LIBNODE"]; - if (option === undefined && fromEnv === undefined) return undefined; - const source = option !== undefined && option !== "beside" ? option : (fromEnv ?? ""); + const explicit = option !== undefined && option !== "beside" ? option : fromEnv; + const source = + explicit !== undefined && explicit !== "" + ? explicit + : findInstalledLibnodePack({ targetName: libnodeTarget(env), projectDir }); + if (source === undefined || source === null) return option === undefined ? undefined : option; let manifest = "none"; if (isAbsolute(source)) { try { diff --git a/packages/compiler/src/node-islands/modules.test.ts b/packages/compiler/src/node-islands/modules.test.ts index 63201e1e0..6207f5809 100644 --- a/packages/compiler/src/node-islands/modules.test.ts +++ b/packages/compiler/src/node-islands/modules.test.ts @@ -30,6 +30,10 @@ describe("module specifiers in JavaScript", () => { export * from "./f"; export { g } from "./g"; const h = await import("./h.js"); + const require2 = createRequire(import.meta.url); + const j = require2("./j.cjs"); + const k = require2.resolve("k"); + const i = import.meta.resolve("i"); import.meta.url; obj.require("x"); const t = \`\${require("./t")}\`; @@ -44,6 +48,9 @@ describe("module specifiers in JavaScript", () => { "./f", "./g", "./h.js", + "./j.cjs", + "k", + "i", "./t", ]); expect(scan.computed).toEqual([]); @@ -67,10 +74,19 @@ describe("module specifiers in JavaScript", () => { expect(forms("require(name)")).toEqual(["require() with a computed specifier"]); expect(forms("require('./a' + b)")).toEqual(["require() with a computed specifier"]); expect(forms("import(`./x/${y}`)")).toEqual(["import() with a computed specifier"]); + expect(forms("import.meta.resolve(name)")).toEqual([ + "import.meta.resolve() with a computed specifier", + ]); expect(forms("const r = require; r('./a')")).toEqual(["require as a value"]); expect(forms("createRequire(import.meta.url)('./a')")).toEqual([ "a require() from createRequire()", ]); + expect(forms("const req = createRequire(import.meta.url); req(name)")).toEqual([ + "require() with a computed specifier", + ]); + expect(forms("const req = createRequire(import.meta.url); use(req)")).toEqual([ + "require as a value", + ]); expect(forms("function require(x) {} typeof require")).toEqual([]); }); }); @@ -138,6 +154,36 @@ describe("the collected module tree", () => { ]); }); + test("packages a package loads without declaring them ship too", () => { + const root = project({ + "main.ts": "", + "node_modules/bridge/package.json": JSON.stringify({ + type: "module", + imports: { "#mapped": { node: "mapped", default: "./local.js" } }, + }), + "node_modules/bridge/index.js": + 'import { a } from "zoo";\nexport const b = () => import("zoo/extra.js");', + "node_modules/bridge/index.d.ts": 'import type { T } from "typesonly";', + "node_modules/zoo/package.json": "{}", + "node_modules/zoo/index.js": "exports.a = 1;", + "node_modules/zoo/extra.js": "", + "node_modules/typesonly/index.d.ts": "", + "node_modules/mapped/index.js": "", + }); + const collector = new NodeIslandModuleCollector(); + expect(collector.addLoaded(join(root, "node_modules/bridge/index.js"))).toBeNull(); + // The root stays above node_modules, where Node looks for packages. + expect(collector.tree()!.files.map((f) => f.path)).toEqual([ + "node_modules/bridge/index.d.ts", + "node_modules/bridge/index.js", + "node_modules/bridge/package.json", + "node_modules/mapped/index.js", + "node_modules/zoo/extra.js", + "node_modules/zoo/index.js", + "node_modules/zoo/package.json", + ]); + }); + test("pnpm's links between package directories stay links inside the tree", () => { const root = project({ "app/main.ts": "", diff --git a/packages/compiler/src/node-islands/modules.ts b/packages/compiler/src/node-islands/modules.ts index e1551c070..4ed57a038 100644 --- a/packages/compiler/src/node-islands/modules.ts +++ b/packages/compiler/src/node-islands/modules.ts @@ -5,18 +5,25 @@ * Each `call`/`callSync` names its module with a string literal. The build * resolves it from the calling module as Node's require() does: * - a relative or absolute specifier names a module of the program: the - * module and every module its string-literal `require()`, `import` and - * `export ... from` reach are collected; a computed specifier in such a - * module is refused, since the build cannot know what it loads; + * module and every module its string-literal `require()` (also through a + * `createRequire()` function), `import` and `export ... from` reach are + * collected; a computed specifier in such a module is refused, since the + * build cannot know what it loads; * - a package specifier names a package: its whole directory is collected * (computed requires inside packages keep working), and so are its - * `dependencies`, `optionalDependencies` and `peerDependencies`, found as - * Node finds them; + * `dependencies`, `optionalDependencies` and `peerDependencies`, the + * packages its "imports" map to and the packages its JavaScript names + * with string literals, all found as Node finds them; * - a Node built-in module needs nothing, and neither does a module the * build cannot find: loading it throws MODULE_NOT_FOUND when the call * runs, as under Node. + * A load of a resolved file (`requireModule` and the compiler's own Node + * islands, lower-node-island.ts) collects the package that holds the file + * the same way, or, for a module of the program, the module and what it + * loads; the load then names the file by its path in the tree. * Paths keep their layout relative to one root directory (the deepest - * directory that holds every collected path), with symbolic links between + * directory that holds every collected path, above any node_modules + * directory, where Node looks for packages), with symbolic links between * package directories (pnpm's layout) kept as links. Only these relative * paths enter the executable. Every read goes through the frontend input * tracker, so an edit to an island module invalidates cached builds. */ @@ -33,6 +40,7 @@ import { trackedRealpath, } from "../frontend/input-tracker.js"; import type { IrNodeIslandModules } from "../ir/ir.js"; +import { packageDirOf } from "./origin.js"; export type NodeIslandModuleTree = IrNodeIslandModules; type NodeIslandModuleFile = IrNodeIslandModules["files"][number]; @@ -46,6 +54,9 @@ const FILE_EXTENSIONS = [".js", ".json", ".node"]; /** Module files whose imports are traced. */ const TRACED = /\.(?:[cm]?js|[cm]?ts)$/; +/** A package's JavaScript files, whose imports of other packages are followed. */ +const SCRIPT = /\.[cm]?js$/; + function isBuiltin(specifier: string): boolean { return specifier.startsWith("node:") || BUILTINS.has(specifier); } @@ -138,6 +149,16 @@ function commonDirectory(paths: readonly string[]): string { return joined === "" ? sep : joined; } +/** The string targets of a package.json "imports" map, through its + * conditions and fallback arrays. */ +function importTargets(value: unknown): string[] { + if (typeof value === "string") return [value]; + if (Array.isArray(value)) return value.flatMap(importTargets); + if (value !== null && typeof value === "object") + return Object.values(value).flatMap(importTargets); + return []; +} + function portable(path: string): string { return sep === "/" ? path : path.split(sep).join("/"); } @@ -151,6 +172,8 @@ export class NodeIslandModuleCollector { private readonly traced = new Set(); /** Real paths of the calling modules. */ private readonly callers = new Set(); + /** Real paths of the files loads name (addLoaded). */ + private readonly loaded = new Set(); /** Collects the module `specifier` names from `caller` (a real path). * Returns null, or the reason the build cannot ship it. */ @@ -159,6 +182,19 @@ export class NodeIslandModuleCollector { return this.resolveFrom(caller, specifier); } + /** Collects the module a load names by its resolved path (a real path): + * the package that holds it, or a module of the program with what it + * loads. Returns null, or the reason the build cannot ship it. */ + addLoaded(file: string): string | null { + this.loaded.add(file); + const at = packageDirOf(file); + if (at !== null) { + this.addPackage(at.dir); + return null; + } + return this.addModule(this.placed(file)); + } + /** The collected tree, or null when no call loads a file. */ tree(): NodeIslandModuleTree | null { if (this.files.size === 0 && this.links.size === 0) return null; @@ -167,8 +203,14 @@ export class NodeIslandModuleCollector { ...this.links.keys(), ...this.links.values(), ...[...this.callers].map((caller) => dirname(caller)), + ...[...this.loaded].map((file) => dirname(file)), ]; - const root = commonDirectory(anchors); + // Node finds a package through the node_modules directory that holds + // it, so a root inside one moves above it. + let root = commonDirectory(anchors); + const parts = root.split(sep); + const outer = parts.indexOf("node_modules"); + if (outer >= 0) root = parts.slice(0, outer).join(sep) || sep; const files: NodeIslandModuleFile[] = []; for (const file of [...this.files].sort()) { const bytes = trackedReadBytes(file); @@ -193,6 +235,12 @@ export class NodeIslandModuleCollector { return tree === null ? basename(caller) : portable(relative(tree.root, caller)); } + /** The name a load gives a collected file: `./` and its path relative to + * the tree root (the runtime resolves it under the root). */ + static loadPath(tree: NodeIslandModuleTree | null, file: string): string { + return tree === null ? file : `./${portable(relative(tree.root, file))}`; + } + private resolveFrom(fromFile: string, specifier: string): string | null { if (isBuiltin(specifier)) return null; if (specifier.startsWith("#")) @@ -270,7 +318,21 @@ export class NodeIslandModuleCollector { const dir = this.placed(found); if (this.packages.has(dir)) return; this.packages.add(dir); - this.addDirectory(dir, new Set()); + const scripts: string[] = []; + this.addDirectory(dir, new Set(), scripts); + // A package may load another it does not declare (one installed beside + // it): Node finds that one by walking up node_modules, and so does the + // build, from each string-literal specifier of the package's JavaScript. + for (const file of scripts) { + const text = trackedReadFile(file); + if (text === null) continue; + for (const specifier of scanModuleSpecifiers(text).specifiers) { + if (isBuiltin(specifier) || isPathSpecifier(specifier)) continue; + if (specifier.startsWith("#") || /^[a-z][a-z0-9+.-]*:/i.test(specifier)) continue; + const other = packageDirectory(dirname(file), packageName(specifier)); + if (other !== null) this.addPackage(other); + } + } const manifest = trackedReadFile(join(dir, "package.json")); if (manifest === null) return; let parsed: Record; @@ -285,6 +347,9 @@ export class NodeIslandModuleCollector { if (deps !== null && typeof deps === "object") for (const name of Object.keys(deps)) names.add(name); } + // A package import ("imports") may map to another package. + for (const target of importTargets(parsed["imports"])) + if (!isPathSpecifier(target) && !isBuiltin(target)) names.add(packageName(target)); for (const name of [...names].sort()) { // A dependency that is not installed (optional, or a peer the program // does not use) is not loaded either. @@ -294,17 +359,21 @@ export class NodeIslandModuleCollector { } /** Every file below `dir`, except nested node_modules (dependencies come - * through package.json) and .git. */ - private addDirectory(dir: string, visited: Set): void { + * through package.json and the package's imports) and .git; the + * JavaScript files among them also go to `scripts`. */ + private addDirectory(dir: string, visited: Set, scripts: string[]): void { const real = trackedRealpath(dir) ?? dir; if (visited.has(real)) return; visited.add(real); const entries = trackedAccessibleEntries(dir); if (entries === null) return; - for (const file of entries.files) this.files.add(join(dir, file)); + for (const file of entries.files) { + this.files.add(join(dir, file)); + if (SCRIPT.test(file)) scripts.push(join(dir, file)); + } for (const sub of entries.directories) { if (sub === "node_modules" || sub === ".git") continue; - this.addDirectory(join(dir, sub), visited); + this.addDirectory(join(dir, sub), visited, scripts); } } } @@ -490,8 +559,8 @@ export function moduleTokens(text: string): Token[] { } export interface ModuleSpecifierScan { - /** String-literal specifiers of require(), require.resolve(), import() - * and import/export declarations, in source order. */ + /** String-literal specifiers of require(), require.resolve(), import(), + * import.meta.resolve() and import/export declarations, in source order. */ specifiers: string[]; /** Loads whose module the source does not name with a string literal. */ computed: { at: number; form: string }[]; @@ -513,11 +582,27 @@ export function scanModuleSpecifiers(text: string): ModuleSpecifierScan { (is(k + 2, "punct", ")") || is(k + 2, "punct", ",")) ? at(k + 1)!.value : null; + // `const r = createRequire(...)` declares a require function of the + // module: calls of `r` load modules as require() calls do. + const aliases = new Set(); + const declarations = new Set(); + for (let k = 0; k + 4 < tokens.length; k++) { + if ( + (is(k, "name", "const") || is(k, "name", "let") || is(k, "name", "var")) && + is(k + 1, "name") && + is(k + 2, "punct", "=") && + is(k + 3, "name", "createRequire") && + is(k + 4, "punct", "(") + ) { + aliases.add(at(k + 1)!.value); + declarations.add(k + 1).add(k + 3); + } + } for (let k = 0; k < tokens.length; k++) { const t = tokens[k]!; - if (t.kind !== "name") continue; + if (t.kind !== "name" || declarations.has(k)) continue; const member = is(k - 1, "punct", ".") || is(k - 1, "punct", "?."); - if (t.value === "require" && !member && !is(k + 1, "punct", ":")) { + if ((t.value === "require" || aliases.has(t.value)) && !member && !is(k + 1, "punct", ":")) { if (is(k - 1, "name", "function") || is(k - 1, "name", "typeof")) continue; let call = k + 1; let form = "require()"; @@ -541,7 +626,16 @@ export function scanModuleSpecifiers(text: string): ModuleSpecifierScan { continue; } if (t.value === "import" && !member) { - if (is(k + 1, "punct", ".")) continue; // import.meta + if (is(k + 1, "punct", ".")) { + // import.meta.resolve() names a module, as require.resolve() does. + if (is(k + 2, "name", "meta") && is(k + 3, "punct", ".") && is(k + 4, "name", "resolve")) { + const literal = literalCall(k + 5); + if (literal !== null) specifiers.push(literal); + else if (is(k + 5, "punct", "(")) + computed.push({ at: t.at, form: "import.meta.resolve() with a computed specifier" }); + } + continue; + } if (is(k + 1, "punct", "(")) { const literal = literalCall(k + 1); if (literal === null) diff --git a/packages/compiler/src/node-islands/resolve.ts b/packages/compiler/src/node-islands/resolve.ts new file mode 100644 index 000000000..c30959a1b --- /dev/null +++ b/packages/compiler/src/node-islands/resolve.ts @@ -0,0 +1,16 @@ +/** `require.resolve(spec)` from `fromFile` for island code (scope-B NI-4). + * The Node-hosted compiler installs the resolver (index.ts, Node's own + * createRequire); the self-hosted compiler builds no islands and never + * installs one. Throws when the specifier does not resolve. */ +type IslandResolver = (fromFile: string, spec: string) => string; + +let islandResolver: IslandResolver | null = null; + +export function setIslandResolver(resolver: IslandResolver): void { + islandResolver = resolver; +} + +export function resolveFrom(fromFile: string, spec: string): string { + if (islandResolver === null) throw new Error(`no module resolver for '${spec}'`); + return islandResolver(fromFile, spec); +} diff --git a/packages/compiler/test/inline-array-fields.test.ts b/packages/compiler/test/inline-array-fields.test.ts index 6811c1d4a..7adeb0a81 100644 --- a/packages/compiler/test/inline-array-fields.test.ts +++ b/packages/compiler/test/inline-array-fields.test.ts @@ -191,6 +191,36 @@ byPos.set(list.pos, list.items); } }); +test("arrays fields hold do not reach unchecked code inside other values", async () => { + for (const [use, where] of [ + // Inside an array passed through a function value. + [ + `const cb = (xs: Item[][]): void => { + xs[0]!.push(new Item(9)); +}; +const f: (xs: Item[][]) => void = Math.random() < 2 ? cb : () => {}; +f([new List(0, [new Item(1)]).items as Item[]]);`, + "a call through a function value", + ], + // In a field of a class instance that becomes a dynamic value. + [ + `class Box { + readonly arr: readonly Item[]; + constructor(arr: readonly Item[]) { + this.arr = arr; + } +} +const u: unknown = new Box(new List(0, [new Item(1)]).items); +(u as { arr: Item[] }).arr.push(new Item(9));`, + "a dynamic value", + ], + ]) { + const result = await build(classes + use); + expect(result.ok, use).toBe(false); + expect(result.messages.join("\n"), use).toContain(`reaches ${where}`); + } +}); + test("a write names the field read whose array it may reach", async () => { const result = await build( classes + @@ -438,12 +468,7 @@ console.log(new List(2).items.length); expect(computed.messages.join("\n")).toContain("constructor parameter"); }); -test.runIf(lowRegion)( - "an instance inside a dynamic value shows the field as an array", - async () => { - const { ok, llvm } = await build( - classes + - ` +const holder = ` class Holder { readonly list: List; constructor(list: List) { @@ -451,17 +476,36 @@ class Holder { } } const list = new List(3, [new Item(1), new Item(2)]); -console.log(JSON.stringify({ list }), JSON.stringify(new Holder(list))); -`, +`; + +test("an instance inside a dynamic value is refused", async () => { + for (const use of [ + "const u: unknown = new Holder(list);\n(u as { list: { items: Item[] } }).list.items.push(new Item(3));", + "const u: unknown = [{ list }];\nconsole.log(u);", + "const u: unknown = new Map([[1, new Holder(list)]]);\nconsole.log(u);", + ]) { + const result = await build(classes + holder + use); + expect(result.ok, use).toBe(false); + expect(result.messages.join("\n"), use).toContain( + "an instance, or a value that can hold one, becomes a dynamic value", ); - expect(ok).toBe(true); - // The instance's dynamic view reads the field as an array of the elements. - const views = llvm - .split(/\n(?=define )/) - .filter((f) => f.startsWith("define") && f.includes("_materialize(")); - expect(views.some((f) => f.includes("call ptr @scr_arr_view_cref("))).toBe(true); - }, -); + expect(result.messages.join("\n"), use).toContain("(at "); + } +}); + +test.runIf(lowRegion)("printing and JSON.stringify show the field as an array", async () => { + const { ok, llvm } = await build( + classes + + holder + + "console.log(JSON.stringify({ list }), JSON.stringify(new Holder(list)));\nconsole.log(new Holder(list));", + ); + expect(ok).toBe(true); + // The instance's dynamic view reads the field as an array of the elements. + const views = llvm + .split(/\n(?=define )/) + .filter((f) => f.startsWith("define") && f.includes("_materialize(")); + expect(views.some((f) => f.includes("call ptr @scr_arr_view_cref("))).toBe(true); +}); test("the constructor stores the array parameter and does nothing else with it", async () => { const result = await build(` diff --git a/packages/compiler/test/node-index.test.ts b/packages/compiler/test/node-index.test.ts index 23fbc5203..5c1776fa2 100644 --- a/packages/compiler/test/node-index.test.ts +++ b/packages/compiler/test/node-index.test.ts @@ -3,7 +3,8 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { beforeAll, expect, test } from "vitest"; import { compile, deserializeModule } from "../src/index.js"; -import type { IrExpr, IrModule, IrStmt, IrType } from "../src/ir/ir.js"; +import type { IrExpr, IrFunction, IrModule, IrStmt, IrType } from "../src/ir/ir.js"; +import { emitRangeStarts } from "../src/backend/llvm/emit-module.js"; import { eachNodeOfKinds, IrNodeIndex, type IrNodeKind } from "../src/ir/node-index.js"; import { everyModuleNode, everyStmtList } from "../src/ir/traverse.js"; @@ -125,6 +126,31 @@ test("a false answer stops the visit, with or without the index", () => { } }); +test("a function's node count is the number of nodes its visit notes", () => { + const all = index.scanExcept(NO_KINDS); + for (const fn of mod.functions) expect(index.nodeCount(fn)).toBe(noted(fn, all).length); + expect(index.nodeCount({ ...mod.functions[0]! })).toBeUndefined(); +}); + +test("emission ranges cover the functions in order, none empty, of about equal cost", () => { + const count = (fn: IrFunction): number | undefined => index.nodeCount(fn); + for (const ranges of [1, 2, 3, mod.functions.length, mod.functions.length + 4]) { + const starts = emitRangeStarts(mod.functions, ranges, count); + expect(starts[0]).toBe(0); + expect(starts.at(-1)).toBe(mod.functions.length); + expect(starts.length - 1).toBeLessThanOrEqual(ranges); + for (let i = 1; i < starts.length; i++) expect(starts[i]!).toBeGreaterThan(starts[i - 1]!); + } + // Equal costs split into equal ranges. A range ends after the function + // that reaches its share of the cost, so a large function ends its range. + const functions = Array.from({ length: 12 }, () => mod.functions[0]!); + expect(emitRangeStarts(functions, 4, () => 0)).toEqual([0, 3, 6, 9, 12]); + const sizes = [0, 0, 1000, 0, 0, 0]; + const uneven = sizes.map((_, i) => ({ ...mod.functions[0]!, name: `f${i}` })); + const bySize = new Map(uneven.map((fn, i) => [fn, sizes[i]!])); + expect(emitRangeStarts(uneven, 3, (fn) => bySize.get(fn))).toEqual([0, 3, 4, 6]); +}); + test("a function the index did not record is walked", () => { const all = index.scanExcept(NO_KINDS); for (const fn of mod.functions) { diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6804-node-island-signals/main.mjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6804-node-island-signals/main.mjs.json new file mode 100644 index 000000000..ba3904bc3 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6804-node-island-signals/main.mjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/6804-node-island-signals/main.mjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6805-node-island-exit-code/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6805-node-island-exit-code/main.ts.json new file mode 100644 index 000000000..d00f9ae2d --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6805-node-island-exit-code/main.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/6805-node-island-exit-code/main.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6806-node-island-exit-in-call/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6806-node-island-exit-in-call/main.ts.json new file mode 100644 index 000000000..fbf105213 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6806-node-island-exit-in-call/main.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/6806-node-island-exit-in-call/main.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6807-node-island-exit-in-island-timer/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6807-node-island-exit-in-island-timer/main.ts.json new file mode 100644 index 000000000..94e8c0207 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6807-node-island-exit-in-island-timer/main.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/6807-node-island-exit-in-island-timer/main.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6808-node-island-exit-from-program/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6808-node-island-exit-from-program/main.ts.json new file mode 100644 index 000000000..c7c22efaf --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6808-node-island-exit-from-program/main.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/6808-node-island-exit-from-program/main.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6809-node-island-stdio/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6809-node-island-stdio/main.ts.json new file mode 100644 index 000000000..a33259b1d --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6809-node-island-stdio/main.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/6809-node-island-stdio/main.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6810-node-island-children/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6810-node-island-children/main.ts.json new file mode 100644 index 000000000..220ebcd2f --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6810-node-island-children/main.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/6810-node-island-children/main.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6811-node-island-uncaught-listener/main.mjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6811-node-island-uncaught-listener/main.mjs.json new file mode 100644 index 000000000..f2385e3b2 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6811-node-island-uncaught-listener/main.mjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/6811-node-island-uncaught-listener/main.mjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6812-node-island-handles/main.mjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6812-node-island-handles/main.mjs.json new file mode 100644 index 000000000..403480de2 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6812-node-island-handles/main.mjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/6812-node-island-handles/main.mjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6813-node-island-handle-errors/main.mjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6813-node-island-handle-errors/main.mjs.json new file mode 100644 index 000000000..ab16e10cd --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6813-node-island-handle-errors/main.mjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/6813-node-island-handle-errors/main.mjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6814-node-island-handle-bytes/main.mjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6814-node-island-handle-bytes/main.mjs.json new file mode 100644 index 000000000..b5edf29a7 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6814-node-island-handle-bytes/main.mjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/6814-node-island-handle-bytes/main.mjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6815-node-island-handle-promises/main.mjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6815-node-island-handle-promises/main.mjs.json new file mode 100644 index 000000000..77f6f3a32 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6815-node-island-handle-promises/main.mjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/6815-node-island-handle-promises/main.mjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6816-node-island-objects/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6816-node-island-objects/main.ts.json new file mode 100644 index 000000000..015f2d9da --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6816-node-island-objects/main.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/6816-node-island-objects/main.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6817-node-island-esm/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6817-node-island-esm/main.ts.json new file mode 100644 index 000000000..fecb7ed9f --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6817-node-island-esm/main.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/6817-node-island-esm/main.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6822-node-island-gc/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6822-node-island-gc/main.ts.json new file mode 100644 index 000000000..6222bdbac --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6822-node-island-gc/main.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/6822-node-island-gc/main.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6823-node-island-nested-stack/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6823-node-island-nested-stack/main.ts.json new file mode 100644 index 000000000..aa900980e --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6823-node-island-nested-stack/main.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/6823-node-island-nested-stack/main.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6839-node-island-path-mapped/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6839-node-island-path-mapped/main.ts.json new file mode 100644 index 000000000..8ae987602 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6839-node-island-path-mapped/main.ts.json @@ -0,0 +1,18 @@ +{ + "order": [ + "/tests/corpus/6839-node-island-path-mapped/main.ts" + ], + "diags": [ + { + "code": "SC1010", + "message": "the 'pathkit' import (it resolves only to type declarations — there is no runtime implementation to compile) is not supported yet", + "loc": { + "file": "/tests/corpus/6839-node-island-path-mapped/main.ts", + "start": 234, + "end": 294 + }, + "milestone": "M4", + "hint": "relative imports (./file, ../dir/file), package.json-mediated project imports (#alias via the imports field, self-name references via exports), statically compiled npm packages, and the built-in fs, fs/promises, path, os, url, crypto, zlib, child_process, net, http, tls, https, http2, dgram, dns, dns/promises, util, util/types, string_decoder, querystring, readline, events, stream, stream/promises, stream/consumers, stream/web, buffer, assert, assert/strict, worker_threads, cluster, tty, async_hooks, timers, timers/promises, diagnostics_channel, perf_hooks, module, v8, and console modules (bare or node:-prefixed) and node:test (node:-prefixed only, like in Node) are supported" + } + ] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6890-node-island-timer-refresh/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6890-node-island-timer-refresh/main.ts.json new file mode 100644 index 000000000..2da15efc8 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/6890-node-island-timer-refresh/main.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/6890-node-island-timer-refresh/main.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/generic-conditional-member-bindings.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/generic-conditional-member-bindings.ts.json new file mode 100644 index 000000000..398cc39e9 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/generic-conditional-member-bindings.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/generic-conditional-member-bindings.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/generic-conditional-member-units.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/generic-conditional-member-units.ts.json new file mode 100644 index 000000000..35b3ba808 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/generic-conditional-member-units.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/generic-conditional-member-units.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/unloaded-dynamic-import/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/unloaded-dynamic-import/main.ts.json new file mode 100644 index 000000000..2470af846 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/unloaded-dynamic-import/main.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/fixtures/npm/cases/unloaded-dynamic-import/main.ts" + ], + "diags": [] +} diff --git a/packages/experimental/README.md b/packages/experimental/README.md index 4e2a89281..03288d84f 100644 --- a/packages/experimental/README.md +++ b/packages/experimental/README.md @@ -144,8 +144,8 @@ function visit(list: NodeList): void { - The constructor of the class that declares the field writes it once, from one of its parameters. A subclass constructor passes one of its own parameters there through `super(...)`. Nothing else writes the field. - Indexing the field, reading its `length` and iterating it with `for...of` read the elements from the instance, directly or through a `const` local used only in those ways. Any other use, such as passing the field to a function or calling one of its methods, gets an array of the elements, which is frozen in scriptc builds. - The array passed to the constructor is one that nothing else keeps: a new array (a literal, a copy, or the result of a function that returns only such arrays or its own argument), a local array at its last use, a parameter whose callers each pass such an array, an array that an InlineArray field already holds, or a `const` array that is never written. -- The arrays fields hold are never written, through a read of the field (a cast) or through any other reference. scriptc follows them through locals, parameters, return values, fields, the elements of arrays, Maps and Sets, and closures, and reports one that reaches a write, a dynamic value, a call through a function value, or a library call other than printing and `JSON.stringify` without a replacer. +- The arrays fields hold are never written, through a read of the field (a cast) or through any other reference. scriptc follows them through locals, parameters, return values, fields, the elements of arrays, Maps and Sets, and closures, and reports one that reaches a write, a dynamic value (also inside the fields of an object that becomes one), a call through a function value (also inside an array, Map or Set passed there), or a library call other than printing and `JSON.stringify` without a replacer. - Two reads of the field need not give the same array, so the arrays fields hold are never compared by identity: with `===` or `!==`, as an `indexOf` or `includes` argument, or as a Map key or Set element. Compare the elements instead, or keep a copy made with `slice()` where identity matters. -- Instances of the class are not constructed through a class value, and do not become dynamic values (for example through `JSON.stringify` or a conversion to `unknown`). +- Instances of the class are not constructed through a class value, and do not become dynamic values (for example through a conversion to `unknown`), directly or inside another value that can hold one: an instance of a class with such a field, a record, array, Map or Set, or a value typed as a base class. A dynamic view reads the field as a new array each time, so a class whose instances can become dynamic values cannot hold an InlineArray field. Printing, `JSON.stringify` without a replacer, `publish` and a worker's `workerData` only read or copy the value and are allowed. scriptc reports a field that does not keep these rules (SC7009). Under Node.js the field holds the array the constructor stored. In builds with sanitizers, in programs that use `LifetimeRegion`, and on platforms other than 64-bit Linux, the elements stay in an ordinary array object. diff --git a/packages/libnode-common/bridge/scr_node_bridge.cc b/packages/libnode-common/bridge/scr_node_bridge.cc index 7365b08dc..a5caa2022 100644 --- a/packages/libnode-common/bridge/scr_node_bridge.cc +++ b/packages/libnode-common/bridge/scr_node_bridge.cc @@ -9,21 +9,34 @@ // no default signal handling, no debug-signal handler or inspector, no // stdio reinitialization, one platform worker thread. Node's process exit // (process.exit, an uncaught exception) goes to the runtime's exit -// sequence. Calls run in a CallbackScope with a nested TryCatch, so ticks -// and microtasks drain when the scope closes, as after any libuv callback. +// sequence. +// +// One loop (scope-B NI-3). Once the runtime calls host_loop, its queues +// live on Node's: its microtasks are V8 microtasks, its ticks, timers and +// immediates are Node's (scheduled through the bootstrap's helpers), and +// its I/O is watched from Node's libuv loop (uv_poll handles over the fds +// the runtime reports, one prepare handle whose ref is the runtime's +// liveness, an idle handle while runtime work is ready without any fd). +// Calls into JavaScript therefore do not drain queues themselves: the +// runtime's checkpoints do, through `checkpoint`, and Node's own after +// every callback it dispatches. #include "scr_node_bridge.h" #include #include +#include #include #include #include #include +#include #include #include +#include #include #include +#include #include namespace { @@ -44,12 +57,17 @@ using v8::TryCatch; using v8::Value; const ScrNodeBridgeHost *g_host = nullptr; +void DropAllHandles(); std::unique_ptr g_platform; node::EmbedderSnapshotData::Pointer g_snapshot; std::unique_ptr g_setup; Locker *g_locker = nullptr; Global g_call; -Global g_finish; +// The bootstrap's hosted-loop helpers (scr_node_island_host.cjs). +Global g_set_timer, g_clear_timer, g_ref_timer, g_refresh_timer; +Global g_set_immediate, g_clear_immediate, g_ref_immediate; +Global g_next_tick, g_emit; +Global g_load; // the bootstrap's module loader (NI-4) char *Copy(const std::string &text, size_t *len) { char *out = static_cast(std::malloc(text.size() + 1)); @@ -60,6 +78,29 @@ char *Copy(const std::string &text, size_t *len) { return out; } +// V8's stack limit follows the stack the runtime calls in from: its main +// thread's, or an async fiber's (a few hundred KiB, mapped elsewhere), so +// JavaScript entered from a fiber neither trips a false stack overflow nor +// overruns the fiber. Restored when the entry returns. +uintptr_t g_stack_limit = 0; + +struct StackLimit { + uintptr_t previous; + explicit StackLimit(Isolate *isolate) : previous(g_stack_limit) { + uintptr_t want = g_host->stack_limit(); + if (want != 0 && want != g_stack_limit) { + isolate->SetStackLimit(want); + g_stack_limit = want; + } + } + ~StackLimit() { + if (previous != 0 && previous != g_stack_limit) { + g_setup->isolate()->SetStackLimit(previous); + g_stack_limit = previous; + } + } +}; + // The start of the runtime's stack when the caller runs on it (see the // v8::base::Stack definitions at the end of this file), else 0. uintptr_t HostStackStart() { @@ -114,6 +155,39 @@ void Settle(const FunctionCallbackInfo &info) { g_host->settle(id, outcome.data(), outcome.size()); } +uint32_t Uint32Arg(const FunctionCallbackInfo &info, int i) { + return info[i]->Uint32Value(info.GetIsolate()->GetCurrentContext()).FromMaybe(0); +} + +// run(kind, token): a runtime job Node's queues dispatched. +void Run(const FunctionCallbackInfo &info) { + g_host->run(Uint32Arg(info, 0), Uint32Arg(info, 1)); +} + +// listen(event, name, listener, add, once): island process listeners. +void Listen(const FunctionCallbackInfo &info) { + std::string name = ToStdString(info.GetIsolate(), info[1]); + g_host->listen(Uint32Arg(info, 0), name.c_str(), Uint32Arg(info, 2), Uint32Arg(info, 3) != 0, + Uint32Arg(info, 4) != 0); +} + +void SetMethod(Local context, Local target, const char *name, + v8::FunctionCallback callback) { + target + ->Set(context, Utf8(context->GetIsolate(), name), + Function::New(context, callback).ToLocalChecked()) + .Check(); +} + +bool TakeFunction(Isolate *isolate, Local context, Local from, const char *name, + Global *to) { + Local value; + if (!from->Get(context, Utf8(isolate, name)).ToLocal(&value) || !value->IsFunction()) + return false; + to->Reset(isolate, value.As()); + return true; +} + char *Start(const ScrNodeBridgeHost *host, const char *bootstrap, const char *argv0, uintptr_t stack_limit) { if (g_setup) return nullptr; @@ -167,8 +241,10 @@ char *Start(const ScrNodeBridgeHost *host, const char *bootstrap, const char *ar g_locker = new Locker(isolate); isolate->Enter(); // The runtime calls into Node on a dedicated stack of its own - // (scr_node_island.c), not the thread's: V8's limit follows it. + // (scr_node_island.c), not the thread's: V8's limit follows it, and + // follows any other stack an entry comes from (StackLimit). if (stack_limit != 0) isolate->SetStackLimit(stack_limit); + g_stack_limit = stack_limit; node::SetProcessExitHandler(g_setup->env(), [](node::Environment *, int code) { g_host->exit(code); }); HandleScope handles(isolate); @@ -176,9 +252,9 @@ char *Start(const ScrNodeBridgeHost *host, const char *bootstrap, const char *ar Context::Scope context_scope(context); Local global = context->Global(); Local native = Object::New(isolate); - native - ->Set(context, Utf8(isolate, "settle"), Function::New(context, Settle).ToLocalChecked()) - .Check(); + SetMethod(context, native, "settle", Settle); + SetMethod(context, native, "run", Run); + SetMethod(context, native, "listen", Listen); global->Set(context, Utf8(isolate, "__scriptcNodeIslandHost"), native).Check(); TryCatch try_catch(isolate); if (node::LoadEnvironment(g_setup->env(), bootstrap).IsEmpty()) { @@ -197,17 +273,22 @@ char *Start(const ScrNodeBridgeHost *host, const char *bootstrap, const char *ar return Copy(message, nullptr); } Local dispatcher; - Local call; - Local finish; if (!global->Get(context, Utf8(isolate, "__scriptcNodeIsland")).ToLocal(&dispatcher) || - !dispatcher->IsObject() || - !dispatcher.As()->Get(context, Utf8(isolate, "call")).ToLocal(&call) || - !call->IsFunction() || - !dispatcher.As()->Get(context, Utf8(isolate, "finish")).ToLocal(&finish) || - !finish->IsFunction()) + !dispatcher->IsObject()) return Copy("the Node island bootstrap left no dispatcher", nullptr); - g_call.Reset(isolate, call.As()); - g_finish.Reset(isolate, finish.As()); + Local d = dispatcher.As(); + if (!TakeFunction(isolate, context, d, "call", &g_call) || + !TakeFunction(isolate, context, d, "setTimer", &g_set_timer) || + !TakeFunction(isolate, context, d, "clearTimer", &g_clear_timer) || + !TakeFunction(isolate, context, d, "refTimer", &g_ref_timer) || + !TakeFunction(isolate, context, d, "refreshTimer", &g_refresh_timer) || + !TakeFunction(isolate, context, d, "setImmediate", &g_set_immediate) || + !TakeFunction(isolate, context, d, "clearImmediate", &g_clear_immediate) || + !TakeFunction(isolate, context, d, "refImmediate", &g_ref_immediate) || + !TakeFunction(isolate, context, d, "nextTick", &g_next_tick) || + !TakeFunction(isolate, context, d, "emit", &g_emit) || + !TakeFunction(isolate, context, d, "load", &g_load)) + return Copy("the Node island bootstrap left an incomplete dispatcher", nullptr); global->Delete(context, Utf8(isolate, "__scriptcNodeIsland")).FromMaybe(false); global->Delete(context, Utf8(isolate, "__scriptcNodeIslandHost")).FromMaybe(false); return nullptr; @@ -216,12 +297,14 @@ char *Start(const ScrNodeBridgeHost *host, const char *bootstrap, const char *ar char *Call(uint32_t id, int sync, const char *file, const char *specifier, const char *name, const char *args, size_t args_len, size_t *len) { Isolate *isolate = g_setup->isolate(); + StackLimit limit(isolate); HandleScope handles(isolate); Local context = g_setup->context(); Context::Scope context_scope(context); std::string outcome; { - node::CallbackScope scope(isolate, context->Global(), node::async_context{0, 0}); + // No callback scope: the runtime's checkpoint drains what the call + // queued, at the end of the runtime callback that made it (NI-3). TryCatch try_catch(isolate); Local args_text; if (!MaybeUtf8(isolate, args, args_len).ToLocal(&args_text)) { @@ -260,43 +343,288 @@ char *Call(uint32_t id, int sync, const char *file, const char *specifier, const void Release(void *memory) { std::free(memory); } -int BackendFd() { return uv_backend_fd(g_setup->event_loop()); } +// ── the hosted loop ────────────────────────────────────────────────────── -int BackendTimeout() { return uv_backend_timeout(g_setup->event_loop()); } +struct Enter { + Isolate *isolate; + StackLimit limit; + HandleScope handles; + Local context; + Context::Scope scope; + Enter() + : isolate(g_setup->isolate()), limit(isolate), handles(isolate), + context(g_setup->context()), scope(context) {} +}; -int Alive() { return uv_loop_alive(g_setup->event_loop()); } - -void Pump() { - Isolate *isolate = g_setup->isolate(); - HandleScope handles(isolate); - Context::Scope context_scope(g_setup->context()); - uv_run(g_setup->event_loop(), UV_RUN_NOWAIT); - g_platform->DrainTasks(isolate); +// Calls a bootstrap helper; helpers do not throw, a failure answers empty. +Local Helper(Enter &e, Global &fn, int argc, Local *argv) { + TryCatch try_catch(e.isolate); + Local result; + if (!fn.Get(e.isolate)->Call(e.context, v8::Undefined(e.isolate), argc, argv).ToLocal(&result)) + return Local(); + return result; } -// The program ends with `code`: Node's process ends with it too. Node's -// 'beforeExit' runs (and the work it schedules), then 'exit'; the result -// is the exit code Node's process ends with (process.exitCode). -int Finish(int code) { - if (!g_setup) return code; - Isolate *isolate = g_setup->isolate(); - { - HandleScope handles(isolate); - Local context = g_setup->context(); - Context::Scope context_scope(context); - TryCatch try_catch(isolate); - Local argv[] = {Integer::New(isolate, code)}; - (void)g_finish.Get(isolate)->Call(context, v8::Undefined(isolate), 1, argv); +Local Num(Enter &e, double value) { return v8::Number::New(e.isolate, value); } + +uint32_t HelperHandle(Enter &e, Local result) { + if (result.IsEmpty()) return 0; + return result->Uint32Value(e.context).FromMaybe(0); +} + +struct PollEntry { + uv_poll_t handle; + int fd; + bool seen; + bool active; +}; + +uv_prepare_t g_prepare; +uv_check_t g_check; +uv_idle_t g_idle; +uv_timer_t g_cap; +bool g_hosted = false; +bool g_trace = false; // SCRIPTC_NODE_ISLANDS_TRACE=loop: the I/O state on stderr +bool g_dispatched = false; // the runtime's stations ran this iteration +std::unordered_map g_polls; + +void Dispatch() { + if (g_dispatched) return; + g_dispatched = true; + g_host->io_dispatch(); +} + +void SyncIo(int phase); + +void OnPoll(uv_poll_t *, int, int) { + Dispatch(); + SyncIo(SCR_NODE_IO_JOBS); +} + +void OnIdle(uv_idle_t *) { + Dispatch(); + SyncIo(SCR_NODE_IO_JOBS); +} + +void OnCap(uv_timer_t *) { + Dispatch(); + SyncIo(SCR_NODE_IO_JOBS); +} + +void OnPrepare(uv_prepare_t *) { SyncIo(SCR_NODE_IO_PREPARE); } + +// The end of an iteration: the next one may dispatch again (from its idle +// phase when work is ready, else from its poll phase). +void OnCheck(uv_check_t *) { + g_dispatched = false; + SyncIo(SCR_NODE_IO_CHECK); +} + +void ClosePoll(uv_handle_t *handle) { delete reinterpret_cast(handle->data); } + +// Mirrors the runtime's I/O state onto the loop: one unref'd poll handle +// per fd, the prepare handle's ref as the runtime's liveness, the idle +// handle while work is ready, the cap timer for work no fd announces. A +// poll handle whose fd leaves the set is stopped, not closed, and started +// again when the fd returns (the worker pool's wake pipe comes and goes +// with every job): libuv cannot watch an fd with a second handle while the +// first is still closing. +void SyncIo(int phase) { + if (!g_hosted) return; + ScrNodeIoState state; + state.phase = phase; + g_host->io_state(&state); + if (g_trace) { + std::fprintf(stderr, "[node-islands] %s alive=%d ready=%d cap=%d fds=", + phase == SCR_NODE_IO_CHECK ? "check" : phase == SCR_NODE_IO_JOBS ? "jobs" : "prepare", + state.alive, state.ready, state.cap_ms); + for (int i = 0; i < state.nfds; i++) std::fprintf(stderr, "%s%d", i ? "," : "", state.fds[i]); + std::fprintf(stderr, "\n"); } - return node::SpinEventLoop(g_setup->env()).FromMaybe(code); + uv_loop_t *loop = g_setup->event_loop(); + for (auto &entry : g_polls) entry.second->seen = false; + for (int i = 0; i < state.nfds; i++) { + int fd = state.fds[i]; + auto found = g_polls.find(fd); + PollEntry *entry; + if (found != g_polls.end()) { + entry = found->second; + } else { + entry = new PollEntry(); + entry->fd = fd; + entry->active = false; + if (uv_poll_init(loop, &entry->handle, fd) != 0) { + delete entry; + continue; + } + entry->handle.data = entry; + uv_unref(reinterpret_cast(&entry->handle)); + g_polls.emplace(fd, entry); + } + entry->seen = true; + if (!entry->active && uv_poll_start(&entry->handle, UV_READABLE, OnPoll) == 0) + entry->active = true; + } + for (auto &it : g_polls) { + if (it.second->seen || !it.second->active) continue; + uv_poll_stop(&it.second->handle); + it.second->active = false; + } + if (state.alive) uv_ref(reinterpret_cast(&g_prepare)); + else uv_unref(reinterpret_cast(&g_prepare)); + if (state.ready) uv_idle_start(&g_idle, OnIdle); + else uv_idle_stop(&g_idle); + if (state.cap_ms >= 0) { + if (!uv_is_active(reinterpret_cast(&g_cap))) + uv_timer_start(&g_cap, OnCap, static_cast(state.cap_ms), 0); + } else { + uv_timer_stop(&g_cap); + } +} + +// After every microtask checkpoint, which ends every callback Node +// dispatches (closing-phase ones included): the runtime's rejections, and +// its liveness, which the callback may have changed after the turn's check. +void OnMicrotasksDone(Isolate *, void *) { + if (!g_hosted) return; + g_host->microtasks_done(); + SyncIo(SCR_NODE_IO_JOBS); +} + +void HostLoop() { + if (g_hosted) return; + uv_loop_t *loop = g_setup->event_loop(); + uv_prepare_init(loop, &g_prepare); + uv_prepare_start(&g_prepare, OnPrepare); + uv_check_init(loop, &g_check); + uv_check_start(&g_check, OnCheck); + uv_unref(reinterpret_cast(&g_check)); + uv_idle_init(loop, &g_idle); + uv_unref(reinterpret_cast(&g_idle)); + uv_timer_init(loop, &g_cap); + uv_unref(reinterpret_cast(&g_cap)); + g_setup->isolate()->AddMicrotasksCompletedCallback(OnMicrotasksDone, nullptr); + const char *trace = std::getenv("SCRIPTC_NODE_ISLANDS_TRACE"); + g_trace = trace != nullptr && std::strcmp(trace, "loop") == 0; + g_hosted = true; + SyncIo(SCR_NODE_IO_JOBS); +} + +void UnhostLoop() { + if (!g_hosted) return; + g_hosted = false; + g_setup->isolate()->RemoveMicrotasksCompletedCallback(OnMicrotasksDone, nullptr); + for (auto &entry : g_polls) { + uv_poll_stop(&entry.second->handle); + uv_close(reinterpret_cast(&entry.second->handle), ClosePoll); + } + g_polls.clear(); + uv_close(reinterpret_cast(&g_prepare), nullptr); + uv_close(reinterpret_cast(&g_check), nullptr); + uv_close(reinterpret_cast(&g_idle), nullptr); + uv_close(reinterpret_cast(&g_cap), nullptr); + uv_run(g_setup->event_loop(), UV_RUN_NOWAIT); +} + +void RunMicrotask(void *data) { + g_host->run(SCR_NODE_JOB_MICROTASK, static_cast(reinterpret_cast(data))); +} + +void Microtask(uint32_t token) { + g_setup->isolate()->EnqueueMicrotask(RunMicrotask, + reinterpret_cast(static_cast(token))); +} + +void Tick(uint32_t token) { + Enter e; + Local argv[] = {Num(e, token)}; + Helper(e, g_next_tick, 1, argv); +} + +uint32_t Timer(double delay, double repeat, int ref, uint32_t token) { + Enter e; + Local argv[] = {Num(e, delay), Num(e, repeat), v8::Boolean::New(e.isolate, ref != 0), + Num(e, token)}; + return HelperHandle(e, Helper(e, g_set_timer, 4, argv)); +} + +void TimerClear(uint32_t handle) { + Enter e; + Local argv[] = {Num(e, handle)}; + Helper(e, g_clear_timer, 1, argv); +} + +void TimerRef(uint32_t handle, int ref) { + Enter e; + Local argv[] = {Num(e, handle), v8::Boolean::New(e.isolate, ref != 0)}; + Helper(e, g_ref_timer, 2, argv); +} + +void TimerRefresh(uint32_t handle) { + Enter e; + Local argv[] = {Num(e, handle)}; + Helper(e, g_refresh_timer, 1, argv); +} + +uint32_t Immediate(int ref, uint32_t token) { + Enter e; + Local argv[] = {v8::Boolean::New(e.isolate, ref != 0), Num(e, token)}; + return HelperHandle(e, Helper(e, g_set_immediate, 2, argv)); +} + +void ImmediateClear(uint32_t handle) { + Enter e; + Local argv[] = {Num(e, handle)}; + Helper(e, g_clear_immediate, 1, argv); +} + +void ImmediateRef(uint32_t handle, int ref) { + Enter e; + Local argv[] = {Num(e, handle), v8::Boolean::New(e.isolate, ref != 0)}; + Helper(e, g_ref_immediate, 2, argv); +} + +// The checkpoint after a runtime macrotask: an empty callback scope, whose +// close runs Node's tick queue and V8's microtasks (InternalCallbackScope:: +// Close) unless Node's own dispatch is on the stack, which drains after. +void Checkpoint(int microtasks_first) { + Enter e; + if (microtasks_first) e.context->GetMicrotaskQueue()->PerformCheckpoint(e.isolate); + node::CallbackScope scope(e.isolate, Object::New(e.isolate), node::async_context{0, 0}); +} + +int RunLoop(int continue_turn) { + Enter e; + // The rest of the turn the switch interrupted: libuv's UV_RUN_ONCE skips + // the leading timers phase and runs poll, check, closing, then timers. + if (continue_turn) { + uv_run(g_setup->event_loop(), UV_RUN_ONCE); + g_platform->DrainTasks(e.isolate); + } + return node::SpinEventLoop(g_setup->env()).FromMaybe(0); +} + +// An island listener of a process event: through MakeCallback, so a throw +// is an uncaught exception as in Node's own emit. +void Emit(uint32_t event, uint32_t listener, const char *name, int arg) { + Enter e; + Local argv[] = {Num(e, event), Num(e, listener), Utf8(e.isolate, name), Num(e, arg)}; + node::MakeCallback(e.isolate, e.context->Global(), g_emit.Get(e.isolate), 4, argv, + node::async_context{0, 0}); } void Stop() { if (!g_setup) return; + UnhostLoop(); Isolate *isolate = g_setup->isolate(); node::Stop(g_setup->env()); g_call.Reset(); - g_finish.Reset(); + g_load.Reset(); + DropAllHandles(); + for (Global *fn : + {&g_set_timer, &g_clear_timer, &g_ref_timer, &g_refresh_timer, &g_set_immediate, + &g_clear_immediate, &g_ref_immediate, &g_next_tick, &g_emit}) + fn->Reset(); isolate->Exit(); delete g_locker; g_locker = nullptr; @@ -306,10 +634,629 @@ void Stop() { node::TearDownOncePerProcess(); } + +// ── object handles (scope-B NI-4) ─────────────────────────────────────── +// The import table: V8 values the runtime holds, one v8::Global per value +// and one id per value while held (found again through the value's +// identity hash), so a V8 object crossing twice is the same runtime +// handle. A typed array's slot also keeps its backing store, so memory the +// runtime aliases stays valid even if JavaScript detaches the buffer. + +struct Slot { + Global value; + std::shared_ptr store; + int hash = 0; + bool live = false; +}; + +std::vector g_slots(1); // id 0 is never used +std::vector g_free_ids; +std::unordered_multimap g_ids; +uint32_t g_live_handles = 0; +// Strings of out-values: valid until the next outermost entry. +std::deque g_scratch; +int g_entry_depth = 0; + +int IdentityHash(Local v) { + if (v->IsObject()) return v.As()->GetIdentityHash(); + if (v->IsSymbol()) return v.As()->GetIdentityHash(); + return 0; +} + +uint32_t Import(Isolate *isolate, Local v, std::shared_ptr store = nullptr) { + int hash = IdentityHash(v); + auto range = g_ids.equal_range(hash); + for (auto it = range.first; it != range.second; ++it) { + Slot &slot = g_slots[it->second]; + if (slot.value.Get(isolate)->StrictEquals(v)) return it->second; + } + uint32_t id; + if (!g_free_ids.empty()) { + id = g_free_ids.back(); + g_free_ids.pop_back(); + } else { + id = static_cast(g_slots.size()); + g_slots.emplace_back(); + } + Slot &slot = g_slots[id]; + slot.value.Reset(isolate, v); + slot.store = std::move(store); + slot.hash = hash; + slot.live = true; + g_ids.emplace(hash, id); + g_live_handles++; + return id; +} + +Local HandleValue(Isolate *isolate, uint32_t id) { + if (id == 0 || id >= g_slots.size() || !g_slots[id].live) return v8::Undefined(isolate); + return g_slots[id].value.Get(isolate); +} + +void Drop(uint32_t id) { + if (id == 0 || id >= g_slots.size() || !g_slots[id].live) return; + Slot &slot = g_slots[id]; + auto range = g_ids.equal_range(slot.hash); + for (auto it = range.first; it != range.second; ++it) { + if (it->second == id) { + g_ids.erase(it); + break; + } + } + slot.value.Reset(); + slot.store.reset(); + slot.live = false; + g_free_ids.push_back(id); + g_live_handles--; +} + +void DropAllHandles() { + for (uint32_t id = 1; id < g_slots.size(); id++) Drop(id); +} + +uint32_t LiveHandles() { return g_live_handles; } + +const char *Scratch(std::string text, size_t *len) { + g_scratch.push_back(std::move(text)); + *len = g_scratch.back().size(); + return g_scratch.back().data(); +} + +// WTF-8: V8's UTF-8 writer encodes a lone surrogate as its 3-byte form. +std::string Wtf8(Isolate *isolate, Local s) { + size_t n = s->Utf8LengthV2(isolate); + std::string out(n, '\0'); + if (n > 0) s->WriteUtf8V2(isolate, out.data(), n); + return out; +} + +bool HasLoneSurrogate(const char *p, size_t n) { + for (size_t i = 0; i + 1 < n; i++) + if (static_cast(p[i]) == 0xED && static_cast(p[i + 1]) >= 0xA0) + return true; + return false; +} + +// A runtime string in V8: NewFromUtf8 would replace lone surrogates, so a +// string that holds one decodes WTF-8 to UTF-16 here. +Local FromWtf8(Isolate *isolate, const char *p, size_t n) { + if (!HasLoneSurrogate(p, n)) return Utf8(isolate, p, n); + std::vector units; + units.reserve(n); + for (size_t i = 0; i < n;) { + unsigned char c = static_cast(p[i]); + uint32_t cp; + size_t k; + if (c < 0x80) { cp = c; k = 1; } + else if ((c & 0xE0) == 0xC0 && i + 1 < n) { cp = ((c & 0x1F) << 6) | (p[i + 1] & 0x3F); k = 2; } + else if ((c & 0xF0) == 0xE0 && i + 2 < n) { + cp = ((c & 0x0F) << 12) | ((p[i + 1] & 0x3F) << 6) | (p[i + 2] & 0x3F); k = 3; + } else if ((c & 0xF8) == 0xF0 && i + 3 < n) { + cp = ((c & 0x07) << 18) | ((p[i + 1] & 0x3F) << 12) | ((p[i + 2] & 0x3F) << 6) | (p[i + 3] & 0x3F); + k = 4; + } else { cp = 0xFFFD; k = 1; } + if (cp >= 0x10000) { + cp -= 0x10000; + units.push_back(static_cast(0xD800 + (cp >> 10))); + units.push_back(static_cast(0xDC00 + (cp & 0x3FF))); + } else { + units.push_back(static_cast(cp)); + } + i += k; + } + return String::NewFromTwoByte(isolate, units.data(), NewStringType::kNormal, + static_cast(units.size())) + .ToLocalChecked(); +} + +// A decimal BigInt text in V8, through 64-bit words. +Local BigIntFromText(Local context, const char *p, size_t n) { + int sign = 0; + size_t i = 0; + if (i < n && p[i] == '-') { sign = 1; i++; } + std::vector words(1, 0); + for (; i < n; i++) { + unsigned __int128 carry = static_cast(p[i] - '0'); + for (uint64_t &w : words) { + unsigned __int128 v = static_cast(w) * 10 + carry; + w = static_cast(v); + carry = v >> 64; + } + if (carry != 0) words.push_back(static_cast(carry)); + } + return v8::BigInt::NewFromWords(context, sign, static_cast(words.size()), words.data()) + .ToLocalChecked(); +} + +uint32_t ElemOf(Local view) { + // ScrBytesElem numbering (scr_bytes.h). + if (view->IsUint8Array()) return 0; + if (view->IsUint32Array()) return 1; + if (view->IsFloat32Array()) return 2; + if (view->IsInt32Array()) return 3; + if (view->IsFloat64Array()) return 4; + if (view->IsInt8Array()) return 5; + if (view->IsUint16Array()) return 6; + if (view->IsInt16Array()) return 7; + if (view->IsUint8ClampedArray()) return 8; + if (view->IsFloat16Array()) return 9; + return 0; // DataView: bytes +} + +size_t ElemSize(uint32_t elem) { + static const size_t sizes[] = {1, 4, 4, 4, 8, 1, 2, 2, 1, 2}; + return elem < 10 ? sizes[elem] : 1; +} + +void FromV8(Isolate *isolate, Local context, Local v, ScrNodeValue *out) { + std::memset(out, 0, sizeof *out); + if (v.IsEmpty() || v->IsUndefined()) { out->tag = SCR_NV_UNDEFINED; return; } + if (v->IsNull()) { out->tag = SCR_NV_NULL; return; } + if (v->IsBoolean()) { out->tag = SCR_NV_BOOL; out->u.boolean = v->IsTrue() ? 1 : 0; return; } + if (v->IsNumber()) { out->tag = SCR_NV_NUMBER; out->u.num = v.As()->Value(); return; } + if (v->IsString()) { + out->tag = SCR_NV_STRING; + out->u.str.data = Scratch(Wtf8(isolate, v.As()), &out->u.str.len); + return; + } + if (v->IsBigInt()) { + out->tag = SCR_NV_BIGINT; + Local text = v->ToString(context).ToLocalChecked(); + out->u.str.data = Scratch(Wtf8(isolate, text), &out->u.str.len); + return; + } + if (v->IsArrayBufferView()) { + Local view = v.As(); + bool dataview = view->IsDataView(); + // Buffer() moves on-heap elements off the heap: the data stays put. + Local buffer = view->Buffer(); + std::shared_ptr store = buffer->GetBackingStore(); + if (!store->IsShared() && !store->IsResizableByUserJavaScript()) { + uint32_t elem = dataview ? 0 : ElemOf(view); + out->tag = SCR_NV_BYTES; + out->u.bytes.elem = elem; + out->u.bytes.data = static_cast(store->Data()) + view->ByteOffset(); + out->u.bytes.len = view->ByteLength() / ElemSize(elem); + if (node::Buffer::HasInstance(v)) out->flags |= SCR_NVF_BUFFER; + if (dataview) out->flags |= SCR_NVF_DATAVIEW; + out->u.bytes.handle = Import(isolate, v, store); + return; + } + } + out->tag = SCR_NV_HANDLE; + uint32_t flags = 0; + if (v->IsSymbol()) flags |= SCR_NVF_SYMBOL; + if (v->IsFunction()) { + flags |= SCR_NVF_FUNCTION; + if (v.As()->IsConstructor()) flags |= SCR_NVF_CONSTRUCTOR; + } + if (v->IsNativeError()) flags |= SCR_NVF_ERROR; + if (v->IsArray()) flags |= SCR_NVF_ARRAY; + if (v->IsPromise()) flags |= SCR_NVF_PROMISE; + out->flags = flags; + out->u.handle = Import(isolate, v); +} + +void FreeRuntimeBytes(void *, size_t, void *owner) { + if (owner != nullptr) g_host->bytes_free(owner); +} + +Local ErrorConstructor(Isolate *isolate, Local context, uint32_t kind) { + static const char *const names[] = {"Error", "TypeError", "RangeError", "SyntaxError", + "Error", "ReferenceError", "EvalError", + "URIError"}; + const char *name = kind < 8 ? names[kind] : "Error"; + return context->Global()->Get(context, Utf8(isolate, name)).ToLocalChecked(); +} + +Local ToV8(Isolate *isolate, Local context, const ScrNodeValue *in) { + if (in == nullptr) return v8::Undefined(isolate); + switch (in->tag) { + case SCR_NV_NULL: return v8::Null(isolate); + case SCR_NV_BOOL: return v8::Boolean::New(isolate, in->u.boolean != 0); + case SCR_NV_NUMBER: return v8::Number::New(isolate, in->u.num); + case SCR_NV_STRING: return FromWtf8(isolate, in->u.str.data, in->u.str.len); + case SCR_NV_BIGINT: return BigIntFromText(context, in->u.str.data, in->u.str.len); + case SCR_NV_HANDLE: return HandleValue(isolate, in->u.handle); + case SCR_NV_BYTES: { + if (in->u.bytes.handle != 0) return HandleValue(isolate, in->u.bytes.handle); + size_t elem_size = ElemSize(in->u.bytes.elem); + size_t byte_len = in->u.bytes.len * elem_size; + Local buffer; + if (byte_len == 0 || in->u.bytes.data == nullptr) { + buffer = v8::ArrayBuffer::New(isolate, 0); + if (in->u.bytes.owner != nullptr) g_host->bytes_free(in->u.bytes.owner); + } else { + std::unique_ptr store = v8::ArrayBuffer::NewBackingStore( + in->u.bytes.data, byte_len, FreeRuntimeBytes, in->u.bytes.owner); + buffer = v8::ArrayBuffer::New(isolate, std::move(store)); + } + size_t n = in->u.bytes.len; + if (in->flags & SCR_NVF_DATAVIEW) return v8::DataView::New(buffer, 0, byte_len); + if (in->flags & SCR_NVF_BUFFER) + return node::Buffer::New(isolate, buffer, 0, byte_len).ToLocalChecked(); + switch (in->u.bytes.elem) { + case 1: return v8::Uint32Array::New(buffer, 0, n); + case 2: return v8::Float32Array::New(buffer, 0, n); + case 3: return v8::Int32Array::New(buffer, 0, n); + case 4: return v8::Float64Array::New(buffer, 0, n); + case 5: return v8::Int8Array::New(buffer, 0, n); + case 6: return v8::Uint16Array::New(buffer, 0, n); + case 7: return v8::Int16Array::New(buffer, 0, n); + case 8: return v8::Uint8ClampedArray::New(buffer, 0, n); + case 9: return v8::Float16Array::New(buffer, 0, n); + default: return v8::Uint8Array::New(buffer, 0, n); + } + } + case SCR_NV_ERROR: { + Local ctor = ErrorConstructor(isolate, context, in->u.error.kind); + Local argv[] = {FromWtf8(isolate, in->u.error.message, in->u.error.message_len)}; + Local error = ctor.As()->NewInstance(context, 1, argv).ToLocalChecked(); + std::string name(in->u.error.name, in->u.error.name_len); + Local default_name; + if (!error->Get(context, Utf8(isolate, "name")).ToLocal(&default_name) || + ToStdString(isolate, default_name) != name) + error->Set(context, Utf8(isolate, "name"), FromWtf8(isolate, name.data(), name.size())).Check(); + if (in->u.error.stack != nullptr) + error->Set(context, Utf8(isolate, "stack"), + FromWtf8(isolate, in->u.error.stack, in->u.error.stack_len)).Check(); + if (in->u.error.code != nullptr) + error->Set(context, Utf8(isolate, "code"), + FromWtf8(isolate, in->u.error.code, in->u.error.code_len)).Check(); + return error; + } + case SCR_NV_JSON: { + Local parsed; + if (v8::JSON::Parse(context, FromWtf8(isolate, in->u.str.data, in->u.str.len)).ToLocal(&parsed)) + return parsed; + return v8::Undefined(isolate); + } + default: return v8::Undefined(isolate); + } +} + +// One entry from the runtime: the isolate scope, V8's stack limit for the +// caller's stack, and the out-value scratch (reset by outermost entries). +struct Op { + Enter e; + TryCatch try_catch; + Op() : try_catch(e.isolate) { + if (g_entry_depth++ == 0) g_scratch.clear(); + } + ~Op() { g_entry_depth--; } + // The operation's outcome: 0 and the result, or 1 and the thrown value. + int Finish(Local result, bool ok, ScrNodeValue *out) { + if (ok) { + FromV8(e.isolate, e.context, result, out); + return 0; + } + if (try_catch.HasCaught() && !try_catch.HasTerminated()) { + Local exception = try_catch.Exception(); + try_catch.Reset(); + FromV8(e.isolate, e.context, exception, out); + } else { + std::memset(out, 0, sizeof *out); + out->tag = SCR_NV_UNDEFINED; + } + return 1; + } + Local V(const ScrNodeValue *in) { return ToV8(e.isolate, e.context, in); } +}; + +int Load(const char *path, int esm, ScrNodeValue *out) { + Op op; + Local argv[] = {Utf8(op.e.isolate, path), v8::Boolean::New(op.e.isolate, esm != 0)}; + Local result; + bool ok = g_load.Get(op.e.isolate) + ->Call(op.e.context, v8::Undefined(op.e.isolate), 2, argv) + .ToLocal(&result); + return op.Finish(result, ok, out); +} + +// The target of a property operation: an object, or a primitive whose +// wrapper answers (ToObject throws for undefined and null). +bool Target(Op &op, uint32_t h, Local *target) { + return HandleValue(op.e.isolate, h)->ToObject(op.e.context).ToLocal(target); +} + +bool Key(Op &op, const ScrNodeValue *key, Local *out) { + Local k = op.V(key); + if (k->IsString() || k->IsSymbol() || k->IsNumber()) { + *out = k; + return true; + } + Local name; + if (!k->ToString(op.e.context).ToLocal(&name)) return false; + *out = name; + return true; +} + +int Get(uint32_t h, const ScrNodeValue *key, ScrNodeValue *out) { + Op op; + Local target; + Local k, result; + bool ok = Target(op, h, &target) && Key(op, key, &k) && target->Get(op.e.context, k).ToLocal(&result); + return op.Finish(result, ok, out); +} + +int SetProp(uint32_t h, const ScrNodeValue *key, const ScrNodeValue *value, ScrNodeValue *out) { + Op op; + Local target; + Local k; + bool ok = Target(op, h, &target) && Key(op, key, &k); + if (ok) { + v8::Maybe done = target->Set(op.e.context, k, op.V(value)); + ok = done.IsJust(); + // Strict-mode semantics: a refused assignment throws, as in a module. + if (ok && !done.FromJust()) { + op.e.isolate->ThrowException(v8::Exception::TypeError( + Utf8(op.e.isolate, "Cannot assign to read only property"))); + ok = false; + } + } + return op.Finish(v8::Undefined(op.e.isolate), ok, out); +} + +int Has(uint32_t h, const ScrNodeValue *key, ScrNodeValue *out) { + Op op; + Local target = HandleValue(op.e.isolate, h); + if (!target->IsObject()) { + op.e.isolate->ThrowException(v8::Exception::TypeError( + Utf8(op.e.isolate, "Cannot use 'in' operator to search for a key in a primitive"))); + return op.Finish(Local(), false, out); + } + Local k; + bool ok = Key(op, key, &k); + v8::Maybe found = ok ? target.As()->Has(op.e.context, k) : v8::Nothing(); + ok = found.IsJust(); + return op.Finish(v8::Boolean::New(op.e.isolate, ok && found.FromJust()), ok, out); +} + +int Del(uint32_t h, const ScrNodeValue *key, ScrNodeValue *out) { + Op op; + Local target; + Local k; + bool ok = Target(op, h, &target) && Key(op, key, &k); + v8::Maybe done = ok ? target->Delete(op.e.context, k) : v8::Nothing(); + ok = done.IsJust(); + if (ok && !done.FromJust()) { + op.e.isolate->ThrowException(v8::Exception::TypeError( + Utf8(op.e.isolate, "Cannot delete property"))); + ok = false; + } + return op.Finish(v8::Boolean::New(op.e.isolate, ok), ok, out); +} + +std::vector> Args(Op &op, const ScrNodeValue *args, size_t argc) { + std::vector> argv; + argv.reserve(argc); + for (size_t i = 0; i < argc; i++) argv.push_back(op.V(&args[i])); + return argv; +} + +void NotAFunction(Op &op, const char *what) { + op.e.isolate->ThrowException(v8::Exception::TypeError( + Utf8(op.e.isolate, (std::string(what) + " is not a function").c_str()))); +} + +int Invoke(uint32_t fn, const ScrNodeValue *this_, const ScrNodeValue *args, size_t argc, + ScrNodeValue *out) { + Op op; + Local callee = HandleValue(op.e.isolate, fn); + if (!callee->IsFunction()) { + NotAFunction(op, "value"); + return op.Finish(Local(), false, out); + } + std::vector> argv = Args(op, args, argc); + Local result; + bool ok = callee.As() + ->Call(op.e.context, op.V(this_), static_cast(argv.size()), argv.data()) + .ToLocal(&result); + return op.Finish(result, ok, out); +} + +int InvokeMethod(uint32_t h, const ScrNodeValue *key, const ScrNodeValue *args, size_t argc, + ScrNodeValue *out) { + Op op; + Local receiver = HandleValue(op.e.isolate, h); + Local target; + Local k, callee; + if (!receiver->ToObject(op.e.context).ToLocal(&target) || !Key(op, key, &k) || + !target->Get(op.e.context, k).ToLocal(&callee)) + return op.Finish(Local(), false, out); + if (!callee->IsFunction()) { + std::string what = "property"; + if (k->IsString()) what = ToStdString(op.e.isolate, k); + NotAFunction(op, what.c_str()); + return op.Finish(Local(), false, out); + } + std::vector> argv = Args(op, args, argc); + Local result; + bool ok = callee.As() + ->Call(op.e.context, receiver, static_cast(argv.size()), argv.data()) + .ToLocal(&result); + return op.Finish(result, ok, out); +} + +int Construct(uint32_t h, const ScrNodeValue *args, size_t argc, ScrNodeValue *out) { + Op op; + Local callee = HandleValue(op.e.isolate, h); + if (!callee->IsFunction() || !callee.As()->IsConstructor()) { + op.e.isolate->ThrowException(v8::Exception::TypeError( + Utf8(op.e.isolate, "value is not a constructor"))); + return op.Finish(Local(), false, out); + } + std::vector> argv = Args(op, args, argc); + Local result; + bool ok = callee.As() + ->NewInstance(op.e.context, static_cast(argv.size()), argv.data()) + .ToLocal(&result); + return op.Finish(result, ok, out); +} + +int InstanceOf(const ScrNodeValue *value, uint32_t ctor, ScrNodeValue *out) { + Op op; + Local v = op.V(value); + Local c = HandleValue(op.e.isolate, ctor); + if (!c->IsObject()) { + op.e.isolate->ThrowException(v8::Exception::TypeError( + Utf8(op.e.isolate, "Right-hand side of 'instanceof' is not callable"))); + return op.Finish(Local(), false, out); + } + v8::Maybe is = v->InstanceOf(op.e.context, c.As()); + bool ok = is.IsJust(); + return op.Finish(v8::Boolean::New(op.e.isolate, ok && is.FromJust()), ok, out); +} + +int OwnKeys(uint32_t h, int mode, ScrNodeValue *out) { + Op op; + Local target; + Local keys; + bool ok = Target(op, h, &target) && + target + ->GetOwnPropertyNames( + op.e.context, + mode == 0 ? static_cast(v8::ONLY_ENUMERABLE | v8::SKIP_SYMBOLS) + : v8::ALL_PROPERTIES, + v8::KeyConversionMode::kConvertToString) + .ToLocal(&keys); + return op.Finish(keys, ok, out); +} + +void OnSettled(const FunctionCallbackInfo &info, int rejected) { + Isolate *isolate = info.GetIsolate(); + Local context = isolate->GetCurrentContext(); + uint32_t token = info.Data()->Uint32Value(context).FromMaybe(0); + ScrNodeValue value; + FromV8(isolate, context, info[0], &value); + g_host->settle_value(token, rejected, &value); +} + +void OnFulfilled(const FunctionCallbackInfo &info) { OnSettled(info, 0); } +void OnRejected(const FunctionCallbackInfo &info) { OnSettled(info, 1); } + +// await of a value: PromiseResolve (a native promise is itself) and one +// reaction, as `await` does in JavaScript. +int Then(const ScrNodeValue *value, uint32_t token, ScrNodeValue *out) { + Op op; + Local v = op.V(value); + Local promise; + if (v->IsPromise()) { + promise = v.As(); + } else { + Local resolver; + if (!v8::Promise::Resolver::New(op.e.context).ToLocal(&resolver) || + resolver->Resolve(op.e.context, v).IsNothing()) + return op.Finish(Local(), false, out); + promise = resolver->GetPromise(); + } + Local data = Integer::NewFromUnsigned(op.e.isolate, token); + Local fulfilled, rejected; + Local chained; + bool ok = Function::New(op.e.context, OnFulfilled, data).ToLocal(&fulfilled) && + Function::New(op.e.context, OnRejected, data).ToLocal(&rejected) && + promise->Then(op.e.context, fulfilled, rejected).ToLocal(&chained); + return op.Finish(v8::Undefined(op.e.isolate), ok, out); +} + +void ErrorInfo(uint32_t h, ScrNodeErrorInfo *out) { + Op op; + std::memset(out, 0, sizeof *out); + Local v = HandleValue(op.e.isolate, h); + if (!v->IsObject()) return; + Local error = v.As(); + static const struct { const char *name; uint32_t kind; } kinds[] = { + {"TypeError", 1}, {"RangeError", 2}, {"SyntaxError", 3}, + {"ReferenceError", 5}, {"EvalError", 6}, {"URIError", 7}}; + for (const auto &k : kinds) { + Local ctor = op.e.context->Global()->Get(op.e.context, Utf8(op.e.isolate, k.name)).ToLocalChecked(); + if (error->InstanceOf(op.e.context, ctor.As()).FromMaybe(false)) { + out->kind = k.kind; + break; + } + } + auto read = [&](const char *key, const char **data, size_t *len, bool optional) { + Local value; + if (!error->Get(op.e.context, Utf8(op.e.isolate, key)).ToLocal(&value)) { + op.try_catch.Reset(); + return; + } + if (optional && !value->IsString()) return; + Local text; + if (!value->ToString(op.e.context).ToLocal(&text)) { + op.try_catch.Reset(); + return; + } + *data = Scratch(Wtf8(op.e.isolate, text), len); + }; + read("name", &out->name, &out->name_len, false); + read("message", &out->message, &out->message_len, false); + read("stack", &out->stack, &out->stack_len, true); + read("code", &out->code, &out->code_len, true); +} + +void DropHandle(uint32_t h) { + if (!g_setup) return; + Isolate *isolate = g_setup->isolate(); + HandleScope handles(isolate); + Drop(h); +} + const ScrNodeBridge kBridge = { - SCR_NODE_BRIDGE_ABI, NODE_VERSION_STRING, NODE_MODULE_VERSION, Start, Call, Release, - BackendFd, BackendTimeout, Alive, Pump, Stop, - Finish, + SCR_NODE_BRIDGE_ABI, + NODE_VERSION_STRING, + NODE_MODULE_VERSION, + Start, + Call, + Release, + HostLoop, + Microtask, + Tick, + Timer, + TimerClear, + TimerRef, + TimerRefresh, + Immediate, + ImmediateClear, + ImmediateRef, + Checkpoint, + RunLoop, + Emit, + Stop, + Load, + Get, + SetProp, + Has, + Del, + Invoke, + InvokeMethod, + Construct, + InstanceOf, + OwnKeys, + Then, + ErrorInfo, + DropHandle, + LiveHandles, }; } // namespace diff --git a/packages/libnode-common/build-pack.mjs b/packages/libnode-common/build-pack.mjs index 7b5c2ac8f..53d6c41a1 100644 --- a/packages/libnode-common/build-pack.mjs +++ b/packages/libnode-common/build-pack.mjs @@ -1,8 +1,7 @@ #!/usr/bin/env node /* Assembles a libnode pack: the directory a scriptc program with Node - * islands loads Node from (scope-B design §2.16.3). Until the pack CI - * matrix (NI-13) publishes @scriptc/libnode- packages, packs are - * assembled by hand from a Node `--shared` build and the matching headers: + * islands loads Node from (scope-B design §2.16.3), from a Node `--shared` + * build and the headers of the same release: * * node packages/libnode-common/build-pack.mjs \ * --libnode or libnode..dylib> \ @@ -10,21 +9,32 @@ * --out [--license ] [--cxx ] * * The pack holds libnode, the bridge (bridge/scr_node_bridge.cc) compiled - * against those headers, Node's license and manifest.json, which the build - * of a program with Node islands reads (packages/compiler/src/node-islands/ - * libnode.ts). Programs find a pack through SCRIPTC_LIBNODE or --libnode. */ + * against those headers and linked to libnode through $ORIGIN / + * @loader_path (so the directory can move), Node's license and + * manifest.json, which the build of a program with Node islands reads + * (packages/compiler/src/node-islands/libnode.ts). Programs find a pack + * through --libnode, SCRIPTC_LIBNODE, or the installed + * @scriptc/libnode- package. + * + * The @scriptc/libnode- packs call buildPack() from + * scripts/build.mjs after building libnode from the pinned release; run by + * hand, this is the bridge-only rebuild path (seconds) and the way a + * self-builder pairs the bridge with a compatible Node build of their own. */ import { execFileSync } from "node:child_process"; import { copyFileSync, mkdirSync, readFileSync, statSync, writeFileSync } from "node:fs"; import { createHash } from "node:crypto"; -import { basename, join, resolve } from "node:path"; +import { basename, dirname, join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { parseArgs } from "node:util"; const here = fileURLToPath(new URL(".", import.meta.url)); const runtimeSrc = join(here, "../runtime/src"); +/** The manifest schema; the compiler's pack reader accepts exactly this. */ +export const LIBNODE_PACK_SCHEMA = "scriptc.libnode-pack.v1"; + /** The bridge ABI the runtime expects (scr_node_bridge.h). */ -function bridgeAbi() { +export function bridgeAbi() { const header = readFileSync(join(runtimeSrc, "scr_node_bridge.h"), "utf8"); return Number(/#define SCR_NODE_BRIDGE_ABI (\d+)u/.exec(header)[1]); } @@ -46,38 +56,41 @@ function fileEntry(path) { }; } -if (process.argv[1] === fileURLToPath(import.meta.url)) { - const { values } = parseArgs({ - options: { - libnode: { type: "string" }, - headers: { type: "string" }, - out: { type: "string" }, - license: { type: "string" }, - cxx: { type: "string", default: process.platform === "darwin" ? "clang++" : "g++" }, - }, - }); - if (!values.libnode || !values.headers || !values.out) { - console.error("usage: build-pack.mjs --libnode --headers --out "); - process.exit(2); - } - const out = resolve(values.out); - const headers = resolve(values.headers); - mkdirSync(out, { recursive: true }); - const libnodeName = basename(values.libnode); - copyFileSync(values.libnode, join(out, libnodeName)); - const bridgeName = - process.platform === "darwin" ? "libscr_node_bridge.dylib" : "libscr_node_bridge.so"; - const link = - process.platform === "darwin" - ? ["-dynamiclib", join(out, libnodeName), "-Wl,-rpath,@loader_path"] - : ["-shared", `-L${out}`, `-l:${libnodeName}`, "-Wl,-rpath,$ORIGIN"]; +/** The bridge library name on a platform. */ +export function bridgeName(os = process.platform) { + return os === "darwin" ? "libscr_node_bridge.dylib" : "libscr_node_bridge.so"; +} + +/** Compiles the bridge against `headers` (an include/node directory) and + * links it to `libnode`, which must already sit in the bridge's directory. */ +export function buildBridge({ headers, libnode, out, cxx }) { + const darwin = process.platform === "darwin"; + const compiler = cxx ?? process.env.CXX ?? (darwin ? "clang++" : "g++"); + const link = darwin + ? [ + "-dynamiclib", + libnode, + "-Wl,-rpath,@loader_path", + `-Wl,-install_name,@rpath/${basename(out)}`, + ] + : [ + "-shared", + `-L${dirname(libnode)}`, + `-l:${basename(libnode)}`, + "-Wl,-rpath,$ORIGIN", + "-Wl,-z,defs", + "-Wl,--as-needed", + "-ldl", + "-pthread", + ]; execFileSync( - values.cxx, + compiler, [ "-std=c++20", "-O2", "-fPIC", "-fvisibility=hidden", + "-fno-rtti", "-I", runtimeSrc, "-I", @@ -85,24 +98,79 @@ if (process.argv[1] === fileURLToPath(import.meta.url)) { join(here, "bridge/scr_node_bridge.cc"), ...link, "-o", - join(out, bridgeName), + out, ], { stdio: "inherit" }, ); - if (values.license) copyFileSync(values.license, join(out, "LICENSE")); + if (darwin) { + execFileSync("strip", ["-x", out], { stdio: "inherit" }); + // Stripping invalidated the linker's ad-hoc signature, which arm64 + // macOS requires to map the library. + execFileSync("codesign", ["--force", "--sign", "-", out], { stdio: "inherit" }); + } else { + execFileSync("strip", ["--strip-unneeded", out], { stdio: "inherit" }); + } +} + +/** + * Assembles the pack in `out` and returns its manifest. `extra` adds fields + * to manifest.json (the published packs record their package, target and + * build inputs); schema, nodeVersion, moduleVersion, bridgeAbi, platform, + * arch, libnode, bridge and files are always present. + * + * @param {{ libnode: string, headers: string, out: string, license?: string, + * cxx?: string, extra?: Record }} options + */ +export function buildPack(options) { + const out = resolve(options.out); + const headers = resolve(options.headers); + mkdirSync(out, { recursive: true }); + const libnodeName = basename(options.libnode); + const libnode = join(out, libnodeName); + if (resolve(options.libnode) !== libnode) copyFileSync(options.libnode, libnode); + const bridge = join(out, bridgeName()); + buildBridge({ headers, libnode, out: bridge, cxx: options.cxx }); + if (options.license) copyFileSync(options.license, join(out, "LICENSE")); const { nodeVersion, moduleVersion } = nodeVersionOf(headers); const manifest = { - schema: "scriptc.libnode-pack.v1", + schema: LIBNODE_PACK_SCHEMA, nodeVersion, moduleVersion, bridgeAbi: bridgeAbi(), platform: process.platform, arch: process.arch, + libnode: libnodeName, + bridge: bridgeName(), + ...options.extra, files: { - [libnodeName]: fileEntry(join(out, libnodeName)), - [bridgeName]: fileEntry(join(out, bridgeName)), + [libnodeName]: fileEntry(libnode), + [bridgeName()]: fileEntry(bridge), }, }; writeFileSync(join(out, "manifest.json"), `${JSON.stringify(manifest, null, 2)}\n`); - console.log(`libnode pack for Node ${nodeVersion} in ${out}`); + return manifest; +} + +if (process.argv[1] === fileURLToPath(import.meta.url)) { + const { values } = parseArgs({ + options: { + libnode: { type: "string" }, + headers: { type: "string" }, + out: { type: "string" }, + license: { type: "string" }, + cxx: { type: "string" }, + }, + }); + if (!values.libnode || !values.headers || !values.out) { + console.error("usage: build-pack.mjs --libnode --headers --out "); + process.exit(2); + } + const manifest = buildPack({ + libnode: values.libnode, + headers: values.headers, + out: values.out, + license: values.license, + cxx: values.cxx, + }); + console.log(`libnode pack for Node ${manifest.nodeVersion} in ${resolve(values.out)}`); } diff --git a/packages/libnode-common/fetch-dev-pack.mjs b/packages/libnode-common/fetch-dev-pack.mjs index 5c4bafc90..9d3a0512f 100644 --- a/packages/libnode-common/fetch-dev-pack.mjs +++ b/packages/libnode-common/fetch-dev-pack.mjs @@ -2,8 +2,9 @@ /* Assembles a development libnode pack from the pinned, checksum-verified * inputs in dev-pack.json: a prebuilt libnode of the repository's Node * release and Node's official headers, with the bridge built against them - * by build-pack.mjs. CI's Linux test lanes use it (.github/actions/ - * libnode-dev-pack) until the NI-13 packs are published. + * by build-pack.mjs. CI's Linux x64 and macOS test lanes fall back to it + * (.github/actions/libnode) until the libnode-packs workflow has cached a + * build of @scriptc/libnode- for their ref. * * node packages/libnode-common/fetch-dev-pack.mjs --out */ import { execFileSync } from "node:child_process"; diff --git a/packages/libnode-common/libnode.json b/packages/libnode-common/libnode.json new file mode 100644 index 000000000..9d51abff1 --- /dev/null +++ b/packages/libnode-common/libnode.json @@ -0,0 +1,7 @@ +{ + "nodeVersion": "24.22.0", + "tarballSha256": "41f97cff2c666f1f8293dc5f85c65e8dd605c16b43fd9febd49ede32ee1e0329", + "moduleVersion": 137, + "configureFlags": ["--shared", "--ninja"], + "icu": "full" +} diff --git a/packages/libnode-common/package.json b/packages/libnode-common/package.json new file mode 100644 index 000000000..042f3a7d9 --- /dev/null +++ b/packages/libnode-common/package.json @@ -0,0 +1,6 @@ +{ + "name": "@scriptc/libnode-common", + "version": "0.0.1", + "private": true, + "type": "module" +} diff --git a/packages/libnode-common/scripts/build.mjs b/packages/libnode-common/scripts/build.mjs new file mode 100644 index 000000000..020265bec --- /dev/null +++ b/packages/libnode-common/scripts/build.mjs @@ -0,0 +1,336 @@ +#!/usr/bin/env node +// Builds one libnode pack into its package directory: +// +// 1. libnode: the pinned Node.js release (libnode.json) from its official +// source tarball, SHA-256 verified, configured `--shared` with full ICU, +// only the `libnode` target built, stripped, staged with its headers +// and LICENSE in a cache directory keyed by libnodeCacheKey(). A cache +// hit skips the Node build entirely (about 26 minutes on 8 cores). +// 2. the bridge, compiled against those headers (seconds, every run); +// 3. the pack: libnode, the bridge, Node's LICENSE and manifest.json in +// packages/libnode-/, plus `--compress` copies (.zst) of both +// libraries in /compressed/ for size reports and embedding. +// +// This never runs in `pnpm -r build` or the test suites: each platform +// package runs it through its explicit `build:native` script, and the +// libnode-packs workflow runs that on matching hosts. +// +// Environment: +// SCRIPTC_LIBNODE_CACHE cache root (default node_modules/.cache/scriptc-libnode) +// SCRIPTC_LIBNODE_SOURCE an extracted source tree of the pinned release to +// build in (seeds the cache from an earlier build tree) +// SCRIPTC_LIBNODE_TOOLCHAIN a name for the toolchain, part of the cache key +// (CI: "manylinux_2_28-gcc14", "macos-15-clang") +// SCRIPTC_LIBNODE_JOBS ninja parallelism (default: available cores) +// CC, CXX the compilers for Node and the bridge +import { execFileSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { + copyFileSync, + createReadStream, + existsSync, + mkdirSync, + readFileSync, + renameSync, + rmSync, + statSync, + writeFileSync, +} from "node:fs"; +import { availableParallelism } from "node:os"; +import { join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { parseArgs } from "node:util"; +import { buildPack } from "../build-pack.mjs"; +import { + bridgeAbi, + bridgeHash, + libnodeCacheKey, + packFileNames, + platformInfo, + readPin, + repoRoot, +} from "./pin.mjs"; + +function log(message) { + process.stdout.write(`libnode: ${message}\n`); +} + +function run(command, args, options = {}) { + log(`$ ${command} ${args.join(" ")}`); + execFileSync(command, args, { stdio: "inherit", ...options }); +} + +async function sha256File(path) { + const hash = createHash("sha256"); + for await (const chunk of createReadStream(path)) hash.update(chunk); + return hash.digest("hex"); +} + +/** Downloads and verifies the pinned source tarball, then extracts it. */ +async function fetchSource(cacheRoot, pin) { + const name = `node-v${pin.nodeVersion}`; + const tree = join(cacheRoot, "src", name); + if (existsSync(join(tree, "configure.py"))) return tree; + const tarball = join(cacheRoot, "src", `${name}.tar.xz`); + mkdirSync(join(cacheRoot, "src"), { recursive: true }); + if (!existsSync(tarball) || (await sha256File(tarball)) !== pin.tarballSha256) { + const url = `https://nodejs.org/dist/v${pin.nodeVersion}/${name}.tar.xz`; + run("curl", ["-fsSL", "--retry", "3", "-o", `${tarball}.part`, url]); + renameSync(`${tarball}.part`, tarball); + } + const actual = await sha256File(tarball); + if (actual !== pin.tarballSha256) { + rmSync(tarball, { force: true }); + throw new Error(`${name}.tar.xz has SHA-256 ${actual}, expected ${pin.tarballSha256}`); + } + rmSync(tree, { recursive: true, force: true }); + run("tar", ["-xJf", tarball, "-C", join(cacheRoot, "src")]); + return tree; +} + +/** The version a source tree's node_version.h declares. */ +function sourceVersion(tree) { + const text = readFileSync(join(tree, "src", "node_version.h"), "utf8"); + const define = (name) => new RegExp(`#define ${name} (\\d+)`).exec(text)?.[1]; + return `${define("NODE_MAJOR_VERSION")}.${define("NODE_MINOR_VERSION")}.${define("NODE_PATCH_VERSION")}`; +} + +function moduleVersionOf(headers) { + const text = readFileSync(join(headers, "include", "node", "node_version.h"), "utf8"); + return Number(/#define NODE_MODULE_VERSION (\d+)/.exec(text)?.[1]); +} + +/** The highest GLIBC_x.y symbol version a Linux library needs. */ +function minimumGlibc(library) { + const dynamic = execFileSync("objdump", ["-T", library], { + encoding: "utf8", + maxBuffer: 256 * 1024 * 1024, + }); + const versions = [...new Set(dynamic.match(/GLIBC_\d+(\.\d+)+/g) ?? [])].map((v) => + v.slice("GLIBC_".length), + ); + versions.sort((a, b) => { + const x = a.split(".").map(Number); + const y = b.split(".").map(Number); + for (let i = 0; i < Math.max(x.length, y.length); i++) { + if ((x[i] ?? 0) !== (y[i] ?? 0)) return (x[i] ?? 0) - (y[i] ?? 0); + } + return 0; + }); + return versions.at(-1) ?? null; +} + +/** Configures and builds libnode in `tree`, then stages it into `stage`. */ +function buildLibnode(tree, stage, platform, pin) { + const { os } = platformInfo(platform); + if (sourceVersion(tree) !== pin.nodeVersion) { + throw new Error(`${tree} is Node ${sourceVersion(tree)}, the pin is ${pin.nodeVersion}`); + } + const python = process.env.PYTHON ?? "python3"; + // An earlier tree configured with exactly these flags is reused as is, so + // a seeded build tree is a ninja no-op instead of a reconfigure. + const status = join(tree, "config.status"); + const configured = + existsSync(status) && + existsSync(join(tree, "out", "Release", "build.ninja")) && + readFileSync(status, "utf8") + .trimEnd() + .endsWith(`configure ${pin.configureFlags.join(" ")}`); + const started = Date.now(); + if (!configured) run("./configure", pin.configureFlags, { cwd: tree }); + const gypi = readFileSync(join(tree, "config.gypi"), "utf8"); + for (const [setting, expected] of [ + ["node_shared", "true"], + ["node_use_node_snapshot", "true"], + ["icu_small", pin.icu === "small" ? "true" : "false"], + ]) { + if (!gypi.includes(`"${setting}": "${expected}"`)) { + throw new Error(`config.gypi does not have ${setting}: ${expected}`); + } + } + const jobs = process.env.SCRIPTC_LIBNODE_JOBS ?? String(availableParallelism()); + run("ninja", ["-C", join(tree, "out", "Release"), "-j", jobs, "libnode"]); + const buildSeconds = Math.round((Date.now() - started) / 1000); + + const names = packFileNames(platform, pin.moduleVersion); + const built = [ + join(tree, "out", "Release", "lib", names.libnode), + join(tree, "out", "Release", names.libnode), + ].find((path) => existsSync(path)); + if (built === undefined) throw new Error(`the Node build did not produce ${names.libnode}`); + const partial = `${stage}.partial`; + rmSync(partial, { recursive: true, force: true }); + mkdirSync(join(partial, "lib"), { recursive: true }); + const library = join(partial, "lib", names.libnode); + if (os === "darwin") { + copyFileSync(built, library); + run("install_name_tool", ["-id", `@rpath/${names.libnode}`, library]); + // -x keeps the global symbols addons and the bridge bind to. + run("strip", ["-x", library]); + // Editing the library invalidated the linker's ad-hoc signature, which + // arm64 macOS requires to map it. + run("codesign", ["--force", "--sign", "-", library]); + } else { + run("strip", ["--strip-unneeded", "-o", library, built]); + } + run( + python, + [ + join(tree, "tools", "install.py"), + "install", + "--headers-only", + "--silent", + "--root-dir", + tree, + "--dest-dir", + join(partial, "headers"), + "--prefix", + "/", + ], + { cwd: tree }, + ); + const moduleVersion = moduleVersionOf(join(partial, "headers")); + if (moduleVersion !== pin.moduleVersion) { + throw new Error( + `Node ${pin.nodeVersion} has NODE_MODULE_VERSION ${moduleVersion}, the pin says ${pin.moduleVersion}`, + ); + } + copyFileSync(join(tree, "LICENSE"), join(partial, "LICENSE")); + writeFileSync( + join(partial, "build.json"), + `${JSON.stringify( + { + nodeVersion: pin.nodeVersion, + moduleVersion, + configureFlags: pin.configureFlags, + icu: pin.icu, + toolchain: process.env.SCRIPTC_LIBNODE_TOOLCHAIN ?? "default", + buildSeconds, + builtBytes: statSync(built).size, + minimumGlibc: os === "linux" ? minimumGlibc(library) : null, + }, + null, + 2, + )}\n`, + ); + rmSync(stage, { recursive: true, force: true }); + renameSync(partial, stage); +} + +/** zstd copies of the pack's libraries, for size reports and embedding. */ +function compress(pack, names, out) { + mkdirSync(out, { recursive: true }); + const sizes = {}; + for (const name of [names.libnode, names.bridge]) { + const target = join(out, `${name}.zst`); + const started = Date.now(); + run("zstd", ["-19", "-T0", "-q", "-f", join(pack, name), "-o", target]); + sizes[name] = { zstdBytes: statSync(target).size, seconds: (Date.now() - started) / 1000 }; + } + return sizes; +} + +/** + * @param {{ platform: string, packageRoot: string, compress?: boolean, skipLibnode?: boolean }} options + */ +export async function buildLibnodePack(options) { + const { platform, packageRoot } = options; + const { os, arch } = platformInfo(platform); + const pin = readPin(); + const cacheRoot = resolve( + process.env.SCRIPTC_LIBNODE_CACHE ?? + join(repoRoot, "node_modules", ".cache", "scriptc-libnode"), + ); + const key = libnodeCacheKey(platform); + const stage = join(cacheRoot, `${platform}-${pin.nodeVersion}-${key.slice(0, 16)}`); + const names = packFileNames(platform, pin.moduleVersion); + if (existsSync(join(stage, "build.json"))) { + log(`cached libnode ${pin.nodeVersion} for ${platform} in ${stage}`); + } else { + if (options.skipLibnode) throw new Error(`no cached libnode build in ${stage}`); + const tree = process.env.SCRIPTC_LIBNODE_SOURCE + ? resolve(process.env.SCRIPTC_LIBNODE_SOURCE) + : await fetchSource(cacheRoot, pin); + buildLibnode(tree, stage, platform, pin); + } + const build = JSON.parse(readFileSync(join(stage, "build.json"), "utf8")); + + for (const name of [names.libnode, names.bridge, "LICENSE", "manifest.json"]) { + rmSync(join(packageRoot, name), { force: true }); + } + const packageJson = JSON.parse(readFileSync(join(packageRoot, "package.json"), "utf8")); + const bridgeStarted = Date.now(); + const manifest = buildPack({ + libnode: join(stage, "lib", names.libnode), + headers: join(stage, "headers", "include", "node"), + out: packageRoot, + license: join(stage, "LICENSE"), + extra: { + package: packageJson.name, + version: packageJson.version, + target: platform, + license: "LICENSE", + build: { + libnodeKey: key, + bridgeHash: bridgeHash(), + configureFlags: build.configureFlags, + icu: build.icu, + toolchain: build.toolchain, + minimumGlibc: build.minimumGlibc, + }, + }, + }); + const bridgeSeconds = (Date.now() - bridgeStarted) / 1000; + if ( + manifest.nodeVersion !== pin.nodeVersion || + manifest.moduleVersion !== pin.moduleVersion || + manifest.bridgeAbi !== bridgeAbi() || + manifest.platform !== os || + manifest.arch !== arch || + manifest.libnode !== names.libnode || + manifest.bridge !== names.bridge + ) { + throw new Error(`the assembled pack does not match ${platform}: ${JSON.stringify(manifest)}`); + } + const report = { + platform, + nodeVersion: pin.nodeVersion, + libnodeBuildSeconds: build.buildSeconds, + libnodeBuiltBytes: build.builtBytes, + libnodeStrippedBytes: manifest.files[names.libnode].size, + bridgeBytes: manifest.files[names.bridge].size, + bridgeSeconds, + minimumGlibc: manifest.build.minimumGlibc, + }; + if (options.compress) { + report.compressed = compress(packageRoot, names, join(cacheRoot, "compressed", platform)); + } + log(`pack ${packageJson.name} for Node ${pin.nodeVersion}: ${JSON.stringify(report)}`); + return report; +} + +if (process.argv[1] === fileURLToPath(import.meta.url)) { + const { values } = parseArgs({ + options: { + platform: { type: "string" }, + "package-root": { type: "string" }, + compress: { type: "boolean", default: false }, + "skip-libnode": { type: "boolean", default: false }, + report: { type: "string" }, + }, + }); + if (!values.platform) { + console.error("usage: build.mjs --platform [--package-root ] [--compress]"); + process.exit(2); + } + const packageRoot = resolve( + values["package-root"] ?? join(repoRoot, "packages", `libnode-${values.platform}`), + ); + const report = await buildLibnodePack({ + platform: values.platform, + packageRoot, + compress: values.compress, + skipLibnode: values["skip-libnode"], + }); + if (values.report) writeFileSync(values.report, `${JSON.stringify(report, null, 2)}\n`); +} diff --git a/packages/libnode-common/scripts/cache-key.mjs b/packages/libnode-common/scripts/cache-key.mjs new file mode 100644 index 000000000..6ca07a626 --- /dev/null +++ b/packages/libnode-common/scripts/cache-key.mjs @@ -0,0 +1,29 @@ +#!/usr/bin/env node +// Prints the CI cache identity of one platform's libnode build as +// GitHub Actions outputs: `key` (Node release, configure flags, platform, +// toolchain), `path` (the staged build directory under the cache root, +// relative to it) and `bridge` (the bridge source hash, which is not part +// of the key: a bridge change reuses the cached libnode). +// +// node packages/libnode-common/scripts/cache-key.mjs --platform

[--toolchain ] +import { parseArgs } from "node:util"; +import { bridgeHash, libnodeCacheKey, readPin } from "./pin.mjs"; + +const { values } = parseArgs({ + options: { platform: { type: "string" }, toolchain: { type: "string" } }, +}); +if (!values.platform) { + console.error("usage: cache-key.mjs --platform [--toolchain ]"); + process.exit(2); +} +const pin = readPin(); +const key = libnodeCacheKey(values.platform, values.toolchain); +process.stdout.write( + [ + `key=libnode-${values.platform}-${pin.nodeVersion}-${key}`, + `path=${values.platform}-${pin.nodeVersion}-${key.slice(0, 16)}`, + `bridge=${bridgeHash()}`, + `node=${pin.nodeVersion}`, + "", + ].join("\n"), +); diff --git a/packages/libnode-common/scripts/pin.mjs b/packages/libnode-common/scripts/pin.mjs new file mode 100644 index 000000000..89783c76e --- /dev/null +++ b/packages/libnode-common/scripts/pin.mjs @@ -0,0 +1,90 @@ +// The libnode pin and the platform table shared by the libnode pack build, +// its CI cache keys, the pack verifier, and the smoke test. +// +// A libnode pack (`@scriptc/libnode-`) holds a stripped Node.js +// shared library built from the pinned release, the C++ bridge compiled +// against that release's headers, Node's license, and manifest.json. The +// scriptc runtime dlopens the bridge on its first call into Node and talks +// to it only through the C ABI in packages/runtime/src/scr_node_bridge.h. +import { createHash } from "node:crypto"; +import { readFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; + +export const commonRoot = dirname(dirname(fileURLToPath(import.meta.url))); +export const repoRoot = dirname(dirname(commonRoot)); +export const bridgeSource = join(commonRoot, "bridge", "scr_node_bridge.cc"); +export const bridgeHeader = join(repoRoot, "packages", "runtime", "src", "scr_node_bridge.h"); + +export { LIBNODE_PACK_SCHEMA, bridgeAbi } from "../build-pack.mjs"; + +/** Bump to invalidate every cached libnode build (a change to how the + * library is built or staged that the pin does not capture). */ +const BUILD_RECIPE = 1; + +/** + * @typedef {{ + * nodeVersion: string, + * tarballSha256: string, + * moduleVersion: number, + * configureFlags: string[], + * icu: "full" | "small", + * }} LibnodePin + */ + +/** @returns {LibnodePin} */ +export function readPin() { + return JSON.parse(readFileSync(join(commonRoot, "libnode.json"), "utf8")); +} + +/** Pack platforms: the npm package suffix, the Node `os`/`arch`, and the + * file names inside the pack. The darwin packs build only on macOS hosts. */ +export const LIBNODE_PLATFORMS = { + "linux-x64-gnu": { os: "linux", arch: "x64", target: "linux-x64-gnu" }, + "linux-arm64-gnu": { os: "linux", arch: "arm64", target: "linux-arm64-gnu" }, + "darwin-arm64": { os: "darwin", arch: "arm64", target: "macos-arm64" }, + "darwin-x64": { os: "darwin", arch: "x64", target: "macos-x64" }, +}; + +/** @param {string} platform */ +export function platformInfo(platform) { + const info = LIBNODE_PLATFORMS[/** @type {keyof typeof LIBNODE_PLATFORMS} */ (platform)]; + if (info === undefined) throw new Error(`unknown libnode pack platform ${platform}`); + return info; +} + +/** The library names inside a pack for one platform and module version. */ +export function packFileNames(platform, moduleVersion) { + const { os } = platformInfo(platform); + return os === "darwin" + ? { libnode: `libnode.${moduleVersion}.dylib`, bridge: "libscr_node_bridge.dylib" } + : { libnode: `libnode.so.${moduleVersion}`, bridge: "libscr_node_bridge.so" }; +} + +function sha256(text) { + return createHash("sha256").update(text).digest("hex"); +} + +/** The cache key of the libnode build itself: the Node release, how it is + * configured, the platform and toolchain. The bridge is not part of it; a + * bridge change rebuilds only the bridge against the cached headers. */ +export function libnodeCacheKey(platform, toolchain = process.env.SCRIPTC_LIBNODE_TOOLCHAIN) { + const pin = readPin(); + platformInfo(platform); + return sha256( + JSON.stringify({ + recipe: BUILD_RECIPE, + nodeVersion: pin.nodeVersion, + tarballSha256: pin.tarballSha256, + configureFlags: pin.configureFlags, + icu: pin.icu, + platform, + toolchain: toolchain ?? "default", + }), + ); +} + +/** The hash of everything the bridge is compiled from. */ +export function bridgeHash() { + return sha256(readFileSync(bridgeSource, "utf8") + "\0" + readFileSync(bridgeHeader, "utf8")); +} diff --git a/packages/libnode-common/scripts/smoke.mjs b/packages/libnode-common/scripts/smoke.mjs new file mode 100644 index 000000000..caf2d7725 --- /dev/null +++ b/packages/libnode-common/scripts/smoke.mjs @@ -0,0 +1,108 @@ +#!/usr/bin/env node +// Runs the libnode pack smoke test (smoke/smoke.c): compiles the C program +// with the host's C compiler, runs it against a pack directory and compares +// its stdout with the expected transcript, which pins the Node version, +// the built-in startup snapshot, full ICU, and synchronous, asynchronous +// and throwing calls through the bridge's C ABI. +// +// node packages/libnode-common/scripts/smoke.mjs [--pack

] [--platform

] +// +// The pack defaults to SCRIPTC_LIBNODE, else packages/libnode-. +import { execFileSync, spawnSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { parseArgs } from "node:util"; +import { bridgeAbi, bridgeHeader, commonRoot, readPin, repoRoot } from "./pin.mjs"; + +/** The host's pack platform name. */ +export function hostPlatform() { + if (process.platform === "darwin") return `darwin-${process.arch}`; + return `${process.platform}-${process.arch}-gnu`; +} + +/** The transcript smoke.c prints for a correct pack of the pinned Node. */ +export function expectedTranscript(pin = readPin(), abi = bridgeAbi()) { + const icuSmall = pin.icu === "small"; + return [ + `bridge ABI ${abi}, Node ${pin.nodeVersion}, NODE_MODULE_VERSION ${pin.moduleVersion}`, + "libnode mapped before dlopen: no", + "1 + 2 => R3", + `process.versions.node => R"${pin.nodeVersion}"`, + "process.config.variables.node_use_node_snapshot => Rtrue", + `process.config.variables.icu_small => R${icuSmall}`, + `new Intl.NumberFormat('de-DE').format(1234567.891) => R"${icuSmall ? "1,234,567.891" : "1.234.567,891"}"`, + `new Intl.DateTimeFormat('ja-JP', { timeZone: 'UTC', dateStyle: 'full' }).format(0) => R"${icuSmall ? "Thursday, January 1, 1970" : "1970年1月1日木曜日"}"`, + `require('node:crypto').createHash('sha256').update('scriptc').digest('hex') => R"${sha256Hex("scriptc")}"`, + `require('node:zlib').gunzipSync(require('node:zlib').gzipSync('round trip')).toString() => R"round trip"`, + `[typeof require('node:fs').readFileSync, require('node:path').join('a', 'b')] => R["function","a/b"]`, + `null.property => T{"error":{"name":"TypeError","message":"Cannot read properties of null (reading 'property')"}}`, + `await new Promise((resolve) => setTimeout(() => resolve('timer fired'), 5)) => R"timer fired"`, + "await require('node:fs/promises').readFile(process.execPath).then((b) => b.length > 0) => Rtrue", + `await Promise.reject(new RangeError('rejected')) => T{"error":{"name":"RangeError","message":"rejected"}}`, + "ok", + "", + ].join("\n"); +} + +function sha256Hex(text) { + return createHash("sha256").update(text).digest("hex"); +} + +/** Compiles and runs the smoke test; returns its stdout and stderr. */ +export function runSmoke(pack, cc = process.env.CC ?? "cc") { + const work = mkdtempSync(join(tmpdir(), "scriptc-libnode-smoke-")); + try { + const binary = join(work, "smoke"); + execFileSync( + cc, + [ + "-std=c11", + ...(process.platform === "linux" ? ["-D_POSIX_C_SOURCE=200809L", "-D_DEFAULT_SOURCE"] : []), + "-O2", + "-Wall", + "-Werror", + "-I", + dirname(bridgeHeader), + join(commonRoot, "smoke", "smoke.c"), + ...(process.platform === "linux" ? ["-ldl"] : []), + "-o", + binary, + ], + { stdio: "inherit" }, + ); + const result = spawnSync(binary, [pack], { encoding: "utf8", env: { ...process.env } }); + return { status: result.status, stdout: result.stdout, stderr: result.stderr }; + } finally { + rmSync(work, { recursive: true, force: true }); + } +} + +if (process.argv[1] === fileURLToPath(import.meta.url)) { + const { values } = parseArgs({ + options: { pack: { type: "string" }, platform: { type: "string" } }, + }); + // An explicit --pack or --platform names the pack; SCRIPTC_LIBNODE only + // replaces the host's default. + const pack = resolve( + values.pack ?? + (values.platform !== undefined + ? join(repoRoot, "packages", `libnode-${values.platform}`) + : (process.env.SCRIPTC_LIBNODE ?? join(repoRoot, "packages", `libnode-${hostPlatform()}`))), + ); + const manifest = JSON.parse(readFileSync(join(pack, "manifest.json"), "utf8")); + const { status, stdout, stderr } = runSmoke(pack); + process.stderr.write(stderr); + const expected = expectedTranscript(); + if (status !== 0 || stdout !== expected) { + process.stderr.write( + `smoke test failed (exit ${status}) for ${pack}\n--- expected\n${expected}--- actual\n${stdout}`, + ); + process.exit(1); + } + process.stdout.write( + `libnode smoke test passed: ${manifest.package} (Node ${manifest.nodeVersion})\n`, + ); +} diff --git a/packages/libnode-common/scripts/verify.mjs b/packages/libnode-common/scripts/verify.mjs new file mode 100644 index 000000000..863a20bc6 --- /dev/null +++ b/packages/libnode-common/scripts/verify.mjs @@ -0,0 +1,69 @@ +#!/usr/bin/env node +// Verifies a built libnode pack before it is packed or published: the +// manifest matches the package, the pin and the bridge ABI; every library +// it lists exists with the recorded size and SHA-256; the license files are +// present. +// +// node packages/libnode-common/scripts/verify.mjs +import { createHash } from "node:crypto"; +import { existsSync, readFileSync, statSync } from "node:fs"; +import { join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { LIBNODE_PACK_SCHEMA, bridgeAbi, packFileNames, platformInfo, readPin } from "./pin.mjs"; + +/** The problems with the pack in `root`; empty when it is complete. */ +export function verifyPack(root) { + const problems = []; + const packageJson = JSON.parse(readFileSync(join(root, "package.json"), "utf8")); + const manifestPath = join(root, "manifest.json"); + if (!existsSync(manifestPath)) return [`${manifestPath} is missing; run build:native first`]; + const manifest = JSON.parse(readFileSync(manifestPath, "utf8")); + const pin = readPin(); + const platform = String(packageJson.name).replace(/^@scriptc\/libnode-/, ""); + const expect = (field, actual, expected) => { + if (actual !== expected) problems.push(`manifest ${field} is ${actual}, expected ${expected}`); + }; + expect("schema", manifest.schema, LIBNODE_PACK_SCHEMA); + expect("package", manifest.package, packageJson.name); + expect("version", manifest.version, packageJson.version); + expect("target", manifest.target, platform); + expect("platform", manifest.platform, platformInfo(platform).os); + expect("arch", manifest.arch, platformInfo(platform).arch); + expect("nodeVersion", manifest.nodeVersion, pin.nodeVersion); + expect("moduleVersion", manifest.moduleVersion, pin.moduleVersion); + expect("bridgeAbi", manifest.bridgeAbi, bridgeAbi()); + const names = packFileNames(platform, pin.moduleVersion); + expect("libnode", manifest.libnode, names.libnode); + expect("bridge", manifest.bridge, names.bridge); + for (const name of [names.libnode, names.bridge]) { + const entry = manifest.files?.[name]; + const path = join(root, name); + if (entry === undefined) problems.push(`manifest lists no ${name}`); + else if (!existsSync(path)) problems.push(`${name} is missing`); + else if (statSync(path).size !== entry.size) problems.push(`${name} has the wrong size`); + else if (createHash("sha256").update(readFileSync(path)).digest("hex") !== entry.sha256) { + problems.push(`${name} has the wrong SHA-256`); + } + if (!packageJson.files.includes(name)) problems.push(`package.json files omits ${name}`); + } + for (const name of ["LICENSE", "SCRIPTC_LICENSE", "THIRD_PARTY_NOTICES"]) { + if (!existsSync(join(root, name))) problems.push(`${name} is missing`); + } + if (existsSync(join(root, "LICENSE"))) { + const license = readFileSync(join(root, "LICENSE"), "utf8"); + if (!license.startsWith("Node.js is licensed for use as follows:")) { + problems.push("LICENSE is not the Node.js LICENSE file"); + } + } + return problems; +} + +if (process.argv[1] === fileURLToPath(import.meta.url)) { + const root = resolve(process.argv[2] ?? "."); + const problems = verifyPack(root); + if (problems.length > 0) { + for (const problem of problems) console.error(`libnode pack ${root}: ${problem}`); + process.exit(1); + } + console.log(`libnode pack ${root} verified`); +} diff --git a/packages/libnode-common/smoke/smoke.c b/packages/libnode-common/smoke/smoke.c new file mode 100644 index 000000000..ea3a2e41f --- /dev/null +++ b/packages/libnode-common/smoke/smoke.c @@ -0,0 +1,205 @@ +/* Smoke test for a libnode pack, written the way the scriptc runtime loads + * one: a plain C program that never includes a Node or V8 header, dlopens + * the pack's bridge, checks its ABI and Node version, starts Node (one + * environment from the built-in startup snapshot), evaluates expressions + * synchronously and asynchronously through the bridge's C ABI, runs Node's + * event loop until the asynchronous ones settle, and tears Node down. + * + * smoke (or SCRIPTC_LIBNODE= smoke) + * + * stdout is deterministic and compared by scripts/smoke.mjs; timings and + * memory go to stderr. */ +#include "scr_node_bridge.h" + +#include +#include +#include +#include +#include +#include +#include + +#if defined(__APPLE__) +#define BRIDGE_NAME "libscr_node_bridge.dylib" +#else +#define BRIDGE_NAME "libscr_node_bridge.so" +#endif + +/* The bootstrap runs as the environment's main script. It hands the bridge + * a dispatcher that evaluates `name` as an expression and answers the + * bridge's outcome strings: "R" when it returns, "T" when it + * throws; an asynchronous call settles through the host. The smoke test + * schedules no runtime jobs and listens to no process events, so the rest + * of the dispatcher the bridge requires does nothing. */ +static const char kBootstrap[] = + "const host = globalThis.__scriptcNodeIslandHost;\n" + "globalThis.require = require;\n" + "const outcome = (ok, value) => (ok ? 'R' : 'T') + JSON.stringify(\n" + " ok ? value : { error: { name: value?.name, message: String(value?.message ?? value) } });\n" + "globalThis.__scriptcNodeIsland = {\n" + " call(id, sync, file, specifier, name, args) {\n" + " let value;\n" + " try { value = (0, eval)(name); } catch (error) { return outcome(false, error); }\n" + " if (sync) return outcome(true, value);\n" + " Promise.resolve(value).then((v) => host.settle(id, outcome(true, v)),\n" + " (e) => host.settle(id, outcome(false, e)));\n" + " },\n" + " setTimer() { return 0; }, clearTimer() {}, refTimer() {}, refreshTimer() {},\n" + " setImmediate() { return 0; }, clearImmediate() {}, refImmediate() {},\n" + " nextTick() {}, emit() {}, load() {},\n" + "};\n"; + +static double now_ms(void) { + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return ts.tv_sec * 1e3 + ts.tv_nsec / 1e6; +} + +static long max_rss_kb(void) { + struct rusage usage; + getrusage(RUSAGE_SELF, &usage); +#if defined(__APPLE__) + return usage.ru_maxrss / 1024; +#else + return usage.ru_maxrss; +#endif +} + +/* Whether a library whose path contains `needle` is mapped (Linux only). */ +static int mapped(const char *needle) { + FILE *maps = fopen("/proc/self/maps", "r"); + if (maps == NULL) return -1; + char line[4096]; + int found = 0; + while (!found && fgets(line, sizeof line, maps) != NULL) found = strstr(line, needle) != NULL; + fclose(maps); + return found; +} + +/* The asynchronous expressions, by call id, and their outcomes. */ +static const char *const kAsync[] = { + "new Promise((resolve) => setTimeout(() => resolve('timer fired'), 5))", + "require('node:fs/promises').readFile(process.execPath).then((b) => b.length > 0)", + "Promise.reject(new RangeError('rejected'))", +}; +#define ASYNC_COUNT (sizeof kAsync / sizeof kAsync[0]) +static char *settled[ASYNC_COUNT]; + +static void host_settle(uint32_t id, const char *outcome, size_t len) { + if (id >= ASYNC_COUNT || settled[id] != NULL) return; + settled[id] = malloc(len + 1); + memcpy(settled[id], outcome, len); + settled[id][len] = '\0'; +} + +static void host_exit(int code) { + fflush(stdout); + exit(code); +} + +/* The smoke test never calls host_loop, so Node's loop runs alone: no + * runtime jobs, I/O or listeners, and V8 keeps the thread's stack limit. */ +static void host_run(uint32_t kind, uint32_t token) { (void)kind, (void)token; } +static void host_io_state(ScrNodeIoState *state) { + state->nfds = 0; + state->alive = 0; + state->ready = 0; + state->cap_ms = -1; +} +static void host_io_dispatch(void) {} +static void host_microtasks_done(void) {} +static void host_listen(uint32_t event, const char *name, uint32_t listener, int add, int once) { + (void)event, (void)name, (void)listener, (void)add, (void)once; +} +static uintptr_t host_stack_limit(void) { return 0; } +/* The smoke test runs on its own thread's stack. */ +static uintptr_t host_stack_start(void) { return 0; } + +static const ScrNodeBridgeHost kHost = { + host_settle, host_exit, host_run, host_io_state, + host_io_dispatch, host_microtasks_done, host_listen, host_stack_limit, + host_stack_start, +}; + +static const ScrNodeBridge *bridge; + +static void eval_sync(const char *expression) { + size_t len = 0; + char *outcome = bridge->call(0, 1, "", "", expression, "[]", 2, &len); + printf("%s => %s\n", expression, outcome); + bridge->release(outcome); +} + +/* Starts every asynchronous expression, then runs Node's event loop until + * nothing keeps it alive, as the runtime does at the end of a program. */ +static void eval_async_all(void) { + for (uint32_t id = 0; id < ASYNC_COUNT; id++) { + size_t len = 0; + char *immediate = bridge->call(id, 0, "", "", kAsync[id], "[]", 2, &len); + if (immediate != NULL) bridge->release(immediate); + bridge->checkpoint(0); + } + bridge->run_loop(0); + for (uint32_t id = 0; id < ASYNC_COUNT; id++) { + printf("await %s => %s\n", kAsync[id], settled[id] != NULL ? settled[id] : "(never settled)"); + free(settled[id]); + settled[id] = NULL; + } +} + +int main(int argc, char **argv) { + const char *dir = argc > 1 ? argv[1] : getenv("SCRIPTC_LIBNODE"); + if (dir == NULL || dir[0] == '\0') { + fprintf(stderr, "usage: smoke (or set SCRIPTC_LIBNODE)\n"); + return 2; + } + char path[4096]; + snprintf(path, sizeof path, "%s/%s", dir, BRIDGE_NAME); + int before = mapped("/libnode."); + double t0 = now_ms(); + void *library = dlopen(path, RTLD_NOW | RTLD_GLOBAL); + if (library == NULL) { + fprintf(stderr, "smoke: %s\n", dlerror()); + return 1; + } + double t1 = now_ms(); + const ScrNodeBridge *(*entry)(void) = + (const ScrNodeBridge *(*)(void))dlsym(library, SCR_NODE_BRIDGE_ENTRY); + if (entry == NULL) { + fprintf(stderr, "smoke: %s has no %s\n", path, SCR_NODE_BRIDGE_ENTRY); + return 1; + } + bridge = entry(); + if (bridge->abi != SCR_NODE_BRIDGE_ABI) { + fprintf(stderr, "smoke: bridge ABI %u, expected %u\n", bridge->abi, SCR_NODE_BRIDGE_ABI); + return 1; + } + printf("bridge ABI %u, Node %s, NODE_MODULE_VERSION %u\n", bridge->abi, bridge->node_version, + bridge->module_version); + printf("libnode mapped before dlopen: %s\n", before == 1 ? "yes" : "no"); + char *error = bridge->start(&kHost, kBootstrap, argv[0], 0); + if (error != NULL) { + fprintf(stderr, "smoke: %s\n", error); + return 1; + } + double t2 = now_ms(); + eval_sync("1 + 2"); + double t3 = now_ms(); + eval_sync("process.versions.node"); + eval_sync("process.config.variables.node_use_node_snapshot"); + eval_sync("process.config.variables.icu_small"); + eval_sync("new Intl.NumberFormat('de-DE').format(1234567.891)"); + eval_sync("new Intl.DateTimeFormat('ja-JP', { timeZone: 'UTC', dateStyle: 'full' }).format(0)"); + eval_sync("require('node:crypto').createHash('sha256').update('scriptc').digest('hex')"); + eval_sync("require('node:zlib').gunzipSync(require('node:zlib').gzipSync('round trip')).toString()"); + eval_sync("[typeof require('node:fs').readFileSync, require('node:path').join('a', 'b')]"); + eval_sync("null.property"); + eval_async_all(); + fprintf(stderr, + "smoke: dlopen %.1f ms, start %.1f ms, first result %.2f ms, dlopen to first result " + "%.1f ms, max RSS %ld KB\n", + t1 - t0, t2 - t1, t3 - t2, t3 - t0, max_rss_kb()); + bridge->stop(); + printf("ok\n"); + return 0; +} diff --git a/packages/libnode-common/test/bridge-smoke.c b/packages/libnode-common/test/bridge-smoke.c index f05c96d91..a520dadec 100644 --- a/packages/libnode-common/test/bridge-smoke.c +++ b/packages/libnode-common/test/bridge-smoke.c @@ -2,10 +2,10 @@ * check its ABI, start Node from the snapshot with the runtime's embedded * bootstrap) and makes calls through the bridge: synchronous results and * errors, and an asynchronous call settled by Node's own loop, which this - * host drives like the runtime's loop slot. bridge.test.ts compares the - * transcript. Usage: bridge-smoke */ + * host runs the way the runtime's hosted loop does (host_loop, a + * checkpoint, run_loop) with no runtime work of its own. bridge.test.ts + * compares the transcript. Usage: bridge-smoke */ #include -#include #include #include #include @@ -32,7 +32,29 @@ static void on_exit_code(int code) { _Exit(code); } -static const ScrNodeBridgeHost host = {settle, on_exit_code}; +static void run(uint32_t kind, uint32_t token) { + printf("run %u %u\n", kind, token); +} + +static void io_state(ScrNodeIoState *state) { + state->nfds = 0; + state->alive = 0; + state->ready = 0; + state->cap_ms = -1; +} + +static void io_dispatch(void) {} +static void microtasks_done(void) {} + +static void listen(uint32_t event, const char *name, uint32_t listener, int add, int once) { + printf("listen %u %s %u %d %d\n", event, name, listener, add, once); +} + +static uintptr_t stack_limit(void) { return 0; } +static uintptr_t stack_start(void) { return 0; } + +static const ScrNodeBridgeHost host = {settle, on_exit_code, run, io_state, io_dispatch, + microtasks_done, listen, stack_limit, stack_start}; static void call(const ScrNodeBridge *b, uint32_t id, int sync, const char *module, const char *name, const char *args) { @@ -70,11 +92,9 @@ int main(int argc, char **argv) { call(b, 0, 1, file, "fails", "[]"); call(b, 0, 1, file, "missing", "[]"); call(b, 7, 0, file, "later", "[5]"); - while (settled == 0 || b->alive()) { - struct pollfd fd = {b->backend_fd(), POLLIN, 0}; - (void)poll(&fd, 1, b->backend_timeout()); - b->pump(); - } + b->host_loop(); + b->checkpoint(0); + b->run_loop(0); call(b, 0, 1, file, "quit", "[3]"); return 0; } diff --git a/packages/libnode-common/test/bridge.test.ts b/packages/libnode-common/test/bridge.test.ts index 1fa2415c6..9181e36b7 100644 --- a/packages/libnode-common/test/bridge.test.ts +++ b/packages/libnode-common/test/bridge.test.ts @@ -3,6 +3,7 @@ * calls through the bridge with the runtime's embedded bootstrap. Needs a * pack in SCRIPTC_LIBNODE; required in the Linux x64 and macOS CI lanes * (tests/harness/libnode.ts). */ +import { bridgeAbi } from "../build-pack.mjs"; import { execFile } from "node:child_process"; import { mkdirSync, writeFileSync } from "node:fs"; import { join } from "node:path"; @@ -45,7 +46,7 @@ exports.quit = (code) => process.exit(code); expect(result.stderr).toBe(""); expect(result.stdout).toBe( [ - "abi 3 ok 1", + `abi ${bridgeAbi()} ok 1`, "sync add -> R5", 'sync fails -> T{"error":{"kind":2,"message":"out of range","code":"ERR_X"}}', `sync missing -> T{"error":{"kind":1,"message":"'missing' exported by './smoke.cjs' is not a function"}}`, diff --git a/packages/libnode-common/test/pin.test.ts b/packages/libnode-common/test/pin.test.ts new file mode 100644 index 000000000..e13414803 --- /dev/null +++ b/packages/libnode-common/test/pin.test.ts @@ -0,0 +1,90 @@ +import { readFileSync, readdirSync } from "node:fs"; +import { join } from "node:path"; +import { describe, expect, test } from "vitest"; +import { + LIBNODE_PLATFORMS, + bridgeAbi, + bridgeHash, + libnodeCacheKey, + packFileNames, + readPin, + repoRoot, +} from "../scripts/pin.mjs"; +import { expectedTranscript } from "../scripts/smoke.mjs"; + +const readJson = (path: string) => JSON.parse(readFileSync(join(repoRoot, path), "utf8")); + +describe("the libnode pin", () => { + test("is the repository's pinned Node release", () => { + const pin = readPin(); + expect(pin.nodeVersion).toBe(readJson("internal/compatibility/node-v24.json").version); + expect(pin.nodeVersion).toBe(readFileSync(join(repoRoot, ".node-version"), "utf8").trim()); + expect(pin.tarballSha256).toMatch(/^[0-9a-f]{64}$/); + }); + + test("builds the shared library with full ICU (design §8 question 2)", () => { + const pin = readPin(); + expect(pin.configureFlags).toContain("--shared"); + expect(pin.configureFlags.some((flag: string) => flag.startsWith("--with-intl"))).toBe(false); + expect(pin.icu).toBe("full"); + }); +}); + +describe("the platform packages", () => { + const dirs = readdirSync(join(repoRoot, "packages")).filter( + (name) => name.startsWith("libnode-") && name !== "libnode-common", + ); + + test("match the platform table", () => { + expect(dirs.map((dir) => dir.slice("libnode-".length)).sort()).toEqual( + Object.keys(LIBNODE_PLATFORMS).sort(), + ); + }); + + test.each(Object.entries(LIBNODE_PLATFORMS))("%s ships its libraries", (platform, info) => { + const manifest = readJson(`packages/libnode-${platform}/package.json`); + const names = packFileNames(platform, readPin().moduleVersion); + expect(manifest.name).toBe(`@scriptc/libnode-${platform}`); + expect(manifest.version).toBe(readJson("packages/cli/package.json").version); + expect(manifest.os).toEqual([info.os]); + expect(manifest.cpu).toEqual([info.arch]); + expect(manifest.files).toEqual([ + names.libnode, + names.bridge, + "manifest.json", + "LICENSE", + "SCRIPTC_LICENSE", + "THIRD_PARTY_NOTICES", + ]); + // Never built by `pnpm -r build`. + expect(manifest.scripts.build).toBe("true"); + expect(manifest.scripts["build:native"]).toBe("node scripts/build.mjs"); + }); +}); + +describe("cache identity", () => { + test("the libnode key depends on platform and toolchain, not on the bridge", () => { + const a = libnodeCacheKey("linux-x64-gnu", "manylinux_2_28-gcc-toolset-14"); + expect(a).toMatch(/^[0-9a-f]{64}$/); + expect(libnodeCacheKey("linux-x64-gnu", "manylinux_2_28-gcc-toolset-14")).toBe(a); + expect(libnodeCacheKey("linux-arm64-gnu", "manylinux_2_28-gcc-toolset-14")).not.toBe(a); + expect(libnodeCacheKey("linux-x64-gnu", "other")).not.toBe(a); + expect(bridgeHash()).toMatch(/^[0-9a-f]{64}$/); + }); + + test("unknown platforms are refused", () => { + expect(() => libnodeCacheKey("linux-x64-musl", "x")).toThrow( + "unknown libnode pack platform linux-x64-musl", + ); + }); +}); + +test("the smoke transcript pins the Node version, the snapshot and full ICU", () => { + const lines = expectedTranscript().split("\n"); + expect(lines[0]).toBe( + `bridge ABI ${bridgeAbi()}, Node ${readPin().nodeVersion}, NODE_MODULE_VERSION ${readPin().moduleVersion}`, + ); + expect(lines).toContain("process.config.variables.node_use_node_snapshot => Rtrue"); + expect(lines).toContain(`new Intl.NumberFormat('de-DE').format(1234567.891) => R"1.234.567,891"`); + expect(lines.at(-2)).toBe("ok"); +}); diff --git a/packages/libnode-common/test/smoke.test.ts b/packages/libnode-common/test/smoke.test.ts new file mode 100644 index 000000000..b3930d457 --- /dev/null +++ b/packages/libnode-common/test/smoke.test.ts @@ -0,0 +1,27 @@ +// Loads a real libnode pack from a C program (smoke/smoke.c) the way the +// scriptc runtime does. Packs are not built by the ordinary build, so this +// runs only when SCRIPTC_LIBNODE names a pack directory (a CI artifact of +// the libnode-packs workflow, or a local build:native output). CI's +// fallback development pack (dev-pack.json) is not a package: it has no +// package.json and may carry a different Node patch release, so it is +// exercised by the Node island tests instead. +import { existsSync } from "node:fs"; +import { join } from "node:path"; +import { describe, expect, test } from "vitest"; +import { verifyPack } from "../scripts/verify.mjs"; +import { expectedTranscript, runSmoke } from "../scripts/smoke.mjs"; + +const pack = process.env["SCRIPTC_LIBNODE"]; +const isPackage = pack !== undefined && pack !== "" && existsSync(join(pack, "package.json")); + +describe.skipIf(!isPackage)("libnode pack smoke test", () => { + test("the pack is complete", () => { + expect(verifyPack(pack!)).toEqual([]); + }); + + test("a C program starts Node from the pack and evaluates expressions", () => { + const { status, stdout } = runSmoke(pack!); + expect(stdout).toBe(expectedTranscript()); + expect(status).toBe(0); + }, 60_000); +}); diff --git a/packages/libnode-darwin-arm64/SCRIPTC_LICENSE b/packages/libnode-darwin-arm64/SCRIPTC_LICENSE new file mode 100644 index 000000000..bb2b7da94 --- /dev/null +++ b/packages/libnode-darwin-arm64/SCRIPTC_LICENSE @@ -0,0 +1 @@ +Copyright scriptc contributors. Licensed under Apache-2.0. diff --git a/packages/libnode-darwin-arm64/THIRD_PARTY_NOTICES b/packages/libnode-darwin-arm64/THIRD_PARTY_NOTICES new file mode 100644 index 000000000..8bcf8fa63 --- /dev/null +++ b/packages/libnode-darwin-arm64/THIRD_PARTY_NOTICES @@ -0,0 +1,15 @@ +This package contains libnode.137.dylib, a stripped build of the Node.js shared +library (https://nodejs.org) from the official source release named in +manifest.json, built without modification. Node.js is licensed under the +MIT license. It bundles third-party components, among them V8, ICU, +OpenSSL, libuv, zlib, brotli, nghttp2, nghttp3, ngtcp2, llhttp, c-ares, +simdutf, ada and uvwasi, under the BSD, Apache-2.0, Unicode, MIT and +other permissive licenses. LICENSE is the LICENSE file of that Node.js +release, which carries the complete license text of Node.js and of every +bundled component. + +libscr_node_bridge.dylib is the scriptc bridge between the scriptc runtime and Node.js, +licensed under Apache-2.0 (SCRIPTC_LICENSE). + +A program that `scriptc build` links against this package redistributes +Node.js; scriptc writes these notices next to such a program. diff --git a/packages/libnode-darwin-arm64/package.json b/packages/libnode-darwin-arm64/package.json new file mode 100644 index 000000000..22d1e1889 --- /dev/null +++ b/packages/libnode-darwin-arm64/package.json @@ -0,0 +1,35 @@ +{ + "name": "@scriptc/libnode-darwin-arm64", + "version": "0.4.5", + "private": true, + "description": "Prebuilt Node.js shared library and scriptc bridge for Node islands on macOS arm64", + "license": "SEE LICENSE IN LICENSE", + "homepage": "https://scriptc.dev", + "repository": { + "type": "git", + "url": "git+https://github.com/vercel-labs/scriptc.git", + "directory": "packages/libnode-darwin-arm64" + }, + "os": [ + "darwin" + ], + "cpu": [ + "arm64" + ], + "files": [ + "libnode.137.dylib", + "libscr_node_bridge.dylib", + "manifest.json", + "LICENSE", + "SCRIPTC_LICENSE", + "THIRD_PARTY_NOTICES" + ], + "scripts": { + "build": "true", + "build:native": "node scripts/build.mjs", + "prepack": "node ../libnode-common/scripts/verify.mjs ." + }, + "publishConfig": { + "access": "public" + } +} diff --git a/packages/libnode-darwin-arm64/scripts/build.mjs b/packages/libnode-darwin-arm64/scripts/build.mjs new file mode 100644 index 000000000..e5c8a3534 --- /dev/null +++ b/packages/libnode-darwin-arm64/scripts/build.mjs @@ -0,0 +1,22 @@ +#!/usr/bin/env node +// Builds @scriptc/libnode-darwin-arm64: libnode for the pinned Node.js release +// and the bridge (packages/libnode-common/scripts/build.mjs). The Node +// build takes about half an hour on 8 cores and is cached by Node version, +// configure flags and toolchain; only the libnode-packs workflow and an +// explicit `pnpm --filter @scriptc/libnode-darwin-arm64 build:native` run it. +import { dirname } from "node:path"; +import { fileURLToPath } from "node:url"; +import { buildLibnodePack } from "../../libnode-common/scripts/build.mjs"; + +if (process.platform !== "darwin" || process.arch !== "arm64") { + process.stdout.write("@scriptc/libnode-darwin-arm64: skipped on this host\n"); + process.exit(0); +} +await buildLibnodePack({ + platform: "darwin-arm64", + packageRoot: dirname(dirname(fileURLToPath(import.meta.url))), + compress: process.argv.includes("--compress"), + // Assemble from a restored libnode build only (CI test lanes); never + // start a Node build. + skipLibnode: process.argv.includes("--from-cache"), +}); diff --git a/packages/libnode-darwin-x64/SCRIPTC_LICENSE b/packages/libnode-darwin-x64/SCRIPTC_LICENSE new file mode 100644 index 000000000..bb2b7da94 --- /dev/null +++ b/packages/libnode-darwin-x64/SCRIPTC_LICENSE @@ -0,0 +1 @@ +Copyright scriptc contributors. Licensed under Apache-2.0. diff --git a/packages/libnode-darwin-x64/THIRD_PARTY_NOTICES b/packages/libnode-darwin-x64/THIRD_PARTY_NOTICES new file mode 100644 index 000000000..8bcf8fa63 --- /dev/null +++ b/packages/libnode-darwin-x64/THIRD_PARTY_NOTICES @@ -0,0 +1,15 @@ +This package contains libnode.137.dylib, a stripped build of the Node.js shared +library (https://nodejs.org) from the official source release named in +manifest.json, built without modification. Node.js is licensed under the +MIT license. It bundles third-party components, among them V8, ICU, +OpenSSL, libuv, zlib, brotli, nghttp2, nghttp3, ngtcp2, llhttp, c-ares, +simdutf, ada and uvwasi, under the BSD, Apache-2.0, Unicode, MIT and +other permissive licenses. LICENSE is the LICENSE file of that Node.js +release, which carries the complete license text of Node.js and of every +bundled component. + +libscr_node_bridge.dylib is the scriptc bridge between the scriptc runtime and Node.js, +licensed under Apache-2.0 (SCRIPTC_LICENSE). + +A program that `scriptc build` links against this package redistributes +Node.js; scriptc writes these notices next to such a program. diff --git a/packages/libnode-darwin-x64/package.json b/packages/libnode-darwin-x64/package.json new file mode 100644 index 000000000..dc9f3e06f --- /dev/null +++ b/packages/libnode-darwin-x64/package.json @@ -0,0 +1,35 @@ +{ + "name": "@scriptc/libnode-darwin-x64", + "version": "0.4.5", + "private": true, + "description": "Prebuilt Node.js shared library and scriptc bridge for Node islands on macOS x64", + "license": "SEE LICENSE IN LICENSE", + "homepage": "https://scriptc.dev", + "repository": { + "type": "git", + "url": "git+https://github.com/vercel-labs/scriptc.git", + "directory": "packages/libnode-darwin-x64" + }, + "os": [ + "darwin" + ], + "cpu": [ + "x64" + ], + "files": [ + "libnode.137.dylib", + "libscr_node_bridge.dylib", + "manifest.json", + "LICENSE", + "SCRIPTC_LICENSE", + "THIRD_PARTY_NOTICES" + ], + "scripts": { + "build": "true", + "build:native": "node scripts/build.mjs", + "prepack": "node ../libnode-common/scripts/verify.mjs ." + }, + "publishConfig": { + "access": "public" + } +} diff --git a/packages/libnode-darwin-x64/scripts/build.mjs b/packages/libnode-darwin-x64/scripts/build.mjs new file mode 100644 index 000000000..ce5a13ec1 --- /dev/null +++ b/packages/libnode-darwin-x64/scripts/build.mjs @@ -0,0 +1,22 @@ +#!/usr/bin/env node +// Builds @scriptc/libnode-darwin-x64: libnode for the pinned Node.js release +// and the bridge (packages/libnode-common/scripts/build.mjs). The Node +// build takes about half an hour on 8 cores and is cached by Node version, +// configure flags and toolchain; only the libnode-packs workflow and an +// explicit `pnpm --filter @scriptc/libnode-darwin-x64 build:native` run it. +import { dirname } from "node:path"; +import { fileURLToPath } from "node:url"; +import { buildLibnodePack } from "../../libnode-common/scripts/build.mjs"; + +if (process.platform !== "darwin" || process.arch !== "x64") { + process.stdout.write("@scriptc/libnode-darwin-x64: skipped on this host\n"); + process.exit(0); +} +await buildLibnodePack({ + platform: "darwin-x64", + packageRoot: dirname(dirname(fileURLToPath(import.meta.url))), + compress: process.argv.includes("--compress"), + // Assemble from a restored libnode build only (CI test lanes); never + // start a Node build. + skipLibnode: process.argv.includes("--from-cache"), +}); diff --git a/packages/libnode-linux-arm64-gnu/SCRIPTC_LICENSE b/packages/libnode-linux-arm64-gnu/SCRIPTC_LICENSE new file mode 100644 index 000000000..bb2b7da94 --- /dev/null +++ b/packages/libnode-linux-arm64-gnu/SCRIPTC_LICENSE @@ -0,0 +1 @@ +Copyright scriptc contributors. Licensed under Apache-2.0. diff --git a/packages/libnode-linux-arm64-gnu/THIRD_PARTY_NOTICES b/packages/libnode-linux-arm64-gnu/THIRD_PARTY_NOTICES new file mode 100644 index 000000000..2ff695436 --- /dev/null +++ b/packages/libnode-linux-arm64-gnu/THIRD_PARTY_NOTICES @@ -0,0 +1,15 @@ +This package contains libnode.so.137, a stripped build of the Node.js shared +library (https://nodejs.org) from the official source release named in +manifest.json, built without modification. Node.js is licensed under the +MIT license. It bundles third-party components, among them V8, ICU, +OpenSSL, libuv, zlib, brotli, nghttp2, nghttp3, ngtcp2, llhttp, c-ares, +simdutf, ada and uvwasi, under the BSD, Apache-2.0, Unicode, MIT and +other permissive licenses. LICENSE is the LICENSE file of that Node.js +release, which carries the complete license text of Node.js and of every +bundled component. + +libscr_node_bridge.so is the scriptc bridge between the scriptc runtime and Node.js, +licensed under Apache-2.0 (SCRIPTC_LICENSE). + +A program that `scriptc build` links against this package redistributes +Node.js; scriptc writes these notices next to such a program. diff --git a/packages/libnode-linux-arm64-gnu/package.json b/packages/libnode-linux-arm64-gnu/package.json new file mode 100644 index 000000000..108e179cd --- /dev/null +++ b/packages/libnode-linux-arm64-gnu/package.json @@ -0,0 +1,38 @@ +{ + "name": "@scriptc/libnode-linux-arm64-gnu", + "version": "0.4.5", + "private": true, + "description": "Prebuilt Node.js shared library and scriptc bridge for Node islands on Linux arm64 glibc", + "license": "SEE LICENSE IN LICENSE", + "homepage": "https://scriptc.dev", + "repository": { + "type": "git", + "url": "git+https://github.com/vercel-labs/scriptc.git", + "directory": "packages/libnode-linux-arm64-gnu" + }, + "os": [ + "linux" + ], + "cpu": [ + "arm64" + ], + "libc": [ + "glibc" + ], + "files": [ + "libnode.so.137", + "libscr_node_bridge.so", + "manifest.json", + "LICENSE", + "SCRIPTC_LICENSE", + "THIRD_PARTY_NOTICES" + ], + "scripts": { + "build": "true", + "build:native": "node scripts/build.mjs", + "prepack": "node ../libnode-common/scripts/verify.mjs ." + }, + "publishConfig": { + "access": "public" + } +} diff --git a/packages/libnode-linux-arm64-gnu/scripts/build.mjs b/packages/libnode-linux-arm64-gnu/scripts/build.mjs new file mode 100644 index 000000000..153bfc22b --- /dev/null +++ b/packages/libnode-linux-arm64-gnu/scripts/build.mjs @@ -0,0 +1,22 @@ +#!/usr/bin/env node +// Builds @scriptc/libnode-linux-arm64-gnu: libnode for the pinned Node.js release +// and the bridge (packages/libnode-common/scripts/build.mjs). The Node +// build takes about half an hour on 8 cores and is cached by Node version, +// configure flags and toolchain; only the libnode-packs workflow and an +// explicit `pnpm --filter @scriptc/libnode-linux-arm64-gnu build:native` run it. +import { dirname } from "node:path"; +import { fileURLToPath } from "node:url"; +import { buildLibnodePack } from "../../libnode-common/scripts/build.mjs"; + +if (process.platform !== "linux" || process.arch !== "arm64") { + process.stdout.write("@scriptc/libnode-linux-arm64-gnu: skipped on this host\n"); + process.exit(0); +} +await buildLibnodePack({ + platform: "linux-arm64-gnu", + packageRoot: dirname(dirname(fileURLToPath(import.meta.url))), + compress: process.argv.includes("--compress"), + // Assemble from a restored libnode build only (CI test lanes); never + // start a Node build. + skipLibnode: process.argv.includes("--from-cache"), +}); diff --git a/packages/libnode-linux-x64-gnu/SCRIPTC_LICENSE b/packages/libnode-linux-x64-gnu/SCRIPTC_LICENSE new file mode 100644 index 000000000..bb2b7da94 --- /dev/null +++ b/packages/libnode-linux-x64-gnu/SCRIPTC_LICENSE @@ -0,0 +1 @@ +Copyright scriptc contributors. Licensed under Apache-2.0. diff --git a/packages/libnode-linux-x64-gnu/THIRD_PARTY_NOTICES b/packages/libnode-linux-x64-gnu/THIRD_PARTY_NOTICES new file mode 100644 index 000000000..2ff695436 --- /dev/null +++ b/packages/libnode-linux-x64-gnu/THIRD_PARTY_NOTICES @@ -0,0 +1,15 @@ +This package contains libnode.so.137, a stripped build of the Node.js shared +library (https://nodejs.org) from the official source release named in +manifest.json, built without modification. Node.js is licensed under the +MIT license. It bundles third-party components, among them V8, ICU, +OpenSSL, libuv, zlib, brotli, nghttp2, nghttp3, ngtcp2, llhttp, c-ares, +simdutf, ada and uvwasi, under the BSD, Apache-2.0, Unicode, MIT and +other permissive licenses. LICENSE is the LICENSE file of that Node.js +release, which carries the complete license text of Node.js and of every +bundled component. + +libscr_node_bridge.so is the scriptc bridge between the scriptc runtime and Node.js, +licensed under Apache-2.0 (SCRIPTC_LICENSE). + +A program that `scriptc build` links against this package redistributes +Node.js; scriptc writes these notices next to such a program. diff --git a/packages/libnode-linux-x64-gnu/package.json b/packages/libnode-linux-x64-gnu/package.json new file mode 100644 index 000000000..af2c3bace --- /dev/null +++ b/packages/libnode-linux-x64-gnu/package.json @@ -0,0 +1,38 @@ +{ + "name": "@scriptc/libnode-linux-x64-gnu", + "version": "0.4.5", + "private": true, + "description": "Prebuilt Node.js shared library and scriptc bridge for Node islands on Linux x64 glibc", + "license": "SEE LICENSE IN LICENSE", + "homepage": "https://scriptc.dev", + "repository": { + "type": "git", + "url": "git+https://github.com/vercel-labs/scriptc.git", + "directory": "packages/libnode-linux-x64-gnu" + }, + "os": [ + "linux" + ], + "cpu": [ + "x64" + ], + "libc": [ + "glibc" + ], + "files": [ + "libnode.so.137", + "libscr_node_bridge.so", + "manifest.json", + "LICENSE", + "SCRIPTC_LICENSE", + "THIRD_PARTY_NOTICES" + ], + "scripts": { + "build": "true", + "build:native": "node scripts/build.mjs", + "prepack": "node ../libnode-common/scripts/verify.mjs ." + }, + "publishConfig": { + "access": "public" + } +} diff --git a/packages/libnode-linux-x64-gnu/scripts/build.mjs b/packages/libnode-linux-x64-gnu/scripts/build.mjs new file mode 100644 index 000000000..03ff05093 --- /dev/null +++ b/packages/libnode-linux-x64-gnu/scripts/build.mjs @@ -0,0 +1,22 @@ +#!/usr/bin/env node +// Builds @scriptc/libnode-linux-x64-gnu: libnode for the pinned Node.js release +// and the bridge (packages/libnode-common/scripts/build.mjs). The Node +// build takes about half an hour on 8 cores and is cached by Node version, +// configure flags and toolchain; only the libnode-packs workflow and an +// explicit `pnpm --filter @scriptc/libnode-linux-x64-gnu build:native` run it. +import { dirname } from "node:path"; +import { fileURLToPath } from "node:url"; +import { buildLibnodePack } from "../../libnode-common/scripts/build.mjs"; + +if (process.platform !== "linux" || process.arch !== "x64") { + process.stdout.write("@scriptc/libnode-linux-x64-gnu: skipped on this host\n"); + process.exit(0); +} +await buildLibnodePack({ + platform: "linux-x64-gnu", + packageRoot: dirname(dirname(fileURLToPath(import.meta.url))), + compress: process.argv.includes("--compress"), + // Assemble from a restored libnode build only (CI test lanes); never + // start a Node build. + skipLibnode: process.argv.includes("--from-cache"), +}); diff --git a/packages/node/index.d.ts b/packages/node/index.d.ts index 87a8eb541..de353d264 100644 --- a/packages/node/index.d.ts +++ b/packages/node/index.d.ts @@ -36,3 +36,16 @@ export declare function call( name: string, ...args: unknown[] ): Promise; + +/** Loads the module `specifier` with require() from the calling module and + * returns its exports **by reference** (scope-B Node islands, NI-4). + * + * In scriptc builds the module runs in Node.js loaded into the program on + * first use; the specifier must be a string literal, resolved at build time + * as `require.resolve` resolves it from the caller. The exports and every + * object reached through them stay Node objects: the program holds handles + * to them, so identity, aliasing and mutation are shared with Node. + * Primitives, strings and BigInt are copied; promises become promises; + * Buffers and typed arrays share their memory. Under Node.js this is + * `createRequire(callerFile)(specifier)`. */ +export declare function requireModule(specifier: string): T; diff --git a/packages/node/index.js b/packages/node/index.js index ed5fd6870..b058574a8 100644 --- a/packages/node/index.js +++ b/packages/node/index.js @@ -135,3 +135,8 @@ export async function call(specifier, name, ...args) { throw copyThrown(error); } } + +/** requireModule(specifier): the module's exports, by reference. */ +export function requireModule(specifier) { + return createRequire(callerFile(requireModule))(specifier); +} diff --git a/packages/node/package.json b/packages/node/package.json index 6f588445a..6a080da99 100644 --- a/packages/node/package.json +++ b/packages/node/package.json @@ -1,6 +1,6 @@ { "name": "@scriptc/node", - "version": "0.4.1", + "version": "0.4.5", "description": "Call into Node.js from scriptc programs: run an export of a JavaScript module in a Node.js helper process and copy JSON data both ways. Under Node.js the calls run in-process.", "license": "Apache-2.0", "homepage": "https://scriptc.dev", diff --git a/packages/runtime-pack-common/runtime-pack-matrix.mjs b/packages/runtime-pack-common/runtime-pack-matrix.mjs index e26fdb8d8..5f6eae7ba 100644 --- a/packages/runtime-pack-common/runtime-pack-matrix.mjs +++ b/packages/runtime-pack-common/runtime-pack-matrix.mjs @@ -81,7 +81,10 @@ export const OPTIONAL_RUNTIME_SOURCES = [ ["scr_js_value.c", "dynTier"], ["scr_js_object.c", "dynTier"], ["scr_event_target.c", any("eventTarget", "nativeFetch", "abortSignal")], - ["scr_async_dyn.c", any("dynAsync", "dynInvoke", "dc", "fileHandle", "nativeFetch")], + [ + "scr_async_dyn.c", + any("dynAsync", "dynInvoke", "dc", "fileHandle", "nativeFetch", "nodeIsland"), + ], // The crypto global's object extends scr_async_dyn.c's crypto object. ["scr_webcrypto_object.c", all("webcrypto", "dynAsync")], // The performance timeline emits its buffer warning through scr_async_dyn.c. @@ -191,6 +194,7 @@ export const OPTIONAL_RUNTIME_SOURCES = [ // Node islands: libnode loaded on first use (explicit @scriptc/node calls), // over the SCR_NODE_ISLANDS build of scr_async.c. ["scr_node_island.c", "nodeIsland"], + ["scr_node_island_handle.c", "nodeIsland"], ["scr_stream_from_web.c", all("stream", "nativeFetch")], ]; @@ -201,6 +205,18 @@ export function variantsFor(source) { // other program links the ordinary object, byte for byte. if (source === "scr_json.c") { variants.push({ id: "dyn-tier", when: { dynTier: true }, defines: ["SCR_JS_TIER"] }); + // Dynamic operations on Node island handles (typeof, calls, `in`, + // delete, Object.keys) route to scr_node_island_handle.c. + variants.push({ + id: "node-island", + when: { nodeIsland: true }, + defines: ["SCR_NODE_ISLANDS"], + }); + variants.push({ + id: "dyn-tier-node-island", + when: { dynTier: true, nodeIsland: true }, + defines: ["SCR_JS_TIER", "SCR_NODE_ISLANDS"], + }); // Programs with web streams: instanceof over their classes' handles. variants.push({ id: "handle-classes", @@ -212,6 +228,26 @@ export function variantsFor(source) { when: { dynTier: true, webStreams: true }, defines: ["SCR_JS_TIER", "SCR_HANDLE_CLASSES"], }); + // Both: the variant with the most conditions is chosen, so programs + // with web streams and Node islands need the builds that define both. + variants.push({ + id: "handle-classes-node-island", + when: { webStreams: true, nodeIsland: true }, + defines: ["SCR_HANDLE_CLASSES", "SCR_NODE_ISLANDS"], + }); + variants.push({ + id: "dyn-tier-handle-classes-node-island", + when: { dynTier: true, webStreams: true, nodeIsland: true }, + defines: ["SCR_JS_TIER", "SCR_HANDLE_CLASSES", "SCR_NODE_ISLANDS"], + }); + } + // util.inspect and instanceof of Node island handles. + if (source === "scr_inspect.c" || source === "scr_dyn_builtin_class.c") { + variants.push({ + id: "node-island", + when: { nodeIsland: true }, + defines: ["SCR_NODE_ISLANDS"], + }); } if (source === "scr_bytes.c") { variants.push({ @@ -239,6 +275,14 @@ export function variantsFor(source) { when: { webStreams: true }, defines: ["SCR_NATIVE_REACTIONS"], }); + // A program with both web streams and Node islands needs both builds: + // the island slot carries the hosted loop, and the variant with the + // most conditions is the one selected. + variants.push({ + id: "native-reactions-node-island", + when: { webStreams: true, nodeIsland: true }, + defines: ["SCR_NATIVE_REACTIONS", "SCR_NODE_ISLANDS"], + }); } if (source === "scr_dyn_builtin_class.c" || source === "scr_fetch.c") { variants.push({ @@ -247,6 +291,14 @@ export function variantsFor(source) { defines: ["SCR_HANDLE_CLASSES"], }); } + // instanceof over both web stream and Node island handles. + if (source === "scr_dyn_builtin_class.c") { + variants.push({ + id: "handle-classes-node-island", + when: { webStreams: true, nodeIsland: true }, + defines: ["SCR_HANDLE_CLASSES", "SCR_NODE_ISLANDS"], + }); + } // stdin setEncoding, string data listeners and stdio 'resize' events. if (source === "scr_events.c") { variants.push({ @@ -254,6 +306,17 @@ export function variantsFor(source) { when: { stdioExtended: true }, defines: ["SCR_STDIO_EXTENDED"], }); + // Island listeners for signals and 'exit' join the registries. + variants.push({ + id: "node-island", + when: { nodeIsland: true }, + defines: ["SCR_NODE_ISLANDS"], + }); + variants.push({ + id: "stdio-extended-node-island", + when: { stdioExtended: true, nodeIsland: true }, + defines: ["SCR_STDIO_EXTENDED", "SCR_NODE_ISLANDS"], + }); } // Programs with Node islands link the event loop with the Node island slot; // every other program keeps the default object, byte for byte. diff --git a/packages/runtime-pack-common/test/runtime-pack-matrix.test.ts b/packages/runtime-pack-common/test/runtime-pack-matrix.test.ts new file mode 100644 index 000000000..9157d7bd0 --- /dev/null +++ b/packages/runtime-pack-common/test/runtime-pack-matrix.test.ts @@ -0,0 +1,42 @@ +import { expect, test } from "vitest"; +import { createRuntimePackMatrix } from "../runtime-pack-matrix.mjs"; + +const matrix = createRuntimePackMatrix({ + target: { object_format: "elf" }, + compileFlags: [], + systemLibraries: [], +}); + +/** The runtime pack's variant selection: the matching variant with the + * most conditions, ties by id. */ +function selected( + source: string, + features: Record, +): { id: string; defines: string[] } { + const unit = matrix.runtime_units.find((u: { source: string }) => u.source === source); + const matches = unit.variants.filter((variant: { when: Record }) => + Object.entries(variant.when).every(([name, expected]) => features[name] === expected), + ); + matches.sort( + (a: { when: object; id: string }, b: { when: object; id: string }) => + Object.keys(b.when).length - Object.keys(a.when).length || a.id.localeCompare(b.id), + ); + return matches[0]; +} + +test("web streams and Node islands select an async build that defines both", () => { + const variant = selected("scr_async.c", { webStreams: true, nodeIsland: true, workers: false }); + expect(variant.defines).toEqual( + expect.arrayContaining(["SCR_NATIVE_REACTIONS", "SCR_NODE_ISLANDS"]), + ); + // Each feature alone still selects its own build, byte for byte. + expect( + selected("scr_async.c", { webStreams: true, nodeIsland: false, workers: false }).defines, + ).toEqual(["SCR_NATIVE_REACTIONS"]); + expect( + selected("scr_async.c", { webStreams: false, nodeIsland: true, workers: false }).defines, + ).toEqual(["SCR_NODE_ISLANDS"]); + expect( + selected("scr_async.c", { webStreams: false, nodeIsland: false, workers: false }).defines, + ).toEqual([]); +}); diff --git a/packages/runtime/src/scr_async.c b/packages/runtime/src/scr_async.c index ef943f356..9d07412c0 100644 --- a/packages/runtime/src/scr_async.c +++ b/packages/runtime/src/scr_async.c @@ -26,6 +26,10 @@ #define _DEFAULT_SOURCE 1 #endif #include "scr_runtime.h" +#ifdef SCR_NODE_ISLANDS +#include "scr_node_bridge.h" +#include "scr_node_island.h" +#endif #include #include @@ -940,7 +944,44 @@ void scr_wasi_coro_started(void *handle) { static SCR_TL ScrFiber **scr_ready = NULL; static SCR_TL size_t scr_ready_head = 0, scr_ready_len = 0, scr_ready_cap = 0; +#ifdef SCR_NODE_ISLANDS +/* The hosted loop (scope-B NI-3, defined below the loop hooks): once Node + * is loaded, the runtime's queues live on Node's and this names the + * libnode bridge that schedules them. NULL until then, and in programs + * that never load Node. */ +static SCR_TL const ScrNodeBridge *scr_loop_hosted = NULL; +/* Runtime ticks and microtasks queued on Node and not yet run: the + * dispatch stations' yield signal (scr_loop_has_ready). */ +static SCR_TL size_t scr_hosted_queued = 0; +/* The end of the check phase the loop pass is running (0: none). */ +static SCR_TL size_t scr_check_end = 0; +/* Where the loop pass is, for a switch in the middle of it: the turn + * continues at Node's poll phase after a timers or I/O phase, and a + * microtask drain in progress finishes before the ticks. */ +enum { SCR_PASS_START, SCR_PASS_IO, SCR_PASS_TIMERS, SCR_PASS_CHECK }; +static SCR_TL int scr_pass_phase = SCR_PASS_START; +static SCR_TL bool scr_in_microtask = false; +static SCR_TL bool scr_hosted_continue_turn = false; +static SCR_TL bool scr_hosted_microtasks_first = false; +#define SCR_PASS_PHASE(phase) (scr_pass_phase = (phase)) +#define SCR_RESUME_JOB(f) (scr_in_microtask = true, scr_resume_fiber(f), scr_in_microtask = false) +static void scr_hosted_microtask(ScrFiber *f); +/* A microtask turn may continue in place only while the runtime's own + * queue is the whole queue: hosted, V8's queue may hold Node's jobs. */ +#define SCR_READY_IDLE() (scr_ready_len == 0 && scr_loop_hosted == NULL) +#else +#define SCR_READY_IDLE() (scr_ready_len == 0) +#define SCR_PASS_PHASE(phase) ((void)0) +#define SCR_RESUME_JOB(f) scr_resume_fiber(f) +#endif + static void scr_ready_push(ScrFiber *f) { +#ifdef SCR_NODE_ISLANDS + if (scr_loop_hosted != NULL) { + scr_hosted_microtask(f); + return; + } +#endif if (scr_ready_head + scr_ready_len == scr_ready_cap) { if (scr_ready_len > 0) { memmove(scr_ready, scr_ready + scr_ready_head, scr_ready_len * sizeof *scr_ready); @@ -1073,7 +1114,25 @@ static void scr_timer_swap(size_t i, size_t j) { scr_timer_set(j, tmp); } +#ifdef SCR_NODE_ISLANDS +static void scr_hosted_timer_push(ScrTimer t); +/* The hosted timer, immediate and tick operations (defined with the + * hosted loop below). Each answers false when the handle names nothing. */ +enum { SCR_HOSTED_CLEAR, SCR_HOSTED_REF, SCR_HOSTED_UNREF, SCR_HOSTED_REFRESH, SCR_HOSTED_HAS_REF }; +static bool scr_hosted_timer_op(unsigned long id, int op); +static double scr_hosted_set_immediate(ScrClosure *cb); +static bool scr_hosted_immediate_op(unsigned long id, int op); +static size_t scr_hosted_count(int kind); +static void scr_hosted_teardown(void); +#endif + static void scr_timer_push(ScrTimer t) { +#ifdef SCR_NODE_ISLANDS + if (scr_loop_hosted != NULL) { + scr_hosted_timer_push(t); + return; + } +#endif if (scr_ntimers == scr_timers_cap) { scr_timers_cap = scr_timers_cap ? scr_timers_cap * 2 : 16; scr_timers = realloc(scr_timers, scr_timers_cap * sizeof *scr_timers); @@ -1206,6 +1265,12 @@ void scr_timer_unref(double handle) { scr_firing_reffed = false; return; } +#ifdef SCR_NODE_ISLANDS + if (scr_loop_hosted != NULL) { + scr_hosted_timer_op(id, SCR_HOSTED_UNREF); + return; + } +#endif ScrTimer *t = scr_timer_find(id); if (t && t->reffed) { t->reffed = false; @@ -1220,6 +1285,12 @@ void scr_timer_ref(double handle) { scr_firing_reffed = true; return; } +#ifdef SCR_NODE_ISLANDS + if (scr_loop_hosted != NULL) { + scr_hosted_timer_op(id, SCR_HOSTED_REF); + return; + } +#endif ScrTimer *t = scr_timer_find(id); if (t && !t->reffed) { t->reffed = true; @@ -1243,6 +1314,12 @@ void scr_timer_refresh(double handle) { scr_firing_refresh = true; return; } +#ifdef SCR_NODE_ISLANDS + if (scr_loop_hosted != NULL) { + scr_hosted_timer_op(id, SCR_HOSTED_REFRESH); + return; + } +#endif ScrTimer *found = scr_timer_find(id); if (found) { ScrTimer t = *found; @@ -1258,6 +1335,9 @@ bool scr_timer_has_ref(double handle) { if (!(handle >= 1)) return false; unsigned long id = (unsigned long)handle; if (id == scr_firing_id) return scr_firing_reffed; +#ifdef SCR_NODE_ISLANDS + if (scr_loop_hosted != NULL) return scr_hosted_timer_op(id, SCR_HOSTED_HAS_REF); +#endif ScrTimer *t = scr_timer_find(id); return t != NULL && t->reffed; } @@ -1274,8 +1354,19 @@ static SCR_TL size_t scr_pending_immediates; /* defined below with the queue */ ScrArr *scr_active_resources(void) { ScrArr *arr = scr_arr_new(SCR_ELEM_STR, 8); size_t timeouts = scr_ntimers + ((scr_firing_id != 0 && !scr_firing_cleared) ? 1 : 0); +#ifdef SCR_NODE_ISLANDS + /* Hosted: the runtime's own (Node's are Node's to report). */ + size_t immediates = scr_pending_immediates; + if (scr_loop_hosted != NULL) { + timeouts = scr_hosted_count(0); + immediates = scr_hosted_count(1); + } + for (size_t i = 0; i < timeouts; i++) scr_arr_push_ref(arr, scr_str_new("Timeout", 7)); + for (size_t i = 0; i < immediates; i++) scr_arr_push_ref(arr, scr_str_new("Immediate", 9)); +#else for (size_t i = 0; i < timeouts; i++) scr_arr_push_ref(arr, scr_str_new("Timeout", 7)); for (size_t i = 0; i < scr_pending_immediates; i++) scr_arr_push_ref(arr, scr_str_new("Immediate", 9)); +#endif return arr; } @@ -1301,6 +1392,9 @@ static SCR_TL ScrNtick *scr_nt_tail = NULL; * it, because 'exit' listeners run AFTER the loop's teardown and may * enqueue ticks that must never run (Node) yet must not leak. */ +#ifdef SCR_NODE_ISLANDS +static void scr_hosted_tick(ScrNtick *t); +#endif static ScrClosure *scr_als_bind(ScrClosure *cb /*moves*/); /* A tick runs with the AsyncLocalStorage context active when it was @@ -1309,6 +1403,12 @@ void scr_next_tick(ScrClosure *cb /*moves*/) { ScrNtick *t = scr_mem_calloc(sizeof *t); if (!t) scr_oom(); t->cb = scr_als_bind(cb); +#ifdef SCR_NODE_ISLANDS + if (scr_loop_hosted != NULL) { + scr_hosted_tick(t); + return; + } +#endif if (scr_nt_tail) scr_nt_tail->next = t; else scr_nt_head = t; scr_nt_tail = t; @@ -1323,6 +1423,12 @@ void scr_process_write_callback(ScrClosure *cb /*moves*/, ScrFsRenameFn fn) { if (!t) scr_oom(); t->cb = cb; t->error_cb = fn; +#ifdef SCR_NODE_ISLANDS + if (scr_loop_hosted != NULL) { + scr_hosted_tick(t); + return; + } +#endif if (scr_nt_tail) scr_nt_tail->next = t; else scr_nt_head = t; scr_nt_tail = t; @@ -1337,6 +1443,12 @@ void scr_next_tick_raw(void (*fn)(void)) { ScrNtick *t = scr_mem_calloc(sizeof *t); if (!t) scr_oom(); t->raw = fn; +#ifdef SCR_NODE_ISLANDS + if (scr_loop_hosted != NULL) { + scr_hosted_tick(t); + return; + } +#endif if (scr_nt_tail) scr_nt_tail->next = t; else scr_nt_head = t; scr_nt_tail = t; @@ -1348,6 +1460,13 @@ static void scr_next_tick_raw_first(void (*fn)(void)) { ScrNtick *t = scr_mem_calloc(sizeof *t); if (!t) scr_oom(); t->raw = fn; + /* Hosted, Node's tick queue has no front: the tick queues last. */ +#ifdef SCR_NODE_ISLANDS + if (scr_loop_hosted != NULL) { + scr_hosted_tick(t); + return; + } +#endif t->next = scr_nt_head; scr_nt_head = t; if (scr_nt_tail == NULL) scr_nt_tail = t; @@ -1365,6 +1484,9 @@ void scr_nticks_teardown(void) { static void scr_immediates_teardown(void); void scr_timers_teardown(void) { +#ifdef SCR_NODE_ISLANDS + scr_hosted_teardown(); +#endif for (size_t i = 0; i < scr_ntimers; i++) scr_closure_release(scr_timers[i].cb); scr_ntimers = 0; scr_reffed_timers = 0; @@ -1382,6 +1504,12 @@ void scr_clear_interval(double handle) { scr_firing_cleared = true; /* the run loop drops the callback */ return; } +#ifdef SCR_NODE_ISLANDS + if (scr_loop_hosted != NULL) { + scr_hosted_timer_op(id, SCR_HOSTED_CLEAR); + return; + } +#endif ScrTimer *found = scr_timer_find(id); if (found) { scr_closure_release(found->cb); @@ -1411,6 +1539,9 @@ static SCR_TL size_t scr_pending_immediates = 0; static SCR_TL size_t scr_reffed_immediates = 0; double scr_set_immediate(ScrClosure *cb) { +#ifdef SCR_NODE_ISLANDS + if (scr_loop_hosted != NULL) return scr_hosted_set_immediate(cb); +#endif if (scr_nimmediates == scr_immediates_cap) { scr_immediates_cap = scr_immediates_cap ? scr_immediates_cap * 2 : 16; scr_immediates = realloc(scr_immediates, scr_immediates_cap * sizeof *scr_immediates); @@ -1435,6 +1566,12 @@ static ScrImmediate *scr_immediate_find(unsigned long id) { void scr_clear_immediate(double handle) { if (!(handle >= 1)) return; /* NaN/0/negative: never a handle; Node tolerates */ +#ifdef SCR_NODE_ISLANDS + if (scr_loop_hosted != NULL) { + scr_hosted_immediate_op((unsigned long)handle, SCR_HOSTED_CLEAR); + return; + } +#endif ScrImmediate *im = scr_immediate_find((unsigned long)handle); if (im == NULL) return; /* already fired/cleared: no-op, like Node */ scr_closure_release(im->cb); @@ -1451,6 +1588,12 @@ void scr_clear_immediate(double handle) { * (Node's Immediate methods on a dead handle do nothing). */ void scr_immediate_unref(double handle) { if (!(handle >= 1)) return; +#ifdef SCR_NODE_ISLANDS + if (scr_loop_hosted != NULL) { + scr_hosted_immediate_op((unsigned long)handle, SCR_HOSTED_UNREF); + return; + } +#endif ScrImmediate *im = scr_immediate_find((unsigned long)handle); if (im && im->reffed) { im->reffed = false; @@ -1460,6 +1603,12 @@ void scr_immediate_unref(double handle) { void scr_immediate_ref(double handle) { if (!(handle >= 1)) return; +#ifdef SCR_NODE_ISLANDS + if (scr_loop_hosted != NULL) { + scr_hosted_immediate_op((unsigned long)handle, SCR_HOSTED_REF); + return; + } +#endif ScrImmediate *im = scr_immediate_find((unsigned long)handle); if (im && !im->reffed) { im->reffed = true; @@ -1469,6 +1618,9 @@ void scr_immediate_ref(double handle) { bool scr_immediate_has_ref(double handle) { if (!(handle >= 1)) return false; +#ifdef SCR_NODE_ISLANDS + if (scr_loop_hosted != NULL) return scr_hosted_immediate_op((unsigned long)handle, SCR_HOSTED_HAS_REF); +#endif ScrImmediate *im = scr_immediate_find((unsigned long)handle); return im != NULL && im->reffed; } @@ -1539,6 +1691,10 @@ static void scr_switch(ScrCtx *from, ScrCtx *to, ScrFiber *to_fiber) { uintptr_t guard = scr_stack_guard; if (to_fiber) SCR_STACK_GUARD_SET(scr_stack_guard_for((uintptr_t)to_fiber->stack, SCR_FIBER_STACK)); #endif +#ifdef SCR_NODE_ISLANDS + /* Leaving Node's stack keeps this frame and those above it live. */ + scr_node_island_leave_stack((uintptr_t)__builtin_frame_address(0)); +#endif #ifdef _WIN32 /* SwitchToFiber snapshots the outgoing fiber's state inside its own * fiber object — `from` has nothing to record. */ @@ -2625,7 +2781,7 @@ void scr_gen_coro_finish(ScrFiber *self) { * continues a fiber in place: resumed by the loop's drain with nothing * else queued, this body is the very next job either way. */ static bool scr_coro_hop_needed(ScrFiber *self) { - if (self->return_to == &scr_loop_ctx && scr_ready_len == 0) return false; + if (self->return_to == &scr_loop_ctx && SCR_READY_IDLE()) return false; scr_ready_push(self); return true; } @@ -2772,7 +2928,7 @@ static void scr_await_yield(void) { * resumed by its spawner (the eager prefix) must still hop: the spawner * has code to run first. */ ScrCtx *to = scr_fiber_park_awaiters(self); - if (to == &scr_loop_ctx && scr_ready_len == 0) return; + if (to == &scr_loop_ctx && SCR_READY_IDLE()) return; scr_ready_push(self); scr_switch(&self->ctx, to, NULL); } @@ -4374,28 +4530,6 @@ void scr_loop_set_watch(bool (*pending)(void), void (*dispatch)(void), int (*pol scr_watch_pollfd_fn = pollfd; } -#ifdef SCR_NODE_ISLANDS -/* The Node island hook (scr_node_island.c): the watch hook's shape plus a - * sleep bound, in the SCR_NODE_ISLANDS build of this unit only, so - * programs without Node islands link the plain object byte for byte. Once - * Node is loaded, `pending` keeps the loop alive while Node's own loop has - * ref'd work, `dispatch` runs one non-blocking turn of it at every turn - * top, `pollfd` is its backend fd for the idle poll(2) sleep and `timeout` - * the milliseconds to its next timer (-1: none). NI-3 replaces this with - * the runtime's queues hosted on Node's loop. */ -static SCR_TL bool (*scr_node_island_pending_fn)(void) = NULL; -static SCR_TL void (*scr_node_island_dispatch_fn)(void) = NULL; -static SCR_TL int (*scr_node_island_pollfd_fn)(void) = NULL; -static SCR_TL int (*scr_node_island_timeout_fn)(void) = NULL; - -void scr_loop_set_node_island(bool (*pending)(void), void (*dispatch)(void), int (*pollfd)(void), - int (*timeout)(void)) { - scr_node_island_pending_fn = pending; - scr_node_island_dispatch_fn = dispatch; - scr_node_island_pollfd_fn = pollfd; - scr_node_island_timeout_fn = timeout; -} -#endif /* The child-process hook (scr_child.c, installed with the first spawned * child or IPC channel). Programs that never create one keep it NULL and @@ -4479,7 +4613,12 @@ static int scr_workers_pollfd(void) { * callbacks or fibers are already queued, so ticks and promise jobs * interleave with event delivery (Node's checkpoint after each * macrotask). */ -bool scr_loop_has_ready(void) { return scr_ready_len > 0 || scr_nt_head != NULL; } +bool scr_loop_has_ready(void) { +#ifdef SCR_NODE_ISLANDS + if (scr_loop_hosted != NULL) return scr_hosted_queued > 0; +#endif + return scr_ready_len > 0 || scr_nt_head != NULL; +} /* The poll phase runs once per loop iteration, like libuv's: the net unit * drains the poller at most once between check phases. I/O that becomes @@ -4500,6 +4639,670 @@ bool scr_loop_io_take(void) { bool scr_loop_io_more(void) { return ++scr_loop_io_batches < SCR_LOOP_IO_BATCHES; } void scr_loop_io_reopen(void) { scr_loop_io_open = true; } +#ifdef SCR_NODE_ISLANDS +/* ── the hosted loop (scope-B NI-3) ─────────────────────────────────────── + * Once Node is loaded into the process there is one event loop, Node's, + * and one set of queues. scr_loop_host (called by scr_node_island.c right + * after Node starts) migrates the runtime's pending work onto Node's + * queues in order and points every scheduling primitive at the bridge: + * + * - promise jobs and queueMicrotask envelopes (the READY queue) become V8 + * microtasks, ticks become process.nextTick callbacks, so the jobs of + * both sides interleave in one FIFO each, exactly as in Node; + * - timers become Node timers, created in (deadline, sequence) order with + * their remaining delay measured from the moment Node began loading (the + * load itself is invisible to timer order, as it is to a program that + * runs on Node from the start); immediates become Node immediates; + * - the runtime's I/O hooks (the scr_loop_set_* slots) are watched by the + * bridge: their fds as uv_poll handles, their liveness as the ref of one + * handle, always-ready work as an idle handle; scr_loop_hosted_io runs + * the dispatch stations with Node's checkpoint after each. + * + * Every runtime job runs through scr_loop_hosted_run on the stack of + * Node's own dispatch, so Node drains ticks and microtasks after it as + * after any callback. A throw that escapes a job goes through the + * runtime's uncaught machinery (process 'uncaughtException' listeners) + * and, unhandled, ends the process as the runtime's main would. */ + +typedef enum { + SCR_HJ_FREE = 0, + SCR_HJ_MICRO = SCR_NODE_JOB_MICROTASK, + SCR_HJ_TICK = SCR_NODE_JOB_TICK, + SCR_HJ_TIMER = SCR_NODE_JOB_TIMER, + SCR_HJ_IMMEDIATE = SCR_NODE_JOB_IMMEDIATE, +} ScrHJobKind; + +/* One scheduled runtime job; its token (index + 1) travels through the + * bridge. Entries move when the table grows: never hold a pointer across + * a callback. */ +typedef struct { + uint8_t kind; + bool reffed; + bool firing; /* a timer whose callback is running */ + bool cleared; /* ... and cleared from inside it */ + bool refreshed; /* ... and refreshed from inside it */ + bool due; /* runs before Node's loop starts (handle 0 until then) */ + uint32_t handle; /* the bridge's timer or immediate */ + uint32_t next_free; + unsigned long id; /* the program's handle; 0 for plain setTimeout */ + double repeat_ms; + double delay_ms; /* the original delay: refresh() of a due timer re-arms with it */ + void *p; /* ScrFiber *, ScrNtick *, or the owned ScrClosure * */ +} ScrHJob; + + +static SCR_TL ScrHJob *scr_hjobs = NULL; +static SCR_TL uint32_t scr_hjobs_len = 0, scr_hjobs_cap = 0, scr_hjobs_free = 0; + +static uint32_t scr_hjob_new(uint8_t kind, void *p) { + uint32_t token; + if (scr_hjobs_free != 0) { + token = scr_hjobs_free; + scr_hjobs_free = scr_hjobs[token - 1].next_free; + } else { + if (scr_hjobs_len == scr_hjobs_cap) { + scr_hjobs_cap = scr_hjobs_cap ? scr_hjobs_cap * 2 : 64; + scr_hjobs = realloc(scr_hjobs, scr_hjobs_cap * sizeof *scr_hjobs); + if (!scr_hjobs) scr_oom(); + } + token = ++scr_hjobs_len; + } + ScrHJob *j = &scr_hjobs[token - 1]; + memset(j, 0, sizeof *j); + j->kind = kind; + j->p = p; + return token; +} + +static ScrHJob *scr_hjob(uint32_t token, uint8_t kind) { + if (token == 0 || token > scr_hjobs_len) return NULL; + ScrHJob *j = &scr_hjobs[token - 1]; + return j->kind == kind ? j : NULL; +} + +static void scr_hjob_free(uint32_t token) { + ScrHJob *j = &scr_hjobs[token - 1]; + j->kind = SCR_HJ_FREE; + j->due = false; + j->p = NULL; + j->next_free = scr_hjobs_free; + scr_hjobs_free = token; +} + +/* Work the loop pass was about to run when Node loaded: immediates left in + * the check phase being run, and timers already due. They run, in that + * order, before Node's loop starts (scr_loop_run_hosted), as they would + * have in the phase the switch interrupted; Node would never have started + * a new turn first. */ +static SCR_TL uint32_t *scr_hosted_due = NULL; +static SCR_TL size_t scr_hosted_due_len = 0, scr_hosted_due_cap = 0; + +static void scr_hosted_due_push(uint32_t token) { + if (scr_hosted_due_len == scr_hosted_due_cap) { + scr_hosted_due_cap = scr_hosted_due_cap ? scr_hosted_due_cap * 2 : 16; + scr_hosted_due = realloc(scr_hosted_due, scr_hosted_due_cap * sizeof *scr_hosted_due); + if (!scr_hosted_due) scr_oom(); + } + scr_hosted_due[scr_hosted_due_len++] = token; + scr_hjobs[token - 1].due = true; +} + +/* Program handle → token, for timers (key id*2) and immediates (id*2+1). + * Open addressing with linear probing and backward-shift deletion. */ +typedef struct { + unsigned long key; /* 0 = empty */ + uint32_t token; +} ScrHSlot; +static SCR_TL ScrHSlot *scr_hmap = NULL; +static SCR_TL size_t scr_hmap_cap = 0, scr_hmap_len = 0; + +static size_t scr_hmap_home(unsigned long key) { + uint64_t h = (uint64_t)key * UINT64_C(0x9E3779B97F4A7C15); + return (size_t)(h >> 32) & (scr_hmap_cap - 1); +} + +static void scr_hmap_put(unsigned long key, uint32_t token); + +static void scr_hmap_grow(void) { + ScrHSlot *old = scr_hmap; + size_t old_cap = scr_hmap_cap; + scr_hmap_cap = old_cap ? old_cap * 2 : 64; + scr_hmap = calloc(scr_hmap_cap, sizeof *scr_hmap); + if (!scr_hmap) scr_oom(); + scr_hmap_len = 0; + for (size_t i = 0; i < old_cap; i++) + if (old[i].key != 0) scr_hmap_put(old[i].key, old[i].token); + free(old); +} + +static void scr_hmap_put(unsigned long key, uint32_t token) { + if ((scr_hmap_len + 1) * 2 > scr_hmap_cap) scr_hmap_grow(); + size_t mask = scr_hmap_cap - 1, i = scr_hmap_home(key); + while (scr_hmap[i].key != 0 && scr_hmap[i].key != key) i = (i + 1) & mask; + if (scr_hmap[i].key == 0) scr_hmap_len++; + scr_hmap[i].key = key; + scr_hmap[i].token = token; +} + +static size_t scr_hmap_find(unsigned long key) { + if (scr_hmap_cap == 0) return SIZE_MAX; + size_t mask = scr_hmap_cap - 1; + for (size_t i = scr_hmap_home(key); scr_hmap[i].key != 0; i = (i + 1) & mask) + if (scr_hmap[i].key == key) return i; + return SIZE_MAX; +} + +static void scr_hmap_delete(unsigned long key) { + size_t i = scr_hmap_find(key); + if (i == SIZE_MAX) return; + size_t mask = scr_hmap_cap - 1; + for (size_t j = (i + 1) & mask; scr_hmap[j].key != 0; j = (j + 1) & mask) { + size_t home = scr_hmap_home(scr_hmap[j].key); + bool stays = i <= j ? (i < home && home <= j) : (i < home || home <= j); + if (stays) continue; + scr_hmap[i] = scr_hmap[j]; + i = j; + } + scr_hmap[i].key = 0; + scr_hmap_len--; +} + +static uint32_t scr_hmap_token(unsigned long key) { + size_t i = scr_hmap_find(key); + return i == SIZE_MAX ? 0 : scr_hmap[i].token; +} + +/* ── scheduling ── */ + +static void scr_hosted_microtask(ScrFiber *f) { + uint32_t token = scr_hjob_new(SCR_HJ_MICRO, f); + scr_hosted_queued++; + scr_loop_hosted->microtask(token); +} + +static void scr_hosted_tick(ScrNtick *t) { + uint32_t token = scr_hjob_new(SCR_HJ_TICK, t); + scr_hosted_queued++; + scr_loop_hosted->tick(token); +} + +/* Node's timer delay for a remaining time: whole milliseconds, at least + * one (Node's clamp), rounded up so a timer never fires before its + * deadline. */ +static double scr_hosted_delay(double remaining) { + if (!(remaining >= 1)) return 1; + if (remaining > 2147483647.0) return 2147483647.0; + double delay = (double)(long long)remaining; + return delay < remaining - 1e-6 ? delay + 1 : delay; +} + +/* A runtime timer on Node's loop; a `due` one waits for scr_loop_run_hosted + * and is armed only if it runs again. */ +static void scr_hosted_timer_arm(ScrClosure *cb, double delay, double repeat, double delay_ms, + unsigned long id, bool reffed, bool due) { + uint32_t token = scr_hjob_new(SCR_HJ_TIMER, cb); + ScrHJob *j = &scr_hjobs[token - 1]; + j->id = id; + j->repeat_ms = repeat; + j->delay_ms = delay_ms; + j->reffed = reffed; + if (id != 0) scr_hmap_put(id * 2, token); + if (due) { + scr_hosted_due_push(token); + return; + } + uint32_t handle = scr_loop_hosted->timer(delay, repeat, reffed ? 1 : 0, token); + scr_hjobs[token - 1].handle = handle; +} + +static void scr_hosted_timer_push(ScrTimer t) { + scr_hosted_timer_arm(t.cb, scr_hosted_delay(t.deadline_ms - scr_now_ms()), t.repeat_ms, + t.delay_ms, t.id, t.reffed, false); +} + +static void scr_hosted_timer_drop(uint32_t token) { + ScrHJob *j = &scr_hjobs[token - 1]; + ScrClosure *cb = (ScrClosure *)j->p; + if (j->id != 0) scr_hmap_delete(j->id * 2); + scr_hjob_free(token); + scr_closure_release(cb); +} + +/* refresh(): re-arm to now + the delay the timer was created with, Node's + * Timeout.refresh(). A migrated timer's Node handle was created with its + * remaining delay at handoff, and a due one has no handle yet, so the + * handle is replaced rather than refreshed. A firing timer's flag makes the + * run loop keep the re-armed handle when its callback returns. */ +static void scr_hosted_timer_rearm(uint32_t token) { + ScrHJob *j = &scr_hjobs[token - 1]; + if (j->handle != 0) scr_loop_hosted->timer_clear(j->handle); + j->due = false; + j->handle = scr_loop_hosted->timer(scr_hosted_delay(j->delay_ms), j->repeat_ms, + j->reffed ? 1 : 0, token); +} + +static bool scr_hosted_timer_op(unsigned long id, int op) { + uint32_t token = scr_hmap_token(id * 2); + ScrHJob *j = scr_hjob(token, SCR_HJ_TIMER); + if (j == NULL) return false; + switch (op) { + case SCR_HOSTED_CLEAR: + scr_loop_hosted->timer_clear(j->handle); + if (j->firing) { + j->cleared = true; /* dropped when its callback returns */ + scr_hmap_delete(id * 2); + } else { + scr_hosted_timer_drop(token); + } + return true; + case SCR_HOSTED_REF: + case SCR_HOSTED_UNREF: + j->reffed = op == SCR_HOSTED_REF; + scr_loop_hosted->timer_ref(j->handle, j->reffed ? 1 : 0); + return true; + case SCR_HOSTED_REFRESH: + if (j->firing) j->refreshed = true; + scr_hosted_timer_rearm(token); + return true; + default: + return j->reffed; + } +} + +static double scr_hosted_set_immediate(ScrClosure *cb) { + unsigned long id = ++scr_immediate_seq; + uint32_t token = scr_hjob_new(SCR_HJ_IMMEDIATE, cb); + scr_hjobs[token - 1].id = id; + scr_hjobs[token - 1].reffed = true; + scr_hmap_put(id * 2 + 1, token); + uint32_t handle = scr_loop_hosted->immediate(1, token); + scr_hjobs[token - 1].handle = handle; + return (double)id; +} + +static bool scr_hosted_immediate_op(unsigned long id, int op) { + uint32_t token = scr_hmap_token(id * 2 + 1); + ScrHJob *j = scr_hjob(token, SCR_HJ_IMMEDIATE); + if (j == NULL) return false; + switch (op) { + case SCR_HOSTED_CLEAR: { + ScrClosure *cb = (ScrClosure *)j->p; + scr_loop_hosted->immediate_clear(j->handle); + scr_hmap_delete(id * 2 + 1); + scr_hjob_free(token); + scr_closure_release(cb); + return true; + } + case SCR_HOSTED_REF: + case SCR_HOSTED_UNREF: + j->reffed = op == SCR_HOSTED_REF; + scr_loop_hosted->immediate_ref(j->handle, j->reffed ? 1 : 0); + return true; + default: + return j->reffed; + } +} + +/* Live runtime timers (kind 0) or immediates (kind 1). */ +static size_t scr_hosted_count(int kind) { + uint8_t want = kind == 0 ? SCR_HJ_TIMER : SCR_HJ_IMMEDIATE; + size_t n = 0; + for (uint32_t i = 0; i < scr_hjobs_len; i++) + if (scr_hjobs[i].kind == want && !scr_hjobs[i].cleared) n++; + return n; +} + +/* Loop exit: timers and immediates that never ran (unref'd ones, or what + * remained at process exit) release their closures, like the plain heap's + * teardown; Node keeps its objects, which can no longer fire. */ +static void scr_hosted_teardown(void) { + for (uint32_t i = 0; i < scr_hjobs_len; i++) { + ScrHJob *j = &scr_hjobs[i]; + if ((j->kind == SCR_HJ_TIMER || j->kind == SCR_HJ_IMMEDIATE) && !j->firing) { + ScrClosure *cb = (ScrClosure *)j->p; + j->kind = SCR_HJ_FREE; + j->p = NULL; + j->next_free = scr_hjobs_free; + scr_hjobs_free = i + 1; + scr_closure_release(cb); + } + } + free(scr_hmap); + scr_hmap = NULL; + scr_hmap_cap = scr_hmap_len = 0; +} + +/* ── running jobs ── */ + +/* After a runtime callback: an escaped throw goes to the runtime's + * uncaught machinery; unhandled, it ends the process like main's + * uncaught path (report, exit listeners, the noted code). */ +static void scr_hosted_settle_exception(void) { + if (!scr_exc_pending()) return; + if (scr_exc_handle_uncaught(false)) return; + scr_exc_print_uncaught(); + scr_process_exit((double)scr_exit_code_hint_get()); +} + +void scr_loop_hosted_run(uint32_t kind, uint32_t token) { + switch (kind) { + case SCR_HJ_MICRO: { + ScrHJob *j = scr_hjob(token, SCR_HJ_MICRO); + if (j == NULL) return; + ScrFiber *f = (ScrFiber *)j->p; + scr_hjob_free(token); + scr_hosted_queued--; + scr_resume_fiber(f); + break; + } + case SCR_HJ_TICK: { + ScrHJob *j = scr_hjob(token, SCR_HJ_TICK); + if (j == NULL) return; + ScrNtick *t = (ScrNtick *)j->p; + scr_hjob_free(token); + scr_hosted_queued--; + ScrClosure *cb = t->cb; + ScrFsRenameFn error_cb = t->error_cb; + void (*raw)(void) = t->raw; + scr_mem_free(t); + if (cb) { + if (error_cb) error_cb(cb, NULL); + else ((void (*)(ScrClosure *))cb->fn)(cb); + scr_closure_release(cb); + } else { + raw(); + } + break; + } + case SCR_HJ_TIMER: { + ScrHJob *j = scr_hjob(token, SCR_HJ_TIMER); + if (j == NULL || j->cleared) return; + ScrClosure *cb = (ScrClosure *)j->p; + j->firing = true; + j->refreshed = false; + ((void (*)(ScrClosure *))cb->fn)(cb); + j = &scr_hjobs[token - 1]; + j->firing = false; + if (j->cleared) { + j->cleared = false; + scr_hjob_free(token); + scr_closure_release(cb); + } else if (j->due) { + /* A due timer ran before Node's loop: arm it only to run again. */ + j->due = false; + if (j->repeat_ms > 0 || j->refreshed) { + double delay = j->repeat_ms > 0 ? j->repeat_ms : j->delay_ms; + uint32_t handle = + scr_loop_hosted->timer(delay, j->repeat_ms, j->reffed ? 1 : 0, token); + scr_hjobs[token - 1].handle = handle; + } else { + scr_hosted_timer_drop(token); + } + } else if (j->repeat_ms <= 0 && !j->refreshed) { + /* A fired one-shot: Node drops its Timeout; the bridge forgets it. */ + scr_loop_hosted->timer_clear(j->handle); + scr_hosted_timer_drop(token); + } + break; + } + case SCR_HJ_IMMEDIATE: { + ScrHJob *j = scr_hjob(token, SCR_HJ_IMMEDIATE); + if (j == NULL) return; + ScrClosure *cb = (ScrClosure *)j->p; + scr_hmap_delete(j->id * 2 + 1); + scr_hjob_free(token); + ((void (*)(ScrClosure *))cb->fn)(cb); + scr_closure_release(cb); + break; + } + default: + return; + } + scr_hosted_settle_exception(); +} + +/* V8 finished a microtask checkpoint: the runtime's unhandled rejections + * are decided here, where Node decides its own. An unhandled one ends the + * process like main's reported-rejection path. */ +void scr_loop_hosted_microtasks_done(void) { + if (scr_nunhandled == 0) return; + if (!scr_report_unhandled_rejections()) { + scr_hosted_settle_exception(); + return; + } + scr_discard_unhandled_rejections(); + scr_process_exit((double)scr_exit_code_hint_get()); +} + +/* ── the runtime's I/O on Node's loop ── */ + +static void scr_hosted_io_fd(ScrNodeIoState *st, int fd) { + if (fd < 0 || st->nfds >= SCR_NODE_IO_MAX_FDS) return; + st->fds[st->nfds++] = fd; +} + +static void scr_hosted_io_cap(ScrNodeIoState *st, double ms) { + int cap = (int)ms; + if (st->cap_ms < 0 || cap < st->cap_ms) st->cap_ms = cap; +} + +/* The loop pass's liveness and sleep decisions, as Node's loop needs them + * before each poll phase and after each check phase. */ +void scr_loop_hosted_io_state(ScrNodeIoState *st) { + st->nfds = 0; + st->alive = 0; + st->ready = 0; + st->cap_ms = -1; + /* Quiescent before the poll phase: the turn's scheduled cycle collection. */ + if (st->phase == SCR_NODE_IO_PREPARE) scr_cyc_collect_scheduled(); + bool kids = scr_loop_children != NULL && scr_loop_children->pending(); + bool kids_reffed = scr_loop_children != NULL && scr_loop_children->reffed_pending(); + bool kids_failed = scr_loop_children != NULL && scr_loop_children->failed_pending(); + bool events = scr_events_pending_fn != NULL && scr_events_pending_fn(); + bool evw = scr_events_watching_fn != NULL && scr_events_watching_fn(); + bool net = scr_net_pending_fn != NULL && scr_net_pending_fn(); + bool net_io = net || (scr_net_io_fn != NULL && scr_net_io_fn()); + bool dgram = scr_dgram_pending_fn != NULL && scr_dgram_pending_fn(); + bool watch = scr_watch_pending_fn != NULL && scr_watch_pending_fn(); + bool ffi = scr_ffi_pending_fn != NULL && scr_ffi_pending_fn(); + bool stream = scr_stream_pending_fn != NULL && scr_stream_pending_fn(); + bool renames = scr_fs_renames_pending(); + bool crypto = scr_crypto_bytes_pending(); + bool deferred = scr_loop_deferred_pending(); + bool workers = scr_workers_pending(); + /* Deferred native callbacks (the fs callbacks) become deliverable at the + * end of the turn that queued them and run in the next one, as the loop + * pass delivers them; Node's own fs requests take a turn or more too. */ + if (st->phase == SCR_NODE_IO_CHECK) scr_loop_deferred_ready = scr_loop_deferred_count; + st->alive = kids_reffed || kids_failed || events || net || dgram || watch || ffi || stream || + renames || crypto || deferred || workers; + /* Queued deferred callbacks keep the poll from sleeping (their work is + * done; only the delivery waits for the next turn). */ + st->ready = stream || crypto || deferred || kids_failed || + (scr_loop_children != NULL && scr_loop_children->ready()); +#ifdef SCR_WORKERS + if (scr_workers_ready_fn && scr_workers_ready_fn()) st->ready = 1; +#endif +#if !defined(_WIN32) && !defined(__wasi__) + if (evw && scr_events_pollfds_fn != NULL) { + int evfds[2]; + int nev = scr_events_pollfds_fn(evfds); + for (int i = 0; i < nev; i++) scr_hosted_io_fd(st, evfds[i]); + } + if (net_io) { + int fd = scr_net_pollfd_fn != NULL ? scr_net_pollfd_fn() : -1; + if (fd >= 0) scr_hosted_io_fd(st, fd); + else scr_hosted_io_cap(st, SCR_SIGNAL_POLL_MS); + } + if (dgram) { + int fd = scr_dgram_pollfd_fn != NULL ? scr_dgram_pollfd_fn() : -1; + if (fd >= 0) scr_hosted_io_fd(st, fd); + else scr_hosted_io_cap(st, SCR_SIGNAL_POLL_MS); + } + if (watch) { + int fd = scr_watch_pollfd_fn != NULL ? scr_watch_pollfd_fn() : -1; + if (fd >= 0) scr_hosted_io_fd(st, fd); + else scr_hosted_io_cap(st, SCR_SIGNAL_POLL_MS); + } + if (ffi) { + int fd = scr_ffi_pollfd_fn != NULL ? scr_ffi_pollfd_fn() : -1; + if (fd >= 0) scr_hosted_io_fd(st, fd); + else scr_hosted_io_cap(st, SCR_CHILD_POLL_MS); + } + if (kids) { + int fd = scr_loop_children->wake_fd(); + if (fd >= 0) scr_hosted_io_fd(st, fd); + else scr_hosted_io_cap(st, SCR_CHILD_POLL_MS); + } + if (renames) { + int fd = scr_fs_renames_pollfd(); + if (fd >= 0) scr_hosted_io_fd(st, fd); + else scr_hosted_io_cap(st, SCR_CHILD_POLL_MS); + } + if (workers) { + int fd = scr_workers_pollfd(); + if (fd >= 0) scr_hosted_io_fd(st, fd); + else scr_hosted_io_cap(st, SCR_CHILD_POLL_MS); + } +#else + (void)evw; + if (evw || net_io || dgram || watch || ffi || kids || renames || workers) + scr_hosted_io_cap(st, SCR_CHILD_POLL_MS); +#endif +} + +/* The end of one runtime macrotask inside a dispatch station: the throw + * it left, then Node's checkpoint (ticks and microtasks of both sides). */ +static void scr_hosted_macrotask_done(void) { + scr_hosted_settle_exception(); + scr_loop_hosted->checkpoint(0); +} + +/* One pass over the dispatch stations, in the loop pass's order: worker + * completions and deferred callbacks one per checkpoint, then the event, + * socket, watch and child stations. */ +void scr_loop_hosted_io(void) { + /* Once per loop iteration (the bridge calls this at most once per turn): + * the poll phase is open again, as after the pass's check phase. */ + scr_loop_io_open = true; + scr_loop_io_batches = 0; + while (scr_fs_renames_pending() && scr_fs_renames_dispatch()) scr_hosted_macrotask_done(); + while (scr_crypto_bytes_pending() && scr_crypto_bytes_dispatch()) scr_hosted_macrotask_done(); + while (scr_loop_deferred_dispatch()) scr_hosted_macrotask_done(); + if (scr_ffi_dispatch_fn != NULL && scr_ffi_dispatch_fn()) scr_hosted_macrotask_done(); + if (scr_stream_dispatch_fn != NULL) { + scr_stream_dispatch_fn(); + scr_hosted_macrotask_done(); + } + scr_workers_dispatch(); + scr_hosted_macrotask_done(); + if (scr_events_dispatch_fn != NULL) { + scr_events_dispatch_fn(); + scr_hosted_macrotask_done(); + } + if (scr_net_dispatch_fn != NULL) { + scr_net_dispatch_fn(); + scr_hosted_macrotask_done(); + } + if (scr_dgram_dispatch_fn != NULL) { + scr_dgram_dispatch_fn(); + scr_hosted_macrotask_done(); + } + if (scr_watch_dispatch_fn != NULL) { + scr_watch_dispatch_fn(); + scr_hosted_macrotask_done(); + } + if (scr_loop_children != NULL && scr_loop_children->pending()) { +#if !defined(_WIN32) && !defined(__wasi__) + (void)scr_loop_children->wait(0); +#endif + scr_loop_children->poll(); + scr_hosted_macrotask_done(); + } +} + +/* ── the switch ── */ + +void scr_loop_host(const ScrNodeBridge *bridge, double since_ms) { + if (scr_loop_hosted != NULL) return; + scr_loop_hosted = bridge; + scr_hosted_continue_turn = scr_pass_phase == SCR_PASS_IO || scr_pass_phase == SCR_PASS_TIMERS; + scr_hosted_microtasks_first = scr_in_microtask; + bridge->host_loop(); + /* Promise jobs, FIFO. */ + while (scr_ready_len > 0) { + ScrFiber *f = scr_ready[scr_ready_head++]; + scr_ready_len--; + scr_hosted_microtask(f); + } + scr_ready_head = 0; + /* Ticks, FIFO. */ + ScrNtick *t = scr_nt_head; + scr_nt_head = scr_nt_tail = NULL; + while (t != NULL) { + ScrNtick *next = t->next; + t->next = NULL; + scr_hosted_tick(t); + t = next; + } + /* Immediates in queue order, keeping their handles. Those of a check + * phase the switch interrupted finish it before Node's loop starts. */ + for (size_t i = scr_immediates_head; i < scr_nimmediates; i++) { + ScrImmediate *im = &scr_immediates[i]; + if (im->cb == NULL) continue; + ScrClosure *cb = im->cb; + bool reffed = im->reffed; + unsigned long id = im->id; + im->cb = NULL; + uint32_t token = scr_hjob_new(SCR_HJ_IMMEDIATE, cb); + scr_hjobs[token - 1].id = id; + scr_hjobs[token - 1].reffed = reffed; + scr_hmap_put(id * 2 + 1, token); + if (i < scr_check_end) { + scr_hosted_due_push(token); + continue; + } + uint32_t handle = bridge->immediate(reffed ? 1 : 0, token); + scr_hjobs[token - 1].handle = handle; + } + scr_check_end = 0; + /* Timers in (deadline, sequence) order; those already due when Node + * began loading run before Node's loop starts, as the interrupted pass + * would have run them. */ + while (scr_ntimers > 0) { + ScrTimer tm = scr_timer_pop(); + scr_hosted_timer_arm(tm.cb, scr_hosted_delay(tm.deadline_ms - since_ms), tm.repeat_ms, + tm.delay_ms, tm.id, tm.reffed, tm.deadline_ms <= since_ms); + } + scr_reffed_timers = 0; + scr_nimmediates = scr_immediates_head = 0; + scr_pending_immediates = scr_reffed_immediates = 0; +} + +bool scr_loop_is_hosted(void) { return scr_loop_hosted != NULL; } + +static void scr_loop_run_hosted(bool first_checkpoint) { + scr_loop_hosted->checkpoint(first_checkpoint || scr_hosted_microtasks_first ? 1 : 0); + /* The interrupted phase's work, each followed by Node's checkpoint. A + * due entry cleared meanwhile is gone (or its token reused, not due). */ + for (size_t i = 0; i < scr_hosted_due_len; i++) { + uint32_t token = scr_hosted_due[i]; + if (token == 0 || token > scr_hjobs_len || !scr_hjobs[token - 1].due) continue; + uint8_t kind = scr_hjobs[token - 1].kind; + if (kind == SCR_HJ_IMMEDIATE) scr_hjobs[token - 1].due = false; + scr_loop_hosted_run(kind, token); + scr_loop_hosted->checkpoint(0); + } + free(scr_hosted_due); + scr_hosted_due = NULL; + scr_hosted_due_len = scr_hosted_due_cap = 0; + /* process.exitCode is one property in Node; island code's assignment + * becomes the program's code when the loop ends. */ + int code = scr_loop_hosted->run_loop(scr_hosted_continue_turn ? 1 : 0); + if (code != 0) scr_process_set_exit_code_tracked((double)code); +} +#endif /* SCR_NODE_ISLANDS */ + /* The process.nextTick queue to exhaustion, ticks queued meanwhile * included. false: a tick threw (uncaught, pending). */ static bool scr_loop_run_ticks(void) { @@ -4532,7 +5335,7 @@ static bool scr_loop_run_checkpoint(void) { while (scr_ready_len > 0) { ScrFiber *f = scr_ready[scr_ready_head++]; scr_ready_len--; - scr_resume_fiber(f); + SCR_RESUME_JOB(f); } } return true; @@ -4547,6 +5350,10 @@ static bool scr_loop_run_pass(ScrPromise *top_level, bool first_checkpoint) { * pinned. Every later checkpoint drains ticks first. */ bool rejection_failed = false; for (;;) { +#ifdef SCR_NODE_ISLANDS + /* Node loaded during this turn: its loop takes over (scr_loop_run). */ + if (scr_loop_hosted != NULL) return rejection_failed; +#endif if (top_level != NULL && top_level->state == SCR_PROM_REJECTED) break; /* process.nextTick callbacks BEFORE promise jobs (Node's checkpoint * order): the tick queue to exhaustion, then the microtask queue to @@ -4559,7 +5366,7 @@ static bool scr_loop_run_pass(ScrPromise *top_level, bool first_checkpoint) { while (scr_ready_len > 0) { ScrFiber *f = scr_ready[scr_ready_head++]; scr_ready_len--; - scr_resume_fiber(f); + SCR_RESUME_JOB(f); } first_checkpoint = false; /* The ESM loader observes a rejected entry evaluation at a promise-job @@ -4593,6 +5400,7 @@ static bool scr_loop_run_pass(ScrPromise *top_level, bool first_checkpoint) { * occupied that thread. Deliver exactly one completion per checkpoint: * Node drains nextTicks and microtasks after each native callback before * invoking the next ready callback. */ + SCR_PASS_PHASE(SCR_PASS_IO); if (scr_fs_renames_pending()) { bool dispatched = scr_fs_renames_dispatch(); if (scr_exc_pending()) return false; @@ -4666,16 +5474,6 @@ static bool scr_loop_run_pass(ScrPromise *top_level, bool first_checkpoint) { if (scr_exc_pending()) return false; /* uncaught throw in a listener */ if (scr_ready_len > 0 || scr_nt_head != NULL) continue; } -#ifdef SCR_NODE_ISLANDS - /* Node island dispatch (scr_node_island.c): one non-blocking turn of - * Node's loop, whose callbacks may settle calls into Node — the watch - * hook's station. */ - if (scr_node_island_dispatch_fn != NULL) { - scr_node_island_dispatch_fn(); - if (scr_exc_pending()) return false; - if (scr_ready_len > 0 || scr_nt_head != NULL) continue; - } -#endif /* Reap spawned children and fire their listeners (before timers, * like Node's nextTick-ish spawn-failure "error"). A callback may * enqueue microtasks or spawn again — restart the turn if so. @@ -4701,9 +5499,6 @@ static bool scr_loop_run_pass(ScrPromise *top_level, bool first_checkpoint) { (scr_ffi_pending_fn != NULL && scr_ffi_pending_fn()) || scr_fs_renames_pending() || scr_crypto_bytes_pending() || scr_loop_deferred_pending() || scr_workers_pending() -#ifdef SCR_NODE_ISLANDS - || (scr_node_island_pending_fn != NULL && scr_node_island_pending_fn()) -#endif ; if (held) { scr_loop_children->poll(); @@ -4717,6 +5512,10 @@ static bool scr_loop_run_pass(ScrPromise *top_level, bool first_checkpoint) { * nextTicks enqueued by a station listener are the same story. */ if (scr_stream_pending_fn != NULL && scr_stream_pending_fn()) continue; if (scr_nt_head != NULL) continue; +#ifdef SCR_NODE_ISLANDS + /* Node loaded during this turn: its loop takes over (scr_loop_run). */ + if (scr_loop_hosted != NULL) return rejection_failed; +#endif bool kids = scr_loop_children != NULL && scr_loop_children->pending(); bool kids_reffed = scr_loop_children != NULL && scr_loop_children->reffed_pending(); bool events = scr_events_pending_fn != NULL && scr_events_pending_fn(); @@ -4730,20 +5529,13 @@ static bool scr_loop_run_pass(ScrPromise *top_level, bool first_checkpoint) { bool renames = scr_fs_renames_pending(); bool crypto = scr_crypto_bytes_pending() || scr_loop_deferred_pending(); bool workers = scr_workers_pending(); -#ifdef SCR_NODE_ISLANDS - bool node_island = scr_node_island_pending_fn != NULL && scr_node_island_pending_fn(); -#endif /* Timer liveness counts only REF'd timers: an unref'd timer stays in * the heap (and fires if the loop runs on for other reasons) but does * not by itself keep the process alive — Node's unref semantics. * Children follow the same rule: an unref'd child is still REAPED * while the loop runs (kids drives the sweeps and sleeps above) but * only reffed ones keep the process alive. */ - if (scr_reffed_timers == 0 && scr_reffed_immediates == 0 && !kids_reffed && !events && !net && !dgram && !watch && !ffi && !renames && !crypto && !workers -#ifdef SCR_NODE_ISLANDS - && !node_island -#endif - ) break; + if (scr_reffed_timers == 0 && scr_reffed_immediates == 0 && !kids_reffed && !events && !net && !dgram && !watch && !ffi && !renames && !crypto && !workers) break; /* Sleep to the earliest deadline, then run every due timer (each may * enqueue microtasks, which the next iteration drains first). Who * sleeps depends on what is pending: @@ -4801,20 +5593,12 @@ static bool scr_loop_run_pass(ScrPromise *top_level, bool first_checkpoint) { #endif bool evw = scr_events_watching_fn != NULL && scr_events_watching_fn(); #ifdef SCR_WORKERS - if (workers && scr_workers_wait_fn && !kids && !evw && !net_io && !dgram && !watch && !ffi && !renames && !crypto -#ifdef SCR_NODE_ISLANDS - && !node_island -#endif - ) { + if (workers && scr_workers_wait_fn && !kids && !evw && !net_io && !dgram && !watch && !ffi && !renames && !crypto) { scr_workers_wait_fn(due > now ? due - now : 0); now = scr_now_ms(); } else #endif - if (evw || net_io || dgram || watch || ffi || workers || renames_fd >= 0 -#ifdef SCR_NODE_ISLANDS - || node_island -#endif - ) { + if (evw || net_io || dgram || watch || ffi || workers || renames_fd >= 0) { #if defined(_WIN32) || defined(__wasi__) /* The win32 arm, and WASI hosts whose poll_oneoff adapters do not * reliably wake for a closed inherited stdin pipe: the sleep is a capped nanosleep and @@ -4830,9 +5614,6 @@ static bool scr_loop_run_pass(ScrPromise *top_level, bool first_checkpoint) { if (evw && due > now + SCR_SIGNAL_POLL_MS) due = now + SCR_SIGNAL_POLL_MS; if ((net_io || dgram || watch) && due > now + SCR_CHILD_POLL_MS) due = now + SCR_CHILD_POLL_MS; if ((ffi || workers) && due > now + SCR_CHILD_POLL_MS) due = now + SCR_CHILD_POLL_MS; -#ifdef SCR_NODE_ISLANDS - if (node_island && due > now + SCR_CHILD_POLL_MS) due = now + SCR_CHILD_POLL_MS; -#endif if (kids && due > now + SCR_CHILD_POLL_MS) due = now + SCR_CHILD_POLL_MS; if (due > now) { double wait = due - now; @@ -4926,22 +5707,6 @@ static bool scr_loop_run_pass(ScrPromise *top_level, bool first_checkpoint) { fds[nfds].events = POLLIN; fds[nfds++].revents = 0; } -#ifdef SCR_NODE_ISLANDS - if (node_island) { - /* Node's backend fd ends the sleep on its I/O; its next timer - * bounds the sleep. */ - int left = scr_node_island_timeout_fn != NULL ? scr_node_island_timeout_fn() : -1; - if (left >= 0 && due > now + left) due = now + left; - int ifd = scr_node_island_pollfd_fn != NULL ? scr_node_island_pollfd_fn() : -1; - if (ifd >= 0) { - fds[nfds].fd = ifd; - fds[nfds].events = POLLIN; - fds[nfds++].revents = 0; - } else if (due > now + SCR_CHILD_POLL_MS) { - due = now + SCR_CHILD_POLL_MS; - } - } -#endif if (workers) { int worker_fd = scr_workers_pollfd(); if (worker_fd >= 0) { @@ -4982,6 +5747,7 @@ static bool scr_loop_run_pass(ScrPromise *top_level, bool first_checkpoint) { #ifdef SCR_WORKERS scr_sa_loop_waiting(false); #endif + if (scr_ntimers > 0 && scr_timers[0].deadline_ms <= now) SCR_PASS_PHASE(SCR_PASS_TIMERS); while (scr_ntimers > 0 && scr_timers[0].deadline_ms <= now) { ScrTimer t = scr_timer_pop(); /* Timer callbacks are plain sync closures, run on the main stack. @@ -5015,6 +5781,10 @@ static bool scr_loop_run_pass(ScrPromise *top_level, bool first_checkpoint) { } scr_firing_id = 0; if (scr_exc_pending()) return false; /* uncaught throw in a callback: main handles it */ +#ifdef SCR_NODE_ISLANDS + /* Node loaded during this turn: its loop takes over (scr_loop_run). */ + if (scr_loop_hosted != NULL) return rejection_failed; +#endif /* Drain ticks and microtasks, and report the rejections the callback * left, before more timers (Node's runNextTicks between timers). */ if (scr_ready_len > 0 || scr_nt_head != NULL || scr_nunhandled > 0) break; @@ -5030,6 +5800,10 @@ static bool scr_loop_run_pass(ScrPromise *top_level, bool first_checkpoint) { bool check_ran = false; if (scr_ready_len == 0 && scr_nt_head == NULL && scr_nunhandled == 0 && scr_pending_immediates > 0) { size_t end = scr_nimmediates; +#ifdef SCR_NODE_ISLANDS + scr_check_end = end; /* a switch inside this phase finishes it first */ + SCR_PASS_PHASE(SCR_PASS_CHECK); +#endif while (scr_immediates_head < end) { size_t i = scr_immediates_head++; ScrClosure *cb = scr_immediates[i].cb; @@ -5043,6 +5817,9 @@ static bool scr_loop_run_pass(ScrPromise *top_level, bool first_checkpoint) { ((void (*)(ScrClosure *))cb->fn)(cb); scr_closure_release(cb); if (scr_exc_pending()) return false; /* uncaught throw: main handles it */ +#ifdef SCR_NODE_ISLANDS + if (scr_loop_hosted != NULL) return rejection_failed; +#endif if (!scr_loop_run_checkpoint()) return false; /* Node's runNextTicks between immediates also reports the * rejections the callback left before the next one runs; a @@ -5053,6 +5830,9 @@ static bool scr_loop_run_pass(ScrPromise *top_level, bool first_checkpoint) { } if (top_level != NULL && top_level->state == SCR_PROM_REJECTED) break; } +#ifdef SCR_NODE_ISLANDS + scr_check_end = 0; +#endif /* Fully drained: reset the cursor so the array reuses its slots. */ if (scr_immediates_head == scr_nimmediates) { scr_immediates_head = 0; @@ -5088,12 +5868,26 @@ bool scr_loop_run(ScrPromise *top_level) { bool first_checkpoint = !scr_loop_cjs_entry; bool rejection_failed; for (;;) { +#ifdef SCR_NODE_ISLANDS + /* Node is loaded (in the main body or a callback of an earlier pass): + * its loop runs everything from here, the runtime's queues included, + * and ends the process itself on an uncaught exception or an + * unhandled rejection. */ + if (scr_loop_hosted != NULL) { + scr_loop_run_hosted(first_checkpoint); + rejection_failed = false; + break; + } +#endif rejection_failed = scr_loop_run_pass(top_level && !top_level->top_level_handled ? top_level : NULL, first_checkpoint); first_checkpoint = false; if (scr_exc_pending()) { if (scr_exc_handle_uncaught(false)) continue; break; /* main reports the original or handler-thrown exception */ } +#ifdef SCR_NODE_ISLANDS + if (scr_loop_hosted != NULL) continue; +#endif if (rejection_failed) break; if (top_level && top_level->state == SCR_PROM_REJECTED && !top_level->top_level_handled) { scr_promise_rethrow(top_level); diff --git a/packages/runtime/src/scr_dyn.h b/packages/runtime/src/scr_dyn.h index 6ac123e58..fafa16eda 100644 --- a/packages/runtime/src/scr_dyn.h +++ b/packages/runtime/src/scr_dyn.h @@ -240,6 +240,7 @@ typedef enum { /* Tags added since the last pin, by name only (see above). */ #define SCR_DYNH_NEW_TAGS(Y) \ + Y(NODE_ISLAND) /* a Node island value (scr_node_island_handle.c) */ \ Y(WS_BYTE_STRATEGY) /* ByteLengthQueuingStrategy (scr_webstreams.c) */ \ Y(WS_COUNT_STRATEGY) /* CountQueuingStrategy (scr_webstreams.c) */ \ Y(WS_RCONTROLLER) /* ReadableStreamDefaultController (scr_webstreams.c) */ \ diff --git a/packages/runtime/src/scr_dyn_builtin_class.c b/packages/runtime/src/scr_dyn_builtin_class.c index f7bec0938..20902e283 100644 --- a/packages/runtime/src/scr_dyn_builtin_class.c +++ b/packages/runtime/src/scr_dyn_builtin_class.c @@ -1587,6 +1587,9 @@ bool scr_dyn_builtin_instanceof(ScrDyn *value, double kind) { /* `value instanceof callee` for an untyped constructor function. */ bool scr_dyn_function_instanceof(ScrDyn *value, ScrDyn *callee) { +#ifdef SCR_NODE_ISLANDS + if (scr_ni_is_handle(callee)) return scr_ni_instanceof(value, callee) == 1; +#endif if (!scr_dyn_is_callable(callee)) { static const char message[] = "Right-hand side of 'instanceof' is not callable"; scr_throw_error_msg(SCR_ERR_TYPE, message, sizeof message - 1); diff --git a/packages/runtime/src/scr_events.c b/packages/runtime/src/scr_events.c index 6e0bf5146..78b7bb0f1 100644 --- a/packages/runtime/src/scr_events.c +++ b/packages/runtime/src/scr_events.c @@ -10,6 +10,9 @@ * zero bytes and keep their exact historical link line. */ #define _XOPEN_SOURCE 700 #include "scr_runtime.h" +#ifdef SCR_NODE_ISLANDS +#include "scr_node_island.h" +#endif #include #include @@ -80,6 +83,11 @@ typedef struct { ScrStr *name; /* aliases of one signum remain distinct event names */ bool once; uint64_t id; +#ifdef SCR_NODE_ISLANDS + /* Nonzero: a listener island code added (its cb is undefined); the + * delivery calls it through the bridge at this position. */ + uint32_t island; +#endif } ScrSigListener; typedef struct ScrSigReg { @@ -280,6 +288,63 @@ void scr_signal_off(ScrStr *name, ScrDyn *cb) { } } +#ifdef SCR_NODE_ISLANDS +/* Island code's signal listeners (scr_node_island.c forwards Node's + * newListener/removeListener for signal events here): each takes its place + * in the registry like a native listener, so scriptc owns every signal + * disposition and libuv never installs one. Removing the last listener of + * either side restores the previous disposition, as for native ones. */ +void scr_signal_on_island(ScrStr *name, uint32_t island) { +#if defined(_WIN32) || defined(__wasi__) + (void)name; + (void)island; +#else + if (!scr_context_is_main()) return; + int sig = scr_signal_from_name(name); + if (sig < 0) sig = 0; + ScrSigReg *reg = scr_sig_regs; + while (reg && reg->sig != sig) reg = reg->next; + if (!reg) { + reg = calloc(1, sizeof *reg); + if (!reg) scr_events_oom(); + reg->sig = sig; + if (sig > 0) { + struct sigaction sa; + memset(&sa, 0, sizeof sa); + sa.sa_handler = scr_sig_handler; + sigemptyset(&sa.sa_mask); + if (sig >= SCR_SIG_MAX || sigaction(sig, &sa, ®->prev) != 0) { + free(reg); + return; + } + scr_sig_watched++; + scr_wake_pipe_init(); + } + reg->next = scr_sig_regs; + scr_sig_regs = reg; + } + if (reg->n == reg->cap) { + reg->cap = reg->cap ? reg->cap * 2 : 2; + reg->ls = realloc(reg->ls, reg->cap * sizeof *reg->ls); + if (!reg->ls) scr_events_oom(); + } + reg->ls[reg->n++] = + (ScrSigListener){scr_dyn_retain(scr_dyn_undefined()), scr_str_retain(name), false, ++scr_sig_id, island}; +#endif +} + +void scr_signal_off_island(uint32_t island) { + for (ScrSigReg *reg = scr_sig_regs; reg; reg = reg->next) { + for (size_t i = 0; i < reg->n; i++) { + if (reg->ls[i].island == island) { + scr_sig_remove(reg, i); + return; + } + } + } +} +#endif + /* Snapshot all pending deliveries before invoking user code. No live * registry pointer survives a callback: off() can remove this signal or * another pending signal and free either registry. */ @@ -315,11 +380,19 @@ static void scr_signals_drain(void) { if (found) break; } } - ScrDyn *args[2] = {scr_dyn_new_str(snap[i].name), scr_dyn_new_num(scr_signal_from_name(snap[i].name))}; - ScrDyn *result = scr_dyn_call(snap[i].cb, args, 2, "listener"); - scr_dyn_release(result); - scr_dyn_release(args[0]); - scr_dyn_release(args[1]); +#ifdef SCR_NODE_ISLANDS + if (snap[i].island != 0) { + scr_node_island_emit(SCR_NODE_ISLAND_EVENT_SIGNAL, snap[i].island, snap[i].name, + scr_signal_from_name(snap[i].name)); + } else +#endif + { + ScrDyn *args[2] = {scr_dyn_new_str(snap[i].name), scr_dyn_new_num(scr_signal_from_name(snap[i].name))}; + ScrDyn *result = scr_dyn_call(snap[i].cb, args, 2, "listener"); + scr_dyn_release(result); + scr_dyn_release(args[0]); + scr_dyn_release(args[1]); + } } scr_dyn_release(snap[i].cb); scr_str_release(snap[i].name); @@ -1541,6 +1614,9 @@ typedef struct { ScrClosure *cb; /* owned */ void (*fn)(ScrClosure *cb, double code); /* adapter: 0-param or (code) */ bool once; +#ifdef SCR_NODE_ISLANDS + uint32_t island; /* nonzero: island code's listener (cb and fn NULL) */ +#endif } ScrExitListener; static SCR_TL ScrExitListener *scr_exit_ls = NULL; @@ -1554,6 +1630,13 @@ void scr_run_exit_listeners(double code) { for (size_t i = 0; i < scr_exit_n; i++) { #ifdef SCR_WORKERS if (scr_context_stopping()) break; +#endif +#ifdef SCR_NODE_ISLANDS + if (scr_exit_ls[i].island != 0) { + ScrStr *exit_name = scr_str_new("exit", 4); + scr_node_island_emit(SCR_NODE_ISLAND_EVENT_EXIT, scr_exit_ls[i].island, exit_name, (int)code); + scr_str_release(exit_name); + } else #endif scr_exit_ls[i].fn(scr_exit_ls[i].cb, code); #ifdef SCR_WORKERS @@ -1571,7 +1654,12 @@ void scr_run_exit_listeners(double code) { _Exit(7); } } - for (size_t i = 0; i < scr_exit_n; i++) scr_closure_release(scr_exit_ls[i].cb); + for (size_t i = 0; i < scr_exit_n; i++) { +#ifdef SCR_NODE_ISLANDS + if (scr_exit_ls[i].cb == NULL) continue; +#endif + scr_closure_release(scr_exit_ls[i].cb); + } free(scr_exit_ls); scr_exit_ls = NULL; scr_exit_n = scr_exit_cap = 0; @@ -1595,11 +1683,43 @@ void scr_process_on_exit(ScrClosure *cb /*moves*/, void (*fn)(ScrClosure *, doub scr_exit_ls[scr_exit_n].cb = cb; scr_exit_ls[scr_exit_n].fn = fn; scr_exit_ls[scr_exit_n].once = once; +#ifdef SCR_NODE_ISLANDS + scr_exit_ls[scr_exit_n].island = 0; +#endif scr_exit_n++; } +#ifdef SCR_NODE_ISLANDS +/* Island code's 'exit' listeners, at their registration position among + * the program's own. */ +void scr_process_on_exit_island(uint32_t island) { + if (scr_exit_ran) return; + if (scr_exit_n == scr_exit_cap) { + scr_exit_cap = scr_exit_cap ? scr_exit_cap * 2 : 2; + scr_exit_ls = realloc(scr_exit_ls, scr_exit_cap * sizeof *scr_exit_ls); + if (!scr_exit_ls) scr_events_oom(); + } + scr_exit_ls[scr_exit_n] = (ScrExitListener){NULL, NULL, false, island}; + scr_exit_n++; +} + +void scr_process_off_exit_island(uint32_t island) { + if (scr_exit_ran) return; + for (size_t i = 0; i < scr_exit_n; i++) { + if (scr_exit_ls[i].island == island) { + memmove(scr_exit_ls + i, scr_exit_ls + i + 1, (scr_exit_n - i - 1) * sizeof *scr_exit_ls); + scr_exit_n--; + return; + } + } +} +#endif + void scr_process_off_exit(ScrClosure *cb /*borrowed*/) { for (size_t i = 0; i < scr_exit_n; i++) { +#ifdef SCR_NODE_ISLANDS + if (scr_exit_ls[i].cb == NULL) continue; +#endif if (scr_closure_identity_equal(scr_exit_ls[i].cb, cb)) { scr_closure_release(scr_exit_ls[i].cb); memmove(scr_exit_ls + i, scr_exit_ls + i + 1, (scr_exit_n - i - 1) * sizeof *scr_exit_ls); diff --git a/packages/runtime/src/scr_inspect.c b/packages/runtime/src/scr_inspect.c index e6df0f4cf..a015c25fe 100644 --- a/packages/runtime/src/scr_inspect.c +++ b/packages/runtime/src/scr_inspect.c @@ -2653,6 +2653,10 @@ ScrStr *scr_insp_dyn(ScrDyn *d, double recurse, double depth) { ScrStr *iterator = scr_insp_native_iterator(d, recurse, depth); if (iterator) return iterator; } +#ifdef SCR_NODE_ISLANDS + /* An island object prints as Node's util.inspect prints it. */ + if (d->v.handle.tag == SCR_DYNH_NODE_ISLAND) return scr_ni_inspect(d, recurse, depth); +#endif const ScrDynHandleOps *ops = scr_dyn_handle_ops_of(d); if (ops->inspect_view) { /* Node's custom inspectors answer the bare class name beyond the diff --git a/packages/runtime/src/scr_json.c b/packages/runtime/src/scr_json.c index 7de149143..1177231ee 100644 --- a/packages/runtime/src/scr_json.c +++ b/packages/runtime/src/scr_json.c @@ -5604,6 +5604,9 @@ static bool scr_dyn_instanceof_function(const ScrDyn *value) { } bool scr_bytes_instanceof_at(const ScrDyn *value, const ScrDyn *callee, const ScrStr *refusal) { +#ifdef SCR_NODE_ISLANDS + if (scr_ni_is_handle(callee)) return scr_ni_instanceof((ScrDyn *)value, (ScrDyn *)callee) == 1; +#endif if (!callee || callee->kind != SCR_DYN_FUNC) { static const char message[] = "Right-hand side of 'instanceof' is not callable"; scr_throw_error_msg(SCR_ERR_TYPE, message, sizeof message - 1); @@ -6459,6 +6462,15 @@ ScrDyn *scr_dyn_class_prototype(ScrDyn *value, ScrDyn *base) { * compiled per signature). Args borrowed; result owned (+1) or NULL with * the exception pending. */ ScrDyn *scr_dyn_call(const ScrDyn *d, ScrDyn *const *args, size_t argc, const char *what) { +#ifdef SCR_NODE_ISLANDS + /* An island function: called in Node with the ambient receiver. */ + if (scr_ni_is_handle(d) && scr_ni_is_function(d)) { + ScrDyn *receiver = scr_dyn_this_get(); + ScrDyn *result = scr_ni_call((ScrDyn *)d, receiver, args, argc); + scr_dyn_release(receiver); + return result; + } +#endif if (d->kind == SCR_DYN_PROXY && scr_dyn_is_callable(d->v.proxy.target)) { ScrDyn *trap = scr_dyn_proxy_trap(d, "apply"); if (scr_exc_pending()) return NULL; @@ -7590,6 +7602,10 @@ void scr_dyn_name_class_prototype(ScrDyn *prototype, const ScrStr *name) { /* Ordinary native function constructors retain their live prototype table. * A constructor may replace the fresh receiver with another object. */ ScrDyn *scr_dyn_construct(const ScrDyn *callee, const ScrDyn *args, const ScrStr *what) { +#ifdef SCR_NODE_ISLANDS + if (scr_ni_is_handle(callee)) + return scr_ni_construct((ScrDyn *)callee, args->v.arr.items, args->v.arr.len); +#endif if (callee->kind != SCR_DYN_FUNC || callee->v.fn.class_obj || (scr_closure_identity(callee->v.fn.clo)->function_kind & SCR_CLOSURE_KIND_MASK) != SCR_CLOSURE_KIND_OWN_PROTOTYPE) return scr_bytes_construct(callee, args, what); @@ -9685,6 +9701,9 @@ bool scr_dyn_truthy(const ScrDyn *d) { * null answers "object" — JS's oldest wart, preserved. */ ScrStr *scr_dyn_typeof(const ScrDyn *d) { const char *s; +#ifdef SCR_NODE_ISLANDS + if (scr_ni_is_handle(d)) return scr_ni_typeof((ScrDyn *)d); +#endif if (d->kind == SCR_DYN_PROXY) return scr_dyn_typeof(d->v.proxy.target); if (d->kind == SCR_DYN_TYPED_REF) { if ((d->v.typed_ref.type_key_len >= 7 && @@ -9721,6 +9740,9 @@ ScrStr *scr_dyn_typeof(const ScrDyn *d) { bool scr_dyn_is_callable(const ScrDyn *value) { while (value->kind == SCR_DYN_PROXY) value = value->v.proxy.target; +#ifdef SCR_NODE_ISLANDS + if (scr_ni_is_handle(value)) return scr_ni_is_function(value); +#endif return value->kind == SCR_DYN_FUNC || scr_dyn_ref_typeof_is(value, "function"); } @@ -11828,6 +11850,14 @@ bool scr_dyn_has_key(const ScrDyn *v, const ScrStr *key) { * Array.prototype before it for arrays) and, for a class instance seen as * `unknown`, its class's compiled methods and accessors. */ bool scr_dyn_in(const ScrDyn *v, const ScrStr *key) { +#ifdef SCR_NODE_ISLANDS + if (scr_ni_is_handle(v)) { + ScrDyn *k = scr_dyn_new_str((ScrStr *)key); + int found = scr_ni_has((ScrDyn *)v, k); + scr_dyn_release(k); + return found == 1; + } +#endif /* Generator objects have no checked view; scr_dyn_has_key answers their * prototype members. */ if (v->kind == SCR_DYN_TYPED_REF && !scr_dyn_generator(v)) { @@ -14205,6 +14235,14 @@ void scr_dyn_obj_delete_field(ScrDyn *obj, const ScrStr *key) { * arrays, typed references and handles keep their explicit boundary; * deleting from a materialized snapshot would lose the mutation. */ void scr_dyn_key_delete(ScrDyn *recv, const ScrStr *key, bool strict) { +#ifdef SCR_NODE_ISLANDS + if (scr_ni_is_handle(recv)) { + ScrDyn *k = scr_dyn_new_str((ScrStr *)key); + (void)scr_ni_delete(recv, k); + scr_dyn_release(k); + return; + } +#endif if (scr_dyn_class_reflection_fence(recv) || scr_dyn_error_prototype_fence(recv)) return; if (recv->kind == SCR_DYN_PROXY) { scr_dyn_proxy_delete(recv, key); return; } if (recv->kind == SCR_DYN_TYPED_REF && recv->v.typed_ref.commit && @@ -16246,6 +16284,34 @@ static ScrDyn *scr_dyn_obj_data_walk(const ScrDyn *v, ScrObjWalk mode) { static ScrDyn *scr_dyn_objwalk(const ScrDyn *v, ScrObjWalk mode) { if (scr_dyn_class_reflection_fence(v)) return NULL; +#ifdef SCR_NODE_ISLANDS + /* Object.keys/values/entries of an island object: its own enumerable + * string keys, read in Node. */ + if (scr_ni_is_handle(v)) { + ScrDyn *keys = scr_ni_own_keys((ScrDyn *)v, 0); + if (keys == NULL || mode == SCR_OBJWALK_KEYS) return keys; + ScrDyn *out = scr_dyn_new_arr(); + for (size_t i = 0; i < keys->v.arr.len; i++) { + ScrDyn *key = keys->v.arr.items[i]; + ScrDyn *value = scr_ni_get((ScrDyn *)v, key); + if (value == NULL) { + scr_dyn_release(out); + scr_dyn_release(keys); + return NULL; + } + if (mode == SCR_OBJWALK_VALUES) { + scr_dyn_arr_push(out, value); + } else { + ScrDyn *pair = scr_dyn_new_arr(); + scr_dyn_arr_push(pair, scr_dyn_retain(key)); + scr_dyn_arr_push(pair, value); + scr_dyn_arr_push(out, pair); + } + } + scr_dyn_release(keys); + return out; + } +#endif if (v->kind == SCR_DYN_PROXY) { ScrDyn *trap = scr_dyn_proxy_trap(v, "ownKeys"); if (scr_exc_pending()) return NULL; diff --git a/packages/runtime/src/scr_node_bridge.h b/packages/runtime/src/scr_node_bridge.h index f88764691..ff8bdae2a 100644 --- a/packages/runtime/src/scr_node_bridge.h +++ b/packages/runtime/src/scr_node_bridge.h @@ -9,17 +9,128 @@ * are "R" (returned; "R" alone is undefined) or "T" (threw; * the JSON is {"error":{kind,message,code?}}, {"value":...} or * {"undefined":true}, see scr_node_island_host.cjs). Every entry point - * runs on the thread that called start. */ + * runs on the thread that called start, and every host function is called + * on it. */ #ifndef SCR_NODE_BRIDGE_H #define SCR_NODE_BRIDGE_H #include #include -#define SCR_NODE_BRIDGE_ABI 3u +#define SCR_NODE_BRIDGE_ABI 5u /* The symbol a bridge library exports: `const ScrNodeBridge *(void)`. */ #define SCR_NODE_BRIDGE_ENTRY "scr_node_bridge_v1" +/* Kinds of runtime jobs the bridge schedules on Node's queues (`run`). */ +enum { + SCR_NODE_JOB_MICROTASK = 1, /* V8's microtask queue */ + SCR_NODE_JOB_TICK = 2, /* Node's process.nextTick queue */ + SCR_NODE_JOB_TIMER = 3, /* a Node timer (setTimeout / setInterval) */ + SCR_NODE_JOB_IMMEDIATE = 4, /* Node's check phase (setImmediate) */ +}; + +/* Process events that island code listens to (`listen`): the listener + * joins the runtime's own registry at its registration position, so one + * registry orders the listeners of both sides and libuv never owns a + * signal. */ +enum { + SCR_NODE_EVENT_SIGNAL = 1, + SCR_NODE_EVENT_EXIT = 2, +}; + +/* The runtime's I/O as Node's loop sees it (io_state): the fds to watch + * for readability, whether runtime work keeps the loop alive, whether + * runtime work is ready without any fd (the loop must not block), and a + * bound in milliseconds on the sleep for work no fd announces (-1: none). + * The bridge sets `phase` (input): when it asks, so that liveness is + * current wherever Node's loop next decides it. */ +enum { + SCR_NODE_IO_PREPARE = 0, /* before a poll phase */ + SCR_NODE_IO_CHECK = 1, /* after a check phase: the end of a turn */ + SCR_NODE_IO_JOBS = 2, /* after a microtask checkpoint (any callback) */ +}; +#define SCR_NODE_IO_MAX_FDS 16 +typedef struct ScrNodeIoState { + int phase; + int fds[SCR_NODE_IO_MAX_FDS]; + int nfds; + int alive; + int ready; + int cap_ms; +} ScrNodeIoState; + + +/* ── object handles (scope-B NI-4) ──────────────────────────────────────── + * A value crossing between the runtime and V8. Primitives, strings and + * BigInt are copies; objects are handles: ids into the bridge's import + * table of v8::Global references, one id per V8 object while the runtime + * holds it (the bridge answers the same id for the same object), released + * with `drop`. Typed arrays and Buffers coming out of V8 also carry their + * backing store's memory (BYTES), which the runtime aliases. + * + * Out-values the bridge writes hold bridge-owned string data valid until + * the next bridge entry; the runtime copies it at once. In-values are + * borrowed for the call. */ +enum { + SCR_NV_UNDEFINED = 0, + SCR_NV_NULL = 1, + SCR_NV_BOOL = 2, + SCR_NV_NUMBER = 3, + SCR_NV_STRING = 4, /* WTF-8: lone surrogates as their 3-byte forms */ + SCR_NV_BIGINT = 5, /* str: decimal text with an optional leading '-' */ + SCR_NV_HANDLE = 6, /* handle: an import-table id; flags: SCR_NVF_* */ + SCR_NV_BYTES = 7, /* bytes: a typed array / Buffer / DataView over shared memory */ + SCR_NV_ERROR = 8, /* into V8 only: a new Error of an intrinsic class */ + SCR_NV_JSON = 9, /* into V8 only: str is JSON text, parsed there (a copy) */ + SCR_NV_EXPORT = 10, /* reserved for NI-5: a runtime object exported by reference */ +}; +/* What a handle is, reported with every handle the bridge answers. */ +enum { + SCR_NVF_FUNCTION = 1, + SCR_NVF_SYMBOL = 2, + SCR_NVF_ERROR = 4, + SCR_NVF_ARRAY = 8, + SCR_NVF_PROMISE = 16, + SCR_NVF_CONSTRUCTOR = 32, + SCR_NVF_BUFFER = 64, /* BYTES: a Buffer; DATAVIEW: a DataView */ + SCR_NVF_DATAVIEW = 128, +}; +/* BYTES element kinds: ScrBytesElem's numbering (scr_bytes.h). */ +typedef struct ScrNodeValue { + uint32_t tag; + uint32_t flags; + union { + int boolean; + double num; + struct { + const char *data; + size_t len; + } str; + uint32_t handle; + struct { + uint8_t *data; /* first element */ + size_t len; /* elements */ + uint32_t elem; /* ScrBytesElem */ + uint32_t handle; /* out of V8: the view's handle (keeps the store alive) */ + void *owner; /* into V8: the runtime ScrBytes root that owns data */ + } bytes; + struct { + uint32_t kind; /* scr_error.h SCR_ERR_* (intrinsic class) */ + const char *name, *message, *stack, *code; /* code NULL: none */ + size_t name_len, message_len, stack_len, code_len; + } error; + } u; +} ScrNodeValue; + +/* An island error's own properties, read by `error_info` (bridge-owned + * strings, valid until the next entry; code NULL when it has no string + * code). kind is the nearest intrinsic class (SCR_ERR_*). */ +typedef struct ScrNodeErrorInfo { + uint32_t kind; + const char *name, *message, *stack, *code; + size_t name_len, message_len, stack_len, code_len; +} ScrNodeErrorInfo; + /* What the runtime provides to the bridge. */ typedef struct ScrNodeBridgeHost { /* An asynchronous call settled with this outcome. */ @@ -27,12 +138,40 @@ typedef struct ScrNodeBridgeHost { /* Node ends the process (process.exit, an uncaught exception): the * runtime runs its exit sequence with this code and never returns. */ void (*exit)(int code); + /* A runtime job came due on Node's queues (a SCR_NODE_JOB_* kind and the + * token the runtime passed when it scheduled it). Runs on the stack of + * Node's own dispatch, so Node drains ticks and microtasks after it + * exactly as after any of its callbacks. */ + void (*run)(uint32_t kind, uint32_t token); + /* The runtime's I/O hooks: its state before each poll phase and after + * each check phase, and one dispatch pass (when a watched fd is readable + * or `ready` was set). */ + void (*io_state)(ScrNodeIoState *state); + void (*io_dispatch)(void); + /* V8 finished a microtask checkpoint: the runtime reports the unhandled + * rejections of its own promises, as Node does at the same point. */ + void (*microtasks_done)(void); + /* Island code added (add=1, once=1 for once listeners) or removed + * (add=0) a listener for a process event (SCR_NODE_EVENT_*, `name` its + * event name). `listener` identifies it in later `emit` calls. */ + void (*listen)(uint32_t event, const char *name, uint32_t listener, int add, int once); + /* The lowest stack address JavaScript may use on the stack the runtime + * is calling in from (its main thread's or an async fiber's), for V8's + * stack limit; 0 leaves V8's own. */ + uintptr_t (*stack_limit)(void); /* The start (highest address) of the stack the caller runs on when the * runtime provides it (Node's own stack), 0 for a thread's own stack. * V8 scans the stack conservatively from the current position up to the * start it records for the thread, so it must not record the thread's * when JavaScript runs on the runtime's stack. */ uintptr_t (*stack_start)(void); + /* NI-4: a thenable the runtime awaits (`then`) settled; `value` is the + * fulfillment or (rejected=1) the reason. Runs as a V8 reaction job. */ + void (*settle_value)(uint32_t token, int rejected, const ScrNodeValue *value); + /* NI-4: V8 freed a backing store over runtime bytes (`owner` of a BYTES + * value that went into V8). May run on any thread: the runtime queues + * the release for its own thread. */ + void (*bytes_free)(void *owner); } ScrNodeBridgeHost; typedef struct ScrNodeBridge { @@ -54,20 +193,73 @@ typedef struct ScrNodeBridge { char *(*call)(uint32_t id, int sync, const char *file, const char *specifier, const char *name, const char *args, size_t args_len, size_t *len); void (*release)(void *memory); - /* Node's event loop, driven from the runtime's loop until NI-3 hosts the - * runtime's queues on it: a pollable backend fd, the timeout to its next - * timer (-1: none), whether ref'd work remains, and one non-blocking - * turn (libuv callbacks plus platform tasks). */ - int (*backend_fd)(void); - int (*backend_timeout)(void); - int (*alive)(void); - void (*pump)(void); + /* The hosted loop (scope-B NI-3). Once Node is loaded, the runtime's + * queues live on Node's: `host_loop` starts the integration (the I/O + * handles that watch the runtime's fds, the microtask-checkpoint hook) + * and holds a callback scope open until the next `checkpoint`, so work + * already pending in the runtime and work the current runtime callback + * goes on to queue drain together, at the end of that callback. */ + void (*host_loop)(void); + /* Schedules runtime jobs (SCR_NODE_JOB_*). Timers take Node's delay + * coercion; `repeat` > 0 re-arms after each run (setInterval), and + * `delay` may differ from `repeat` for intervals migrated mid-period. + * Handles are nonzero and name the Node Timeout or Immediate until the + * runtime clears them; a cleared or fired handle is never reused. */ + void (*microtask)(uint32_t token); + void (*tick)(uint32_t token); + uint32_t (*timer)(double delay, double repeat, int ref, uint32_t token); + void (*timer_clear)(uint32_t handle); + void (*timer_ref)(uint32_t handle, int ref); + void (*timer_refresh)(uint32_t handle); + uint32_t (*immediate)(int ref, uint32_t token); + void (*immediate_clear)(uint32_t handle); + void (*immediate_ref)(uint32_t handle, int ref); + /* The checkpoint after a macrotask: closes the scope `host_loop` held + * (or opens and closes one), which runs Node's tick queue and V8's + * microtasks to exhaustion. `microtasks_first` runs the microtasks once + * before the ticks: the end of an ES module main body. */ + void (*checkpoint)(int microtasks_first); + /* Runs Node's event loop until nothing keeps it alive (beforeExit + * included). Returns when the program ends normally, with the exit code + * island code assigned to process.exitCode (0: none). `continue_turn`: + * the switch interrupted a timers or poll phase, so the first iteration + * starts at the poll phase instead of the timers phase. */ + int (*run_loop)(int continue_turn); + /* Calls island listener `listener` of a process event: a signal (`name`, + * signal number `arg`) or exit (code `arg`). */ + void (*emit)(uint32_t event, uint32_t listener, const char *name, int arg); /* Tears Node down at process exit. */ void (*stop)(void); - /* The program ends with `code`, before stop: Node's 'beforeExit' and - * 'exit' run, and the result is the code Node's process ends with - * (process.exitCode in Node, else `code`). */ - int (*finish)(int code); + /* ── object handles (NI-4). Each operation answers 0 (returned: *out is + * the result) or 1 (threw: *out is the thrown value). ── */ + /* Loads a module by absolute path: require(path) (esm=0, the exports), + * or import(path) (esm=1, a promise of the namespace). */ + int (*load)(const char *path, int esm, ScrNodeValue *out); + /* Property access on handle h; key goes through ToPropertyKey. */ + int (*get)(uint32_t h, const ScrNodeValue *key, ScrNodeValue *out); + int (*set)(uint32_t h, const ScrNodeValue *key, const ScrNodeValue *value, ScrNodeValue *out); + int (*has)(uint32_t h, const ScrNodeValue *key, ScrNodeValue *out); /* out: BOOL */ + int (*del)(uint32_t h, const ScrNodeValue *key, ScrNodeValue *out); /* out: BOOL */ + /* fn(...args) with receiver this_ (NULL: undefined). */ + int (*invoke)(uint32_t fn, const ScrNodeValue *this_, const ScrNodeValue *args, size_t argc, + ScrNodeValue *out); + /* h[key](...args) with h as the receiver. */ + int (*invoke_method)(uint32_t h, const ScrNodeValue *key, const ScrNodeValue *args, size_t argc, + ScrNodeValue *out); + int (*construct)(uint32_t h, const ScrNodeValue *args, size_t argc, ScrNodeValue *out); + int (*instance_of)(const ScrNodeValue *value, uint32_t ctor, ScrNodeValue *out); /* BOOL */ + /* Own keys of h: mode 0 Object.keys, mode 1 Reflect.ownKeys. out: a + * handle to a fresh array, which the runtime reads and drops. */ + int (*own_keys)(uint32_t h, int mode, ScrNodeValue *out); + /* Promise.resolve(value).then(...): settles through host->settle_value + * with `token`. Answers 1 with *out thrown if `then` could not attach. */ + int (*then)(const ScrNodeValue *value, uint32_t token, ScrNodeValue *out); + /* The own name, message, stack and string code of an error handle. */ + void (*error_info)(uint32_t h, ScrNodeErrorInfo *out); + /* Releases handle h's Global. */ + void (*drop)(uint32_t h); + /* The number of live import-table entries (leak tests). */ + uint32_t (*live_handles)(void); } ScrNodeBridge; #endif /* SCR_NODE_BRIDGE_H */ diff --git a/packages/runtime/src/scr_node_island.c b/packages/runtime/src/scr_node_island.c index deefc303e..0215a1c12 100644 --- a/packages/runtime/src/scr_node_island.c +++ b/packages/runtime/src/scr_node_island.c @@ -20,20 +20,36 @@ * the bridge (a callback scope, so ticks and microtasks drain when it * returns) and parses the outcome: the result's JSON copy, or the copied * error (see scr_node_island_host.cjs). An asynchronous call registers its - * promise first, because the bridge may settle it before returning. Until - * NI-3 hosts the runtime's queues on Node's libuv, the runtime's loop drives - * Node's: the loop slot keeps the program alive while Node has ref'd work, - * sleeps on its backend fd up to its next timer, and runs one non-blocking - * turn of it per loop turn. + * promise first, because the bridge may settle it before returning. + * + * One loop (scope-B NI-3). Right after Node starts, scr_loop_host moves the + * runtime's pending ticks, promise jobs, timers and immediates onto Node's + * queues and from then on schedules them there, and the bridge watches the + * runtime's I/O on Node's libuv loop: there is one loop and one set of + * queues, so the jobs of both sides interleave as in Node. Calls into + * JavaScript do not drain queues themselves; the runtime's checkpoints do + * (the bridge's `checkpoint`, and Node's own after every callback it + * dispatches). + * + * Process state is shared, not mirrored: island listeners for signals and + * 'exit' join the runtime's registries (scr_events.c) at their position, + * so the runtime keeps owning every signal disposition; the exit code + * island code leaves in process.exitCode becomes the program's when the + * loop ends; Node's process.exit runs the runtime's exit sequence. * * Node's stack. V8 fixes an isolate's stack limit where it is created, and * calls into Node can come from any async fiber (512 KB stacks without a - * guard page) or from the main stack. Every entry into Node therefore runs - * on one dedicated stack of NI_STACK bytes with a guard page, reached by a - * context switch; the runtime's own stack guard follows it there. */ + * guard page) or from the main stack. Every entry into Node from outside + * it runs on one dedicated stack of NI_STACK bytes with a guard page, + * reached by a context switch; the runtime's own stack guard follows it + * there. Node's loop runs there too once it hosts the runtime's queues, so + * a runtime job it dispatches may resume an async fiber that calls into + * Node again from the fiber's stack: the bridge sets V8's limit for the + * stack each entry comes from (ni_stack_limit). */ #define _XOPEN_SOURCE 700 #define _DARWIN_C_SOURCE 1 #include "scr_runtime.h" +#include "scr_events.h" #include "scr_node_bridge.h" #include "scr_node_island.h" @@ -76,6 +92,7 @@ void __sanitizer_start_switch_fiber(void **fake_stack_save, const void *bottom, void __sanitizer_finish_switch_fiber(void *fake_stack_save, const void **bottom_old, size_t *size_old); void __lsan_register_root_region(const void *p, size_t size) __attribute__((weak)); +void __asan_unpoison_memory_region(void const volatile *addr, size_t size); #endif #endif @@ -672,19 +689,24 @@ static void ni_open_modules(void) { ni_fatal("scriptc: cannot write this program's Node island modules to a private directory\n"); } -/* The calling module's path inside the tree: `/`. */ -static char *ni_caller_path(const ScrStr *relative) { +/* A path inside the tree: `/`. */ +static char *ni_tree_path(const char *relative, size_t len) { size_t root = strlen(ni_root); bool slash = root > 0 && ni_root[root - 1] == '/'; - char *path = malloc(root + 1 + relative->len + 1); + char *path = malloc(root + 1 + len + 1); if (path == NULL) ni_oom(); memcpy(path, ni_root, root); if (!slash) path[root++] = '/'; - memcpy(path + root, relative->data, relative->len); - path[root + relative->len] = '\0'; + memcpy(path + root, relative, len); + path[root + len] = '\0'; return path; } +/* The calling module's path inside the tree. */ +static char *ni_caller_path(const ScrStr *relative) { + return ni_tree_path(relative->data, relative->len); +} + /* The bootstrap: the tree root for the host's resolution guard, then the * host script. +1 malloc'd. */ static char *ni_bootstrap(void) { @@ -773,42 +795,51 @@ static void ni_exit(int code) { ni_fatal(""); } +/* An island listener for a process event was added or removed. */ +static void ni_listen(uint32_t event, const char *name, uint32_t listener, int add, int once) { + (void)once; /* Node's once wrapper removes itself through removeListener */ + if (event == SCR_NODE_EVENT_SIGNAL) { + if (add) { + ScrStr *s = scr_str_new(name, strlen(name)); + scr_signal_on_island(s, listener); + scr_str_release(s); + } else { + scr_signal_off_island(listener); + } + } else if (event == SCR_NODE_EVENT_EXIT) { + if (add) scr_process_on_exit_island(listener); + else scr_process_off_exit_island(listener); + } +} + /* The start of Node's stack for callers on it (V8's stack-scan bound). */ static uintptr_t ni_stack_start(void); -static const ScrNodeBridgeHost ni_host = {&ni_settle, &ni_exit, &ni_stack_start}; +/* The main thread's stack, for V8's stack limit (ni_stack_limit). */ +static uintptr_t ni_main_lo = 0, ni_main_hi = 0; -/* ── the loop slot ────────────────────────────────────────────────────── */ - -static bool ni_alive(void) { return ni_bridge->alive() != 0; } -static void ni_pump_job(void *p); -static void ni_stop_job(void *p); -static void ni_on_node_stack(void (*job)(void *), void *arg); -static void ni_pump(void) { ni_on_node_stack(ni_pump_job, NULL); } -static int ni_backend_fd(void) { return ni_bridge->backend_fd(); } -static int ni_backend_timeout(void) { return ni_bridge->backend_timeout(); } -typedef struct { - int code; -} NiFinish; -static void ni_finish_job(void *p); - -/* At exit, after the program's own exit work: Node's process ends with - * the program's code ('beforeExit', then 'exit'), and Node stops. A code - * Node's process changed (process.exitCode in Node) is the program's. */ -static void ni_stop_atexit(void) { - int code = (int)scr_process_exit_code_or_zero(); - NiFinish finish = {code}; - /* Node writes to the same stdout. */ - fflush(stdout); - ni_on_node_stack(ni_finish_job, &finish); - ni_on_node_stack(ni_stop_job, NULL); - if (finish.code != code) { - fflush(NULL); - if (ni_temp_root[0] != '\0') ni_remove_temp_root(); - _Exit(finish.code & 0xff); +static void ni_main_stack(void) { +#if defined(__APPLE__) + pthread_t self = pthread_self(); + ni_main_hi = (uintptr_t)pthread_get_stackaddr_np(self); + ni_main_lo = ni_main_hi - pthread_get_stacksize_np(self); +#elif defined(__linux__) + pthread_attr_t attr; + if (pthread_getattr_np(pthread_self(), &attr) == 0) { + void *addr = NULL; + size_t size = 0; + if (pthread_attr_getstack(&attr, &addr, &size) == 0) { + ni_main_lo = (uintptr_t)addr; + ni_main_hi = ni_main_lo + size; + } + pthread_attr_destroy(&attr); } +#endif } +static void ni_on_node_stack(void (*job)(void *), void *arg); +static const ScrNodeBridgeHost ni_host; + /* ── Node's stack ─────────────────────────────────────────────────────── */ static char *ni_stack = NULL; /* the mapping; usable above NI_GUARD */ @@ -857,17 +888,103 @@ static void ni_trampoline(void) { } } -static uintptr_t ni_stack_start(void) { - uintptr_t sp = (uintptr_t)__builtin_frame_address(0); - return ni_stack != NULL && sp >= (uintptr_t)ni_stack && sp < (uintptr_t)ni_stack + NI_STACK - ? (uintptr_t)ni_stack + NI_STACK - : 0; +/* Whether `sp` lies on Node's stack. */ +static bool ni_stack_has(uintptr_t sp) { + return ni_stack != NULL && sp > (uintptr_t)ni_stack && sp <= (uintptr_t)ni_stack + NI_STACK; +} + +/* The lowest address of Node's stack in use while runtime code Node's + * frames called runs (a host callback below), 0 outside them: the + * callback's frame, or lower, where that code switched to a fiber's stack + * (scr_node_island_leave_stack) and left its own frames live. Runtime code + * may resume an async fiber there, and the fiber may call into Node again: + * that entry runs on Node's stack below this mark, so every frame of Node + * stays on the one stack V8 scans (heap::base::Stack, bounded by + * ni_stack_start), never on a fiber's stack beyond its bound. */ +static uintptr_t ni_mark = 0; + +/* Host callbacks run runtime code from Node's frames: each records the + * mark for its duration. */ +#define NI_MARK_BEGIN \ + uintptr_t ni_saved_mark = ni_mark; \ + do { \ + uintptr_t ni_here = (uintptr_t)__builtin_frame_address(0); \ + if (ni_stack_has(ni_here)) ni_mark = ni_here; \ + } while (0) +#define NI_MARK_END (ni_mark = ni_saved_mark) + +void scr_node_island_leave_stack(uintptr_t sp) { + if (ni_mark != 0 && ni_stack_has(sp) && sp < ni_mark) ni_mark = sp; +} + +/* A nested entry: job(arg) on Node's stack below the mark, from a fiber. */ +static void (*ni_nested_job)(void *) = NULL; +static void *ni_nested_arg = NULL; +static ucontext_t *ni_nested_return = NULL; + +#ifdef NI_ASAN +static uintptr_t ni_nested_top = 0; /* the nested context's stack top */ +#endif + +static void ni_nested_trampoline(void) { +#ifdef NI_ASAN + const void *caller_bottom; + size_t caller_size; + uintptr_t top = ni_nested_top; + __sanitizer_finish_switch_fiber(NULL, &caller_bottom, &caller_size); +#endif + void (*job)(void *) = ni_nested_job; + void *arg = ni_nested_arg; + ucontext_t *back = ni_nested_return; + job(arg); +#ifdef NI_ASAN + /* This context ends: no fake stack to keep. Its frame never returns, so + * the redzones ASan poisoned around this frame's locals would stay in + * the shadow of Node's stack, where Node's own frames (uninstrumented) + * later trip over them. Linux's ASan clears a context's stack when it is + * switched to (its swapcontext interceptor); macOS's has none, so this + * frame's part of the stack is unpoisoned here. */ + uintptr_t low = ((uintptr_t)__builtin_frame_address(0) - 4096) & ~(uintptr_t)7; + if (top > low) __asan_unpoison_memory_region((const void *)low, top - low); + __sanitizer_start_switch_fiber(NULL, caller_bottom, caller_size); +#endif + setcontext(back); +} + +static void ni_on_node_stack_below_mark(void (*job)(void *), void *arg) { + ucontext_t nested, caller; + char *lo = ni_stack + NI_GUARD; + /* Below the red zone and 16-byte aligned, as makecontext expects. */ + uintptr_t hi = (ni_mark - 512) & ~(uintptr_t)15; + ni_make_context(&nested, lo, hi - (uintptr_t)lo, ni_nested_trampoline); +#ifdef NI_ASAN + ni_nested_top = hi; +#endif + ni_nested_job = job; + ni_nested_arg = arg; + ni_nested_return = &caller; + uintptr_t guard = scr_stack_guard; + SCR_STACK_GUARD_SET(scr_stack_guard_for((uintptr_t)lo, hi - (uintptr_t)lo)); +#ifdef NI_ASAN + void *caller_fake = NULL; + __sanitizer_start_switch_fiber(&caller_fake, lo, hi - (uintptr_t)lo); +#endif + swapcontext(&caller, &nested); +#ifdef NI_ASAN + const void *bottom; + size_t size; + __sanitizer_finish_switch_fiber(caller_fake, &bottom, &size); +#endif + SCR_STACK_GUARD_SET(guard); } /* Runs job(arg) on Node's stack (directly when already there). */ static void ni_on_node_stack(void (*job)(void *), void *arg) { if (ni_on_stack) { - job(arg); + if (ni_mark != 0 && !ni_stack_has((uintptr_t)__builtin_frame_address(0))) + ni_on_node_stack_below_mark(job, arg); + else + job(arg); return; } if (ni_stack == NULL) { @@ -974,25 +1091,221 @@ static void ni_call_job(void *p) { ni_bridge->call(a->id, a->sync, a->file, a->specifier, a->name, a->args, a->args_len, &a->len); } -static void ni_pump_job(void *p) { - (void)p; - ni_bridge->pump(); +/* V8's stack limit for the stack an entry comes from: Node's own stack + * (a reserve above its guard page), the main thread's (Node's default + * --stack-size, 984 KiB, below its top), or an async fiber's (the + * runtime's guard for it: its mapping's start plus the reserve compiled + * code keeps). */ +static uintptr_t ni_stack_limit(void) { + /* The frame address, not a local's: under ASan's use-after-return + * detection locals live on a fake stack elsewhere in memory. */ + uintptr_t sp = (uintptr_t)__builtin_frame_address(0); + if (ni_stack_has(sp)) return (uintptr_t)(ni_stack + NI_GUARD) + NI_STACK / 8; + if (sp > ni_main_lo && sp <= ni_main_hi) { + uintptr_t limit = ni_main_hi > 984 * 1024 ? ni_main_hi - 984 * 1024 : 0; + return limit > ni_main_lo ? limit : ni_main_lo + 64 * 1024; + } + if (scr_stack_guard != 0 && scr_stack_guard < sp) return scr_stack_guard; + return 0; +} + +/* The start of Node's stack for callers on it (V8's stack-scan bound). */ +static uintptr_t ni_stack_start(void) { + uintptr_t sp = (uintptr_t)__builtin_frame_address(0); + return ni_stack_has(sp) ? (uintptr_t)ni_stack + NI_STACK : 0; +} + +/* The host callbacks that run runtime code, with the mark recorded. */ +static void ni_host_settle(uint32_t id, const char *outcome, size_t len) { + NI_MARK_BEGIN; + ni_settle(id, outcome, len); + NI_MARK_END; +} + +static void ni_host_exit(int code) { + NI_MARK_BEGIN; /* never returns */ + (void)ni_saved_mark; + ni_exit(code); +} + +static void ni_host_run(uint32_t kind, uint32_t token) { + NI_MARK_BEGIN; + scr_loop_hosted_run(kind, token); + NI_MARK_END; +} + +static void ni_host_io(void) { + NI_MARK_BEGIN; + scr_loop_hosted_io(); + NI_MARK_END; +} + +static void ni_host_microtasks_done(void) { + NI_MARK_BEGIN; + scr_loop_hosted_microtasks_done(); + NI_MARK_END; +} + +static void ni_host_listen(uint32_t event, const char *name, uint32_t listener, int add, + int once) { + NI_MARK_BEGIN; + ni_listen(event, name, listener, add, once); + NI_MARK_END; +} + +static const ScrNodeBridgeHost ni_host = { + &ni_host_settle, + &ni_host_exit, + &ni_host_run, + &scr_loop_hosted_io_state, + &ni_host_io, + &ni_host_microtasks_done, + &ni_host_listen, + &ni_stack_limit, + &ni_stack_start, + &scr_ni_settle_value, + &scr_ni_bytes_free, +}; + +typedef struct { + uint32_t event, listener; + const char *name; + int arg; +} NiEmit; + +static void ni_emit_job(void *p) { + NiEmit *e = p; + ni_bridge->emit(e->event, e->listener, e->name, e->arg); +} + +/* An island listener of a process event, on Node's stack (an exit listener + * runs from the runtime's exit sequence, on whatever stack called it). */ +void scr_node_island_emit(uint32_t event, uint32_t listener, ScrStr *name, int arg) { + if (ni_bridge == NULL) return; + char *n = malloc(name->len + 1); + if (n == NULL) ni_oom(); + memcpy(n, name->data, name->len); + n[name->len] = '\0'; + NiEmit e = {event, listener, n, arg}; + ni_on_node_stack(ni_emit_job, &e); + free(n); } static void ni_stop_job(void *p) { (void)p; + scr_ni_stop(); ni_bridge->stop(); } -static void ni_finish_job(void *p) { - NiFinish *a = p; - a->code = ni_bridge->finish(a->code); +/* Process end after a normal return: the 'exit' listeners of both sides + * run while Node can still call into JavaScript (this handler runs before + * the events unit's own, which then finds them done), then Node stops. */ +static void ni_stop_atexit(void) { + scr_run_exit_listeners((double)scr_exit_code_hint_get()); + /* SCRIPTC_NODE_ISLANDS_TRACE=handles: the import table at exit (leak + * tests: acyclic island objects the program dropped are released). */ + const char *trace = getenv("SCRIPTC_NODE_ISLANDS_TRACE"); + if (trace != NULL && strcmp(trace, "handles") == 0) { + fflush(stdout); + fprintf(stderr, "[node-islands] live handles %u\n", scr_ni_live_handles()); + } + ni_on_node_stack(ni_stop_job, NULL); } +/* The hosted loop's run_loop on Node's stack. */ +typedef struct { + int continue_turn, code; +} NiRunLoop; + +static void ni_run_loop_job(void *p) { + NiRunLoop *a = p; + a->code = ni_bridge->run_loop(a->continue_turn); +} + +static int ni_run_loop(int continue_turn) { + NiRunLoop a = {continue_turn, 0}; + ni_on_node_stack(ni_run_loop_job, &a); + return a.code; +} + +/* The bridge as the hosted loop sees it. Every entry that runs JavaScript + * (scheduling a tick, timer or immediate goes through the bootstrap's + * helpers; a checkpoint drains queues; Node's loop) runs on Node's stack, + * below the mark when it comes from an async fiber that a host callback + * resumed, like every other entry: V8 scans only Node's stack, so no frame + * of Node may live on a fiber's stack or the main thread's. */ +typedef struct { + int op; + uint32_t a, b; + double d0, d1; + int i0; + uint32_t result; +} NiHosted; + +enum { + NI_H_MICROTASK, + NI_H_TICK, + NI_H_TIMER, + NI_H_TIMER_CLEAR, + NI_H_TIMER_REF, + NI_H_TIMER_REFRESH, + NI_H_IMMEDIATE, + NI_H_IMMEDIATE_CLEAR, + NI_H_IMMEDIATE_REF, + NI_H_CHECKPOINT, +}; + +static void ni_hosted_job(void *p) { + NiHosted *h = p; + switch (h->op) { + case NI_H_MICROTASK: ni_bridge->microtask(h->a); break; + case NI_H_TICK: ni_bridge->tick(h->a); break; + case NI_H_TIMER: h->result = ni_bridge->timer(h->d0, h->d1, h->i0, h->a); break; + case NI_H_TIMER_CLEAR: ni_bridge->timer_clear(h->a); break; + case NI_H_TIMER_REF: ni_bridge->timer_ref(h->a, h->i0); break; + case NI_H_TIMER_REFRESH: ni_bridge->timer_refresh(h->a); break; + case NI_H_IMMEDIATE: h->result = ni_bridge->immediate(h->i0, h->a); break; + case NI_H_IMMEDIATE_CLEAR: ni_bridge->immediate_clear(h->a); break; + case NI_H_IMMEDIATE_REF: ni_bridge->immediate_ref(h->a, h->i0); break; + case NI_H_CHECKPOINT: ni_bridge->checkpoint(h->i0); break; + default: break; + } +} + +static uint32_t ni_hosted_run(int op, uint32_t a, double d0, double d1, int i0) { + NiHosted h = {op, a, 0, d0, d1, i0, 0}; + ni_on_node_stack(ni_hosted_job, &h); + return h.result; +} + +static void ni_h_microtask(uint32_t token) { ni_hosted_run(NI_H_MICROTASK, token, 0, 0, 0); } +static void ni_h_tick(uint32_t token) { ni_hosted_run(NI_H_TICK, token, 0, 0, 0); } +static uint32_t ni_h_timer(double delay, double repeat, int ref, uint32_t token) { + return ni_hosted_run(NI_H_TIMER, token, delay, repeat, ref); +} +static void ni_h_timer_clear(uint32_t h) { ni_hosted_run(NI_H_TIMER_CLEAR, h, 0, 0, 0); } +static void ni_h_timer_ref(uint32_t h, int ref) { ni_hosted_run(NI_H_TIMER_REF, h, 0, 0, ref); } +static void ni_h_timer_refresh(uint32_t h) { ni_hosted_run(NI_H_TIMER_REFRESH, h, 0, 0, 0); } +static uint32_t ni_h_immediate(int ref, uint32_t token) { + return ni_hosted_run(NI_H_IMMEDIATE, token, 0, 0, ref); +} +static void ni_h_immediate_clear(uint32_t h) { ni_hosted_run(NI_H_IMMEDIATE_CLEAR, h, 0, 0, 0); } +static void ni_h_immediate_ref(uint32_t h, int ref) { + ni_hosted_run(NI_H_IMMEDIATE_REF, h, 0, 0, ref); +} +static void ni_h_checkpoint(int microtasks_first) { + ni_hosted_run(NI_H_CHECKPOINT, 0, 0, 0, microtasks_first); +} + +static ScrNodeBridge ni_hosted; + /* ── loading Node ─────────────────────────────────────────────────────── */ static void ni_start(void) { if (ni_bridge != NULL) return; + /* Timers pending now keep their deadlines relative to this moment: the + * time Node takes to load is invisible to their order. */ + double since = scr_now_ms(); const char *dir = ni_pack_dir(); ni_open_modules(); char path[4352]; @@ -1035,6 +1348,7 @@ static void ni_start(void) { /* Flush first: Node writes to the same stdout. */ fflush(stdout); ni_bridge = bridge; + ni_main_stack(); char *bootstrap = ni_bootstrap(); NiStart start = {bootstrap, argv0, NULL}; ni_on_node_stack(ni_start_job, &start); @@ -1046,10 +1360,42 @@ static void ni_start(void) { bridge->release(error); ni_fatal(message); } + /* Island listeners need the runtime's event registries and their + * delivery; installed before the stop handler, whose 'exit' listeners + * run first. */ + scr_events_install(); + scr_ni_handles_install(); scr_atexit(ni_stop_atexit); - scr_loop_set_node_island(&ni_alive, &ni_pump, &ni_backend_fd, &ni_backend_timeout); + ni_hosted = *bridge; + ni_hosted.run_loop = &ni_run_loop; + ni_hosted.microtask = &ni_h_microtask; + ni_hosted.tick = &ni_h_tick; + ni_hosted.timer = &ni_h_timer; + ni_hosted.timer_clear = &ni_h_timer_clear; + ni_hosted.timer_ref = &ni_h_timer_ref; + ni_hosted.timer_refresh = &ni_h_timer_refresh; + ni_hosted.immediate = &ni_h_immediate; + ni_hosted.immediate_clear = &ni_h_immediate_clear; + ni_hosted.immediate_ref = &ni_h_immediate_ref; + ni_hosted.checkpoint = &ni_h_checkpoint; + scr_loop_host(&ni_hosted, since); } +const ScrNodeBridge *scr_ni_bridge(void) { + ni_start(); + return ni_bridge; +} + +const ScrNodeBridge *scr_ni_bridge_loaded(void) { return ni_bridge; } + +char *scr_ni_module_path(const ScrStr *path) { + if (path->len >= 2 && path->data[0] == '.' && path->data[1] == '/') + return ni_tree_path(path->data + 2, path->len - 2); + return ni_cstr_of(path->data, path->len); +} + +void scr_ni_on_node_stack(void (*job)(void *), void *arg) { ni_on_node_stack(job, arg); } + /* ── calls ────────────────────────────────────────────────────────────── */ /* The JSON text of the argument array (+1 dyn string), or NULL with the @@ -1141,6 +1487,13 @@ ScrPromise *scr_node_island_call(ScrStr *file, ScrStr *specifier, ScrStr *name, #else /* _WIN32 || __wasi__ */ +void scr_node_island_emit(uint32_t event, uint32_t listener, ScrStr *name, int arg) { + (void)event; + (void)listener; + (void)name; + (void)arg; +} + /* Node islands need a libnode pack, which these targets do not have: the * compiler refuses them there, so these entry points only report that. */ void scr_node_island_install(const char *libnode, const char *node_version) { diff --git a/packages/runtime/src/scr_node_island.h b/packages/runtime/src/scr_node_island.h index 33d1d17b4..8bb0c5ccc 100644 --- a/packages/runtime/src/scr_node_island.h +++ b/packages/runtime/src/scr_node_island.h @@ -34,10 +34,92 @@ ScrDyn *scr_node_island_call_sync(ScrStr *file, ScrStr *specifier, ScrStr *name, * error (also when the arguments cannot be copied). Never throws. */ ScrPromise *scr_node_island_call(ScrStr *file, ScrStr *specifier, ScrStr *name, ScrDyn *args); -/* The loop slot (SCR_NODE_ISLANDS build of scr_async.c): `pending` keeps - * the loop alive, `dispatch` runs at every turn top, `pollfd` joins the - * idle poll and `timeout` (milliseconds, -1 for none) bounds its sleep. */ -void scr_loop_set_node_island(bool (*pending)(void), void (*dispatch)(void), int (*pollfd)(void), - int (*timeout)(void)); +/* The hosted loop (SCR_NODE_ISLANDS build of scr_async.c, scope-B NI-3). + * scr_loop_host moves the runtime's pending ticks, promise jobs, timers + * and immediates onto Node's queues through `bridge` and schedules every + * later one there; `since_ms` (scr_now_ms() before Node began loading) is + * the clock pending timers count their remaining delay from. The rest are + * the bridge's host callbacks (scr_node_bridge.h). */ +struct ScrNodeBridge; +struct ScrNodeIoState; +void scr_loop_host(const struct ScrNodeBridge *bridge, double since_ms); +bool scr_loop_is_hosted(void); +void scr_loop_hosted_run(uint32_t kind, uint32_t token); +void scr_loop_hosted_io_state(struct ScrNodeIoState *state); +void scr_loop_hosted_io(void); +void scr_loop_hosted_microtasks_done(void); + +/* Runtime code switches stacks (scr_switch) at `sp`: when that is Node's + * stack, the frames above stay live until it switches back, so a call + * into Node.js from the other stack runs below them. */ +void scr_node_island_leave_stack(uintptr_t sp); + +/* Process events island code listens to (SCR_NODE_EVENT_* of the bridge + * ABI). Their listeners sit in the runtime's registries (the + * SCR_NODE_ISLANDS build of scr_events.c) at their registration position, + * and a delivery calls them through scr_node_island_emit: a signal with + * its number, exit with the code. */ +enum { SCR_NODE_ISLAND_EVENT_SIGNAL = 1, SCR_NODE_ISLAND_EVENT_EXIT = 2 }; +void scr_node_island_emit(uint32_t event, uint32_t listener, ScrStr *name, int arg); +void scr_signal_on_island(ScrStr *name, uint32_t listener); +void scr_signal_off_island(uint32_t listener); +void scr_process_on_exit_island(uint32_t listener); +void scr_process_off_exit_island(uint32_t listener); + +/* ── object handles (scope-B NI-4, scr_node_island_handle.c) ───────────── + * Island values the program holds are SCR_DYN_HANDLE boxes with tag + * SCR_DYNH_NODE_ISLAND: one handle per V8 value, so `===` is identity. + * Primitives, strings and BigInt are copies; island promises are scriptc + * promises; typed arrays and Buffers are ScrBytes over the same memory. + * All arguments are borrowed; ScrDyn results are +1, NULL (or false, or + * -1 for the int forms) with the exception pending. The first call loads + * Node. See scriptc-perf/results/ni-4/ABI.md. */ +ScrDyn *scr_ni_require(ScrStr *path); /* require(absolute path): the exports */ +ScrPromise *scr_ni_import(ScrStr *path); /* import(absolute path): the namespace */ +ScrDyn *scr_ni_get(ScrDyn *obj, ScrDyn *key); +ScrDyn *scr_ni_get_str(ScrDyn *obj, const char *key, size_t len); +bool scr_ni_set(ScrDyn *obj, ScrDyn *key, ScrDyn *value); +bool scr_ni_set_str(ScrDyn *obj, const char *key, size_t len, ScrDyn *value); +int scr_ni_has(ScrDyn *obj, ScrDyn *key); +int scr_ni_delete(ScrDyn *obj, ScrDyn *key); +ScrDyn *scr_ni_call(ScrDyn *fn, ScrDyn *this_, ScrDyn *const *args, size_t argc); +ScrDyn *scr_ni_call_method(ScrDyn *obj, const char *key, size_t len, ScrDyn *const *args, + size_t argc); +ScrDyn *scr_ni_construct(ScrDyn *ctor, ScrDyn *const *args, size_t argc); +ScrStr *scr_ni_typeof(ScrDyn *value); +int scr_ni_instanceof(ScrDyn *value, ScrDyn *ctor); +bool scr_ni_is_handle(const ScrDyn *value); +bool scr_ni_is_function(const ScrDyn *value); /* an island handle that is callable */ +/* util.inspect of an island value at nesting `recurse` with limit `depth`, + * by Node's own util.inspect (+1; "" with the exception pending). */ +ScrStr *scr_ni_inspect(const ScrDyn *value, double recurse, double depth); +ScrDyn *scr_ni_own_keys(ScrDyn *obj, int mode); /* 0 Object.keys, 1 Reflect.ownKeys */ +ScrPromise *scr_ni_await(ScrDyn *value); +uint32_t scr_ni_live_handles(void); /* live import-table entries (leak tests) */ + +/* Between the two island units. */ +struct ScrNodeValue; +const struct ScrNodeBridge *scr_ni_bridge(void); /* loads Node */ +const struct ScrNodeBridge *scr_ni_bridge_loaded(void); /* NULL until loaded */ +/* The file a module load names, +1 malloc'd: `./` is a path in the + * program's module tree, under its root once Node is loaded; anything else + * (a built-in module) as it is. */ +char *scr_ni_module_path(const ScrStr *path); +void scr_ni_on_node_stack(void (*job)(void *), void *arg); +void scr_ni_settle_value(uint32_t token, int rejected, const struct ScrNodeValue *value); +void scr_ni_bytes_free(void *owner); +void scr_ni_handles_install(void); +void scr_ni_stop(void); + +/* Island operations the compiler emits (lib-shared.ts adapters). */ +ScrDyn *scr_node_island_apply(ScrDyn *fn, ScrDyn *args); +/* fn called with `this` (a method read before its arguments ran). */ +ScrDyn *scr_node_island_apply_to(ScrDyn *fn, ScrDyn *this_, ScrDyn *args); +ScrDyn *scr_node_island_construct(ScrDyn *ctor, ScrDyn *args); +ScrDyn *scr_node_island_get(ScrDyn *obj, ScrStr *key); +bool scr_node_island_instanceof(ScrDyn *value, ScrDyn *ctor); +ScrDyn *scr_node_island_invoke(ScrDyn *obj, ScrStr *key, ScrDyn *args); +void scr_node_island_set(ScrDyn *obj, ScrStr *key, ScrDyn *value); +void scr_node_island_set_key(ScrDyn *obj, ScrDyn *key, ScrDyn *value); #endif /* SCR_NODE_ISLAND_H */ diff --git a/packages/runtime/src/scr_node_island_handle.c b/packages/runtime/src/scr_node_island_handle.c new file mode 100644 index 000000000..ad51e19af --- /dev/null +++ b/packages/runtime/src/scr_node_island_handle.c @@ -0,0 +1,1139 @@ +/* Node islands: object handles (scope-B milestone NI-4). Linked with + * scr_node_island.c into programs whose IR reaches Node. + * + * Island values the program holds are handles: a ScrNodeRef per V8 value, + * naming an entry of the bridge's import table (a v8::Global), boxed as + * SCR_DYN_HANDLE with tag SCR_DYNH_NODE_ISLAND. The bridge answers one id + * per V8 value while the runtime holds it and the runtime keeps one ref per + * id (ni_refs), so the same V8 object always comes back as the same ref: + * `===` compares handle pointers and identity is exact. When the last box + * releases a ref, the bridge drops its Global and V8 may collect the + * object (acyclic garbage; cycles across both heaps wait for NI-8). + * + * Boundary rules (DESIGN §2.5.1): undefined, null, booleans, numbers + * (exact: -0, NaN), strings (WTF-8, lone surrogates kept) and BigInt are + * copied; everything else is a handle. Island promises cross as scriptc + * promises settled by a V8 reaction, one per V8 promise while it lives. + * Typed arrays and Buffers cross as ScrBytes over the same memory (the V8 + * backing store, kept alive by the bridge while the runtime holds the + * bytes), and runtime bytes cross into V8 as a backing store over the + * runtime's memory. Island errors thrown into the program become ScrErrors + * with V8's name, message, stack and code, and become the original V8 + * error again when they cross back. + * + * Runtime objects that represent a V8 value (a promise, an error, bytes) + * are "pinned": the table below holds the object and its ref, and a lazy + * sweep at bridge entries releases both once the table holds the object's + * last reference. Native objects and arrays that are not island values + * cross by JSON copy until NI-5's exports (a documented NI-4 shortcut). */ +#define _XOPEN_SOURCE 700 +#define _DARWIN_C_SOURCE 1 +#include "scr_runtime.h" +#include "scr_node_bridge.h" +#include "scr_node_island.h" + +#include +#include +#include +#include +#include + +#if !defined(_WIN32) && !defined(__wasi__) + +typedef struct ScrNodeRef { + size_t rc; + uint32_t id; + uint32_t flags; /* SCR_NVF_* */ +} ScrNodeRef; + +static ScrNodeRef **ni_refs = NULL; /* by bridge id */ +static size_t ni_refs_cap = 0; +static bool ni_stopped = false; + +SCR_COLD static void nh_oom(void) { scr_trap("scriptc: out of memory\n"); } + +static const ScrNodeBridge *nh_bridge(void) { return scr_ni_bridge(); } + +/* ── pinned runtime objects ──────────────────────────────────────────── */ + +enum { NH_PIN_PROMISE = 1, NH_PIN_ERROR = 2, NH_PIN_BYTES = 3 }; + +typedef struct { + void *obj; /* retained; its reference count is the first word */ + ScrNodeRef *ref; /* retained */ + int kind; +} NhPin; + +static NhPin *nh_pins = NULL; +static size_t nh_npins = 0, nh_pins_cap = 0, nh_sweep_at = 64; + +static void nh_ref_release(ScrNodeRef *ref); + +static void nh_pin_release_obj(NhPin *pin) { + switch (pin->kind) { + case NH_PIN_PROMISE: scr_promise_release((ScrPromise *)pin->obj); break; + case NH_PIN_ERROR: scr_error_release((ScrError *)pin->obj); break; + case NH_PIN_BYTES: scr_bytes_release((ScrBytes *)pin->obj); break; + } +} + +static void nh_pin(void *obj, int kind, ScrNodeRef *ref) { + if (nh_npins == nh_pins_cap) { + size_t cap = nh_pins_cap ? nh_pins_cap * 2 : 16; + NhPin *grown = realloc(nh_pins, cap * sizeof *grown); + if (grown == NULL) nh_oom(); + nh_pins = grown; + nh_pins_cap = cap; + } + switch (kind) { + case NH_PIN_PROMISE: scr_promise_retain((ScrPromise *)obj); break; + case NH_PIN_ERROR: scr_error_retain((ScrError *)obj); break; + case NH_PIN_BYTES: scr_bytes_retain((ScrBytes *)obj); break; + } + ref->rc++; + nh_pins[nh_npins++] = (NhPin){obj, ref, kind}; +} + +/* The ref a pinned runtime object represents, or NULL. */ +static ScrNodeRef *nh_pinned(const void *obj, int kind) { + for (size_t i = 0; i < nh_npins; i++) + if (nh_pins[i].obj == obj && nh_pins[i].kind == kind) return nh_pins[i].ref; + return NULL; +} + +/* The pinned object of a ref, or NULL. */ +static void *nh_pinned_obj(const ScrNodeRef *ref, int kind) { + for (size_t i = 0; i < nh_npins; i++) + if (nh_pins[i].ref == ref && nh_pins[i].kind == kind) return nh_pins[i].obj; + return NULL; +} + +/* Releases every pin whose object only the table still holds. Releasing a + * pin can free objects that held other pinned objects, so it repeats until + * a pass frees nothing. */ +static void nh_sweep(void) { + bool freed = true; + while (freed) { + freed = false; + for (size_t i = 0; i < nh_npins;) { + size_t rc = *(size_t *)nh_pins[i].obj; + if (rc == 1) { + NhPin pin = nh_pins[i]; + nh_pins[i] = nh_pins[--nh_npins]; + nh_pin_release_obj(&pin); + nh_ref_release(pin.ref); + freed = true; + } else { + i++; + } + } + } + nh_sweep_at = nh_npins * 2 > 64 ? nh_npins * 2 : 64; +} + +/* ── runtime bytes V8 freed (any thread) ─────────────────────────────── */ + +static pthread_mutex_t nh_freed_lock = PTHREAD_MUTEX_INITIALIZER; +static void **nh_freed = NULL; +static size_t nh_nfreed = 0, nh_freed_cap = 0; + +void scr_ni_bytes_free(void *owner) { + pthread_mutex_lock(&nh_freed_lock); + if (nh_nfreed == nh_freed_cap) { + size_t cap = nh_freed_cap ? nh_freed_cap * 2 : 16; + void **grown = realloc(nh_freed, cap * sizeof *grown); + if (grown == NULL) nh_oom(); + nh_freed = grown; + nh_freed_cap = cap; + } + nh_freed[nh_nfreed++] = owner; + pthread_mutex_unlock(&nh_freed_lock); +} + +static void nh_drain_freed(void) { + for (;;) { + pthread_mutex_lock(&nh_freed_lock); + void *owner = nh_nfreed > 0 ? nh_freed[--nh_nfreed] : NULL; + pthread_mutex_unlock(&nh_freed_lock); + if (owner == NULL) return; + scr_bytes_release((ScrBytes *)owner); + } +} + +/* Housekeeping at every operation: bytes V8 freed, and the pin sweep once + * the table has grown. */ +static void nh_enter(void) { + if (nh_nfreed > 0) nh_drain_freed(); + if (nh_npins >= nh_sweep_at) nh_sweep(); +} + +/* ── refs ─────────────────────────────────────────────────────────────── */ + +static void *nh_ref_retain_v(void *h) { + ((ScrNodeRef *)h)->rc++; + return h; +} + +static void nh_ref_release(ScrNodeRef *ref) { + if (--ref->rc != 0) return; + if (ref->id < ni_refs_cap && ni_refs[ref->id] == ref) ni_refs[ref->id] = NULL; + if (!ni_stopped && scr_ni_bridge_loaded() != NULL) scr_ni_bridge_loaded()->drop(ref->id); + free(ref); +} + +static void nh_ref_release_v(void *h) { nh_ref_release((ScrNodeRef *)h); } + +/* The ref of bridge id `id` (+1), created on first sight. */ +static ScrNodeRef *nh_ref(uint32_t id, uint32_t flags) { + if (id >= ni_refs_cap) { + size_t cap = ni_refs_cap ? ni_refs_cap : 64; + while (cap <= id) cap *= 2; + ScrNodeRef **grown = realloc(ni_refs, cap * sizeof *grown); + if (grown == NULL) nh_oom(); + memset(grown + ni_refs_cap, 0, (cap - ni_refs_cap) * sizeof *grown); + ni_refs = grown; + ni_refs_cap = cap; + } + ScrNodeRef *ref = ni_refs[id]; + if (ref != NULL) { + ref->rc++; + return ref; + } + ref = malloc(sizeof *ref); + if (ref == NULL) nh_oom(); + ref->rc = 1; + ref->id = id; + ref->flags = flags; + ni_refs[id] = ref; + return ref; +} + +static const ScrDynHandleOps nh_ops; + +bool scr_ni_is_handle(const ScrDyn *value) { + return value != NULL && value->kind == SCR_DYN_HANDLE && + value->v.handle.tag == SCR_DYNH_NODE_ISLAND; +} + +static ScrNodeRef *nh_ref_of(const ScrDyn *value) { + return scr_ni_is_handle(value) ? (ScrNodeRef *)value->v.handle.ptr : NULL; +} + +/* ── awaiting island thenables ───────────────────────────────────────── */ + +typedef struct { + uint32_t token; + ScrPromise *promise; /* owned */ +} NhWaiter; + +static NhWaiter *nh_waiters = NULL; +static size_t nh_nwaiters = 0, nh_waiters_cap = 0; +static uint32_t nh_next_token = 1; + +static uint32_t nh_wait(ScrPromise *p) { + if (nh_nwaiters == nh_waiters_cap) { + size_t cap = nh_waiters_cap ? nh_waiters_cap * 2 : 8; + NhWaiter *grown = realloc(nh_waiters, cap * sizeof *grown); + if (grown == NULL) nh_oom(); + nh_waiters = grown; + nh_waiters_cap = cap; + } + uint32_t token = nh_next_token++; + if (nh_next_token == 0) nh_next_token = 1; + nh_waiters[nh_nwaiters++] = (NhWaiter){token, scr_promise_retain(p)}; + return token; +} + +/* ── V8 → runtime ─────────────────────────────────────────────────────── */ + +static ScrDyn *nh_from_node(const ScrNodeValue *v); +static void nh_throw_value(const ScrNodeValue *v); +static int nh_then(const ScrNodeValue *value, uint32_t token, ScrNodeValue *thrown); + +static bool nh_has_lone_surrogate(const char *p, size_t n) { + for (size_t i = 0; i + 1 < n; i++) + if ((unsigned char)p[i] == 0xED && (unsigned char)p[i + 1] >= 0xA0) return true; + return false; +} + +static ScrStr *nh_str(const char *data, size_t len) { + if (nh_has_lone_surrogate(data, len)) scr_wtf8_note(); + return scr_str_new(data, len); +} + +/* An island promise as a scriptc promise: one per V8 promise while the + * program holds it, settled by a reaction on the V8 side. */ +static ScrDyn *nh_promise_box(ScrNodeRef *ref) { + ScrPromise *p = nh_pinned_obj(ref, NH_PIN_PROMISE); + if (p == NULL) { + p = scr_promise_new(); + nh_pin(p, NH_PIN_PROMISE, ref); + ScrNodeValue value = {.tag = SCR_NV_HANDLE, .flags = ref->flags}; + value.u.handle = ref->id; + ScrNodeValue thrown; + uint32_t token = nh_wait(p); + if (nh_then(&value, token, &thrown) != 0) { + nh_throw_value(&thrown); + scr_promise_reject_pending(p); + } + scr_promise_release(p); /* the pin holds it */ + } + return scr_dyn_new_promise(p); +} + +/* Bytes from V8: an external root over the V8 store's memory, one per V8 + * view while the program holds it. */ +static ScrDyn *nh_bytes_box(const ScrNodeValue *v) { + ScrNodeRef *ref = nh_ref(v->u.bytes.handle, v->flags); + ScrBytes *b = nh_pinned_obj(ref, NH_PIN_BYTES); + if (b == NULL) { + b = scr_bytes_from_external(v->u.bytes.data, v->u.bytes.len); + b->elem = (ScrBytesElem)v->u.bytes.elem; + b->is_buffer = (v->flags & SCR_NVF_BUFFER) != 0; + b->is_data_view = (v->flags & SCR_NVF_DATAVIEW) != 0; + nh_pin(b, NH_PIN_BYTES, ref); + scr_bytes_release(b); /* the pin holds it */ + } + nh_ref_release(ref); + ScrDyn *d = scr_dyn_new_bytes(b); + d->buffer = b->is_buffer; + return d; +} + +/* A value the bridge answered, as a dyn value (+1). Copies string data at + * once: it is valid only until the next bridge entry. */ +static ScrDyn *nh_from_node(const ScrNodeValue *v) { + switch (v->tag) { + case SCR_NV_NULL: return scr_dyn_new_null(); + case SCR_NV_BOOL: return scr_dyn_new_bool(v->u.boolean != 0); + case SCR_NV_NUMBER: return scr_dyn_new_num(v->u.num); + case SCR_NV_STRING: { + ScrStr *s = nh_str(v->u.str.data, v->u.str.len); + ScrDyn *d = scr_dyn_new_str(s); + scr_str_release(s); + return d; + } + case SCR_NV_BIGINT: { + ScrStr *s = scr_str_new(v->u.str.data, v->u.str.len); + ScrBigInt *n = scr_bigint_parse(s); + scr_str_release(s); + if (n == NULL) return NULL; + ScrDyn *d = scr_dyn_new_bigint(n); + scr_bigint_release(n); + return d; + } + case SCR_NV_HANDLE: { + ScrNodeRef *ref = nh_ref(v->u.handle, v->flags); + ScrDyn *d = (ref->flags & SCR_NVF_PROMISE) ? nh_promise_box(ref) + : scr_dyn_new_handle(ref, SCR_DYNH_NODE_ISLAND); + nh_ref_release(ref); + return d; + } + case SCR_NV_BYTES: return nh_bytes_box(v); + default: return scr_dyn_undefined(); + } +} + +/* Leaves a thrown island value pending as a scriptc exception. An Error + * becomes a ScrError of the nearest intrinsic class with V8's name, + * message, stack and string code, pinned to the V8 error so that it is + * that error again if it crosses back. */ +static void nh_throw_value(const ScrNodeValue *v) { + if (v->tag == SCR_NV_HANDLE && (v->flags & SCR_NVF_ERROR)) { + ScrNodeRef *ref = nh_ref(v->u.handle, v->flags); + /* An island error the program already holds is that ScrError again. */ + ScrError *known = nh_pinned_obj(ref, NH_PIN_ERROR); + if (known != NULL) { + nh_ref_release(ref); + scr_throw_obj(scr_error_retain(known), &scr_error_retain_v, &scr_error_release_v, + scr_error_trace_arg()); + return; + } + ScrNodeErrorInfo info; + nh_bridge()->error_info(ref->id, &info); + ScrStr *message = nh_str(info.message ? info.message : "", info.message_len); + ScrError *e = scr_error_new((int)info.kind, message); + scr_str_release(message); + if (info.name != NULL) { + ScrStr *name = nh_str(info.name, info.name_len); + ScrStr *deflt = scr_error_default_name(e); + bool differs = deflt == NULL || deflt->len != name->len || + memcmp(deflt->data, name->data, name->len) != 0; + if (deflt != NULL) scr_str_release(deflt); + if (differs) { + scr_str_release(e->name); + e->name = name; + e->name_present = true; + } else { + scr_str_release(name); + } + } + if (info.stack != NULL) { + ScrStr *stack = nh_str(info.stack, info.stack_len); + scr_error_set_stack(e, stack); + scr_str_release(stack); + } + if (info.code != NULL) { + ScrStr *code = nh_str(info.code, info.code_len); + scr_error_set_code_value(e, code); + scr_str_release(code); + } + nh_pin(e, NH_PIN_ERROR, ref); + nh_ref_release(ref); + scr_throw_obj(e, &scr_error_retain_v, &scr_error_release_v, scr_error_trace_arg()); + return; + } + ScrDyn *d = nh_from_node(v); + if (d == NULL) return; /* a conversion error is pending */ + scr_dyn_throw(d); +} + +/* A V8 reaction settled an awaited thenable. */ +void scr_ni_settle_value(uint32_t token, int rejected, const ScrNodeValue *value) { + for (size_t i = 0; i < nh_nwaiters; i++) { + if (nh_waiters[i].token != token) continue; + ScrPromise *p = nh_waiters[i].promise; + nh_waiters[i] = nh_waiters[--nh_nwaiters]; + if (rejected) { + nh_throw_value(value); + scr_promise_reject_pending(p); + } else { + ScrDyn *d = nh_from_node(value); + if (d != NULL) + scr_promise_fulfill_ref(p, d, scr_dyn_retain_v, scr_dyn_release_v, scr_dyn_trace_v); + scr_promise_reject_pending(p); /* no-op unless a conversion error is pending */ + } + scr_promise_release(p); + return; + } +} + +/* ── runtime → V8 ─────────────────────────────────────────────────────── */ + +/* What a conversion into V8 keeps alive until the bridge call returns. */ +typedef struct { + void *items[16]; + void (*releases[16])(void *); + size_t n; + void **more; + void (**more_releases)(void *); + size_t nmore, more_cap; +} NhKeep; + +static void nh_keep(NhKeep *k, void *obj, void (*release)(void *)) { + if (k->n < 16) { + k->items[k->n] = obj; + k->releases[k->n++] = release; + return; + } + if (k->nmore == k->more_cap) { + size_t cap = k->more_cap ? k->more_cap * 2 : 16; + void **items = realloc(k->more, cap * sizeof *items); + void (**rels)(void *) = realloc(k->more_releases, cap * sizeof *rels); + if (items == NULL || rels == NULL) nh_oom(); + k->more = items; + k->more_releases = rels; + k->more_cap = cap; + } + k->more[k->nmore] = obj; + k->more_releases[k->nmore++] = release; +} + +static void nh_keep_done(NhKeep *k) { + for (size_t i = 0; i < k->n; i++) k->releases[i](k->items[i]); + for (size_t i = 0; i < k->nmore; i++) k->more_releases[i](k->more[i]); + free(k->more); + free(k->more_releases); +} + +static void nh_str_release_v(void *s) { scr_str_release((ScrStr *)s); } +static void nh_dyn_release_v(void *d) { scr_dyn_release((ScrDyn *)d); } + +SCR_COLD static bool nh_refuse(const char *what) { + char message[256]; + snprintf(message, sizeof message, + "scriptc: %s cannot cross into a Node island yet (scope-B NI-5 exports native values by reference)", + what); + scr_throw_error_msg(SCR_ERR_TYPE, message, strlen(message)); + return false; +} + +/* The values NI-5 exports by reference (closures, promises, typed refs, + * native objects and arrays). Until then: plain dyn objects and arrays + * cross as a JSON copy (the NI-4 shortcut), anything else is refused. */ +static bool nh_export_value(ScrDyn *v, ScrNodeValue *out, NhKeep *keep) { + if (v->kind == SCR_DYN_OBJ || v->kind == SCR_DYN_ARR) { + ScrStr *gap = scr_str_new("", 0); + ScrDyn *json = scr_json_stringify_replacer(v, scr_dyn_undefined(), gap); + scr_str_release(gap); + if (json == NULL || scr_exc_pending()) { + if (json != NULL) scr_dyn_release(json); + return false; + } + if (json->kind != SCR_DYN_STR) { + scr_dyn_release(json); + out->tag = SCR_NV_UNDEFINED; + return true; + } + nh_keep(keep, json, nh_dyn_release_v); + out->tag = SCR_NV_JSON; + out->u.str.data = json->v.str->data; + out->u.str.len = json->v.str->len; + return true; + } + switch (v->kind) { + case SCR_DYN_FUNC: return nh_refuse("a function"); + case SCR_DYN_PROMISE: return nh_refuse("a native promise"); + case SCR_DYN_SYMBOL: return nh_refuse("a native symbol"); + default: return nh_refuse("this native object"); + } +} + +/* The nearest intrinsic class V8 has (subclasses included). */ +static uint32_t nh_error_kind(const ScrError *e) { + for (int k = 1; k <= SCR_ERR_URI; k++) { + if (k == SCR_ERR_DOMEX) continue; + if (scr_error_vts[k].pre <= e->vt->pre && e->vt->pre <= scr_error_vts[k].post) + return (uint32_t)k; + } + return 0; +} + +/* A runtime value as a bridge in-value. False with an exception pending + * when it cannot cross. */ +static bool nh_to_node(ScrDyn *v, ScrNodeValue *out, NhKeep *keep) { + memset(out, 0, sizeof *out); + if (v == NULL) { + out->tag = SCR_NV_UNDEFINED; + return true; + } + switch (v->kind) { + case SCR_DYN_UNDEF: out->tag = SCR_NV_UNDEFINED; return true; + case SCR_DYN_NULL: out->tag = SCR_NV_NULL; return true; + case SCR_DYN_BOOL: out->tag = SCR_NV_BOOL; out->u.boolean = v->v.b ? 1 : 0; return true; + case SCR_DYN_NUM: out->tag = SCR_NV_NUMBER; out->u.num = v->v.num; return true; + case SCR_DYN_STR: + out->tag = SCR_NV_STRING; + out->u.str.data = v->v.str->data; + out->u.str.len = v->v.str->len; + return true; + case SCR_DYN_BIGINT: { + ScrStr *text = scr_bigint_to_string(v->v.bigint, 10); + if (text == NULL) return false; + nh_keep(keep, text, nh_str_release_v); + out->tag = SCR_NV_BIGINT; + out->u.str.data = text->data; + out->u.str.len = text->len; + return true; + } + case SCR_DYN_HANDLE: + if (v->v.handle.tag == SCR_DYNH_NODE_ISLAND) { + ScrNodeRef *ref = v->v.handle.ptr; + out->tag = SCR_NV_HANDLE; + out->flags = ref->flags; + out->u.handle = ref->id; + return true; + } + return nh_export_value(v, out, keep); + case SCR_DYN_PROMISE: { + ScrNodeRef *ref = nh_pinned(v->v.promise, NH_PIN_PROMISE); + if (ref == NULL) return nh_export_value(v, out, keep); + out->tag = SCR_NV_HANDLE; + out->flags = ref->flags; + out->u.handle = ref->id; + return true; + } + case SCR_DYN_BYTES: { + ScrBytes *b = v->v.bytes; + ScrBytes *root = b->backing != NULL ? b->backing : b; + ScrNodeRef *ref = nh_pinned(b, NH_PIN_BYTES); + out->tag = SCR_NV_BYTES; + out->u.bytes.elem = (uint32_t)b->elem; + out->u.bytes.len = b->len; + out->u.bytes.data = b->data; + if (b->is_buffer || v->buffer) out->flags |= SCR_NVF_BUFFER; + if (b->is_data_view) out->flags |= SCR_NVF_DATAVIEW; + if (ref != NULL) { + out->u.bytes.handle = ref->id; /* the same V8 view */ + return true; + } + if (root->shared != NULL) return nh_refuse("shared memory"); + /* V8 frees the store through host->bytes_free, which releases this. */ + out->u.bytes.owner = scr_bytes_retain(root); + return true; + } + default: break; + } + ScrError *e = scr_errdyn_err_of(v); + if (e != NULL) { + ScrNodeRef *ref = nh_pinned(e, NH_PIN_ERROR); + if (ref != NULL) { + scr_error_release(e); + out->tag = SCR_NV_HANDLE; + out->flags = ref->flags; + out->u.handle = ref->id; + return true; + } + ScrStr *name = e->name != NULL ? scr_str_retain(e->name) : scr_error_default_name(e); + ScrStr *message = e->message != NULL ? scr_str_retain(e->message) : scr_str_new("", 0); + ScrStr *stack = scr_error_stack(e); + nh_keep(keep, name, nh_str_release_v); + nh_keep(keep, message, nh_str_release_v); + if (stack != NULL) nh_keep(keep, stack, nh_str_release_v); + out->tag = SCR_NV_ERROR; + out->u.error.kind = nh_error_kind(e); + out->u.error.name = name->data; + out->u.error.name_len = name->len; + out->u.error.message = message->data; + out->u.error.message_len = message->len; + if (stack != NULL) { + out->u.error.stack = stack->data; + out->u.error.stack_len = stack->len; + } + if (e->code != NULL) { + ScrStr *code = scr_str_retain(e->code); + nh_keep(keep, code, nh_str_release_v); + out->u.error.code = code->data; + out->u.error.code_len = code->len; + } + scr_error_release(e); + return true; + } + return nh_export_value(v, out, keep); +} + +/* ── operations on Node's stack ──────────────────────────────────────── */ + +enum { + NH_LOAD, NH_GET, NH_SET, NH_HAS, NH_DEL, NH_INVOKE, NH_INVOKE_METHOD, NH_CONSTRUCT, + NH_INSTANCEOF, NH_KEYS, NH_THEN, +}; + +typedef struct { + int op; + uint32_t h; + const char *path; + int mode; + const ScrNodeValue *a, *b; + const ScrNodeValue *args; + size_t argc; + uint32_t token; + ScrNodeValue *out; + int threw; +} NhCall; + +static void nh_call_job(void *p) { + NhCall *c = p; + const ScrNodeBridge *b = scr_ni_bridge_loaded(); + switch (c->op) { + case NH_LOAD: c->threw = b->load(c->path, c->mode, c->out); break; + case NH_GET: c->threw = b->get(c->h, c->a, c->out); break; + case NH_SET: c->threw = b->set(c->h, c->a, c->b, c->out); break; + case NH_HAS: c->threw = b->has(c->h, c->a, c->out); break; + case NH_DEL: c->threw = b->del(c->h, c->a, c->out); break; + case NH_INVOKE: c->threw = b->invoke(c->h, c->a, c->args, c->argc, c->out); break; + case NH_INVOKE_METHOD: c->threw = b->invoke_method(c->h, c->a, c->args, c->argc, c->out); break; + case NH_CONSTRUCT: c->threw = b->construct(c->h, c->args, c->argc, c->out); break; + case NH_INSTANCEOF: c->threw = b->instance_of(c->a, c->h, c->out); break; + case NH_KEYS: c->threw = b->own_keys(c->h, c->mode, c->out); break; + case NH_THEN: c->threw = b->then(c->a, c->token, c->out); break; + } +} + +static int nh_run(NhCall *c) { + scr_ni_on_node_stack(nh_call_job, c); + return c->threw; +} + +static int nh_then(const ScrNodeValue *value, uint32_t token, ScrNodeValue *thrown) { + NhCall c = {.op = NH_THEN, .a = value, .token = token, .out = thrown}; + return nh_run(&c); +} + +/* The outcome of a call as a dyn value (+1), or NULL with the thrown + * value pending. */ +static ScrDyn *nh_outcome(NhCall *c) { + if (c->threw) { + nh_throw_value(c->out); + return NULL; + } + return nh_from_node(c->out); +} + +/* The receiver of a property operation as an in-value: an island handle + * names itself; anything else is a TypeError (the compiler routes only + * island values here). */ +static bool nh_receiver(ScrDyn *obj, uint32_t *id, const char *what) { + ScrNodeRef *ref = nh_ref_of(obj); + if (ref == NULL) { + char message[160]; + snprintf(message, sizeof message, "scriptc: %s needs a Node island value", what); + scr_throw_error_msg(SCR_ERR_TYPE, message, strlen(message)); + return false; + } + *id = ref->id; + return true; +} + +/* ── the runtime API (scr_node_island.h) ─────────────────────────────── */ + +static ScrDyn *nh_load(ScrStr *path, int esm) { + nh_bridge(); + nh_enter(); + char *p = scr_ni_module_path(path); + ScrNodeValue out; + NhCall c = {.op = NH_LOAD, .path = p, .mode = esm, .out = &out}; + nh_run(&c); + free(p); + return nh_outcome(&c); +} + +ScrDyn *scr_ni_require(ScrStr *path) { return nh_load(path, 0); } + +ScrPromise *scr_ni_import(ScrStr *path) { + ScrDyn *d = nh_load(path, 1); + if (d == NULL) { + ScrPromise *p = scr_promise_new(); + scr_promise_reject_pending(p); + return p; + } + ScrPromise *p = scr_dyn_promise_of(d); + if (p != NULL) { + p = scr_promise_retain(p); + } else { + p = scr_promise_new(); + scr_promise_fulfill_ref(p, scr_dyn_retain(d), scr_dyn_retain_v, scr_dyn_release_v, + scr_dyn_trace_v); + } + scr_dyn_release(d); + return p; +} + +ScrDyn *scr_ni_get(ScrDyn *obj, ScrDyn *key) { + uint32_t id; + if (!nh_receiver(obj, &id, "a property read")) return NULL; + nh_enter(); + NhKeep keep = {0}; + ScrNodeValue k, out; + if (!nh_to_node(key, &k, &keep)) { + nh_keep_done(&keep); + return NULL; + } + NhCall c = {.op = NH_GET, .h = id, .a = &k, .out = &out}; + nh_run(&c); + nh_keep_done(&keep); + return nh_outcome(&c); +} + +ScrDyn *scr_ni_get_str(ScrDyn *obj, const char *key, size_t len) { + uint32_t id; + if (!nh_receiver(obj, &id, "a property read")) return NULL; + nh_enter(); + ScrNodeValue k = {.tag = SCR_NV_STRING}, out; + k.u.str.data = key; + k.u.str.len = len; + NhCall c = {.op = NH_GET, .h = id, .a = &k, .out = &out}; + nh_run(&c); + return nh_outcome(&c); +} + +static bool nh_set(uint32_t id, const ScrNodeValue *k, ScrDyn *value) { + NhKeep keep = {0}; + ScrNodeValue v, out; + if (!nh_to_node(value, &v, &keep)) { + nh_keep_done(&keep); + return false; + } + NhCall c = {.op = NH_SET, .h = id, .a = k, .b = &v, .out = &out}; + nh_run(&c); + nh_keep_done(&keep); + if (c.threw) { + nh_throw_value(&out); + return false; + } + return true; +} + +bool scr_ni_set(ScrDyn *obj, ScrDyn *key, ScrDyn *value) { + uint32_t id; + if (!nh_receiver(obj, &id, "a property write")) return false; + nh_enter(); + NhKeep keep = {0}; + ScrNodeValue k; + if (!nh_to_node(key, &k, &keep)) { + nh_keep_done(&keep); + return false; + } + bool ok = nh_set(id, &k, value); + nh_keep_done(&keep); + return ok; +} + +bool scr_ni_set_str(ScrDyn *obj, const char *key, size_t len, ScrDyn *value) { + uint32_t id; + if (!nh_receiver(obj, &id, "a property write")) return false; + nh_enter(); + ScrNodeValue k = {.tag = SCR_NV_STRING}; + k.u.str.data = key; + k.u.str.len = len; + return nh_set(id, &k, value); +} + +static int nh_bool_op(int op, ScrDyn *obj, ScrDyn *key, const char *what) { + uint32_t id; + if (!nh_receiver(obj, &id, what)) return -1; + nh_enter(); + NhKeep keep = {0}; + ScrNodeValue k, out; + if (!nh_to_node(key, &k, &keep)) { + nh_keep_done(&keep); + return -1; + } + NhCall c = {.op = op, .h = id, .a = &k, .out = &out}; + nh_run(&c); + nh_keep_done(&keep); + if (c.threw) { + nh_throw_value(&out); + return -1; + } + return out.tag == SCR_NV_BOOL && out.u.boolean ? 1 : 0; +} + +int scr_ni_has(ScrDyn *obj, ScrDyn *key) { return nh_bool_op(NH_HAS, obj, key, "the 'in' operator"); } + +int scr_ni_delete(ScrDyn *obj, ScrDyn *key) { return nh_bool_op(NH_DEL, obj, key, "delete"); } + +/* Converts call arguments; NULL with an exception pending on failure. */ +static ScrNodeValue *nh_args(ScrDyn *const *args, size_t argc, ScrNodeValue *small, NhKeep *keep) { + ScrNodeValue *values = argc <= 8 ? small : malloc(argc * sizeof *values); + if (values == NULL) nh_oom(); + for (size_t i = 0; i < argc; i++) { + if (!nh_to_node(args[i], &values[i], keep)) { + if (values != small) free(values); + return NULL; + } + } + return values; +} + +static ScrDyn *nh_call(int op, uint32_t id, ScrDyn *this_, const ScrNodeValue *key, + ScrDyn *const *args, size_t argc) { + nh_enter(); + NhKeep keep = {0}; + ScrNodeValue small[8], recv, out; + ScrNodeValue *values = nh_args(args, argc, small, &keep); + if (values == NULL) { + nh_keep_done(&keep); + return NULL; + } + const ScrNodeValue *a = key; + if (op == NH_INVOKE) { + if (!nh_to_node(this_, &recv, &keep)) { + if (values != small) free(values); + nh_keep_done(&keep); + return NULL; + } + a = &recv; + } + NhCall c = {.op = op, .h = id, .a = a, .args = values, .argc = argc, .out = &out}; + nh_run(&c); + if (values != small) free(values); + nh_keep_done(&keep); + return nh_outcome(&c); +} + +ScrDyn *scr_ni_call(ScrDyn *fn, ScrDyn *this_, ScrDyn *const *args, size_t argc) { + uint32_t id; + if (!nh_receiver(fn, &id, "a call")) return NULL; + return nh_call(NH_INVOKE, id, this_, NULL, args, argc); +} + +ScrDyn *scr_ni_call_method(ScrDyn *obj, const char *key, size_t len, ScrDyn *const *args, + size_t argc) { + uint32_t id; + if (!nh_receiver(obj, &id, "a method call")) return NULL; + ScrNodeValue k = {.tag = SCR_NV_STRING}; + k.u.str.data = key; + k.u.str.len = len; + return nh_call(NH_INVOKE_METHOD, id, NULL, &k, args, argc); +} + +ScrDyn *scr_ni_construct(ScrDyn *ctor, ScrDyn *const *args, size_t argc) { + uint32_t id; + if (!nh_receiver(ctor, &id, "new")) return NULL; + return nh_call(NH_CONSTRUCT, id, NULL, NULL, args, argc); +} + +ScrStr *scr_ni_typeof(ScrDyn *value) { + ScrNodeRef *ref = nh_ref_of(value); + if (ref == NULL) return scr_dyn_typeof(value); + const char *s = (ref->flags & SCR_NVF_FUNCTION) ? "function" + : (ref->flags & SCR_NVF_SYMBOL) ? "symbol" + : "object"; + return scr_str_new(s, strlen(s)); +} + +int scr_ni_instanceof(ScrDyn *value, ScrDyn *ctor) { + uint32_t id; + if (!nh_receiver(ctor, &id, "instanceof")) return -1; + nh_enter(); + NhKeep keep = {0}; + ScrNodeValue v, out; + if (!nh_to_node(value, &v, &keep)) { + nh_keep_done(&keep); + return -1; + } + NhCall c = {.op = NH_INSTANCEOF, .h = id, .a = &v, .out = &out}; + nh_run(&c); + nh_keep_done(&keep); + if (c.threw) { + nh_throw_value(&out); + return -1; + } + return out.tag == SCR_NV_BOOL && out.u.boolean ? 1 : 0; +} + +ScrDyn *scr_ni_own_keys(ScrDyn *obj, int mode) { + uint32_t id; + if (!nh_receiver(obj, &id, "Object.keys")) return NULL; + nh_enter(); + ScrNodeValue out; + NhCall c = {.op = NH_KEYS, .h = id, .mode = mode, .out = &out}; + nh_run(&c); + ScrDyn *array = nh_outcome(&c); + if (array == NULL) return NULL; + /* A fresh V8 array of keys: read its elements, then let it go. */ + ScrDyn *keys = scr_dyn_new_arr(); + ScrDyn *length = scr_ni_get_str(array, "length", 6); + double n = length != NULL && length->kind == SCR_DYN_NUM ? length->v.num : 0; + if (length != NULL) scr_dyn_release(length); + for (double i = 0; i < n; i++) { + ScrDyn *index = scr_dyn_new_num(i); + ScrDyn *key = scr_ni_get(array, index); + scr_dyn_release(index); + if (key == NULL) { + scr_dyn_release(keys); + scr_dyn_release(array); + return NULL; + } + scr_dyn_arr_push(keys, key); + } + scr_dyn_release(array); + return keys; +} + +ScrPromise *scr_ni_await(ScrDyn *value) { + ScrPromise *boxed = value != NULL ? scr_dyn_promise_of(value) : NULL; + if (boxed != NULL) return scr_promise_retain(boxed); + ScrNodeRef *ref = nh_ref_of(value); + ScrPromise *p = scr_promise_new(); + if (ref == NULL) { + scr_promise_fulfill_ref(p, scr_dyn_retain(value != NULL ? value : scr_dyn_undefined()), + scr_dyn_retain_v, scr_dyn_release_v, scr_dyn_trace_v); + return p; + } + nh_enter(); + ScrNodeValue v = {.tag = SCR_NV_HANDLE, .flags = ref->flags}, thrown; + v.u.handle = ref->id; + uint32_t token = nh_wait(p); + if (nh_then(&v, token, &thrown) != 0) { + nh_throw_value(&thrown); + scr_promise_reject_pending(p); + } + return p; +} + +bool scr_ni_is_function(const ScrDyn *value) { + ScrNodeRef *ref = nh_ref_of(value); + return ref != NULL && (ref->flags & SCR_NVF_FUNCTION) != 0; +} + +/* Node's util.inspect, loaded once and held for the program's life. */ +static ScrDyn *nh_inspect_fn = NULL; + +ScrStr *scr_ni_inspect(const ScrDyn *value, double recurse, double depth) { + ScrNodeRef *ref = nh_ref_of(value); + if (ref == NULL) return scr_str_new("", 0); + if (nh_inspect_fn == NULL) { + ScrStr *util = scr_str_new("node:util", 9); + ScrDyn *module = nh_load(util, 0); + scr_str_release(util); + if (module == NULL) return scr_str_new("", 0); + nh_inspect_fn = scr_ni_get_str(module, "inspect", 7); + scr_dyn_release(module); + if (nh_inspect_fn == NULL) return scr_str_new("", 0); + } + /* The remaining depth, as util.inspect counts it (null: unlimited). */ + char options[64]; + double left = depth - recurse; + if (left != left || left > 1e9) + snprintf(options, sizeof options, "{\"depth\":null}"); + else + snprintf(options, sizeof options, "{\"depth\":%.0f}", left < 0 ? 0 : left); + ScrNodeValue args[2], out; + memset(args, 0, sizeof args); + args[0].tag = SCR_NV_HANDLE; + args[0].flags = ref->flags; + args[0].u.handle = ref->id; + args[1].tag = SCR_NV_JSON; + args[1].u.str.data = options; + args[1].u.str.len = strlen(options); + ScrNodeRef *fn = nh_ref_of(nh_inspect_fn); + nh_enter(); + NhCall c = {.op = NH_INVOKE, .h = fn->id, .a = NULL, .args = args, .argc = 2, .out = &out}; + nh_run(&c); + ScrDyn *text = nh_outcome(&c); + if (text == NULL) return scr_str_new("", 0); + ScrStr *s = text->kind == SCR_DYN_STR ? scr_str_retain(text->v.str) : scr_str_new("", 0); + scr_dyn_release(text); + return s; +} + +uint32_t scr_ni_live_handles(void) { + const ScrNodeBridge *b = scr_ni_bridge_loaded(); + if (b == NULL) return 0; + nh_drain_freed(); + nh_sweep(); + return b->live_handles(); +} + +void scr_ni_stop(void) { ni_stopped = true; } + +/* ── the handle tag (dynamic code over island values) ────────────────── */ + +static ScrDyn *nh_op_get(void *h, const char *key, size_t key_len) { + ScrDyn *box = scr_dyn_new_handle(h, SCR_DYNH_NODE_ISLAND); + ScrDyn *result = scr_ni_get_str(box, key, key_len); + scr_dyn_release(box); + return result; +} + +static bool nh_op_set(void *h, const char *key, size_t key_len, const ScrDyn *value) { + ScrDyn *box = scr_dyn_new_handle(h, SCR_DYNH_NODE_ISLAND); + scr_ni_set_str(box, key, key_len, (ScrDyn *)value); + scr_dyn_release(box); + return true; /* handled; a throw stays pending */ +} + +/* obj.m(...) from dynamic code: the member read, then the call with obj + * as the receiver; a member that is not a function throws Node's TypeError + * with the call site's text (`what`), as V8 words it. */ +static ScrDyn *nh_op_invoke(void *h, ScrDyn *self, const char *method, ScrDyn *const *args, + size_t argc, const char *what) { + (void)h; + ScrDyn *fn = scr_ni_get_str(self, method, strlen(method)); + if (fn == NULL) return NULL; + if (!scr_ni_is_function(fn)) { + scr_dyn_release(fn); + char message[512]; + snprintf(message, sizeof message, "%s is not a function", what != NULL ? what : method); + scr_throw_error_msg(SCR_ERR_TYPE, message, strlen(message)); + return NULL; + } + ScrDyn *result = scr_ni_call(fn, self, args, argc); + scr_dyn_release(fn); + return result; +} + +static ScrDyn *nh_op_symbol_get(void *h, const ScrDyn *key) { + if (!scr_ni_is_handle(key)) return NULL; + ScrDyn *box = scr_dyn_new_handle(h, SCR_DYNH_NODE_ISLAND); + ScrDyn *result = scr_ni_get(box, (ScrDyn *)key); + scr_dyn_release(box); + return result; +} + +static const ScrDynHandleOps nh_ops = { + .cls = "Object", + .retain = nh_ref_retain_v, + .release = nh_ref_release_v, + .invoke = nh_op_invoke, + .get = nh_op_get, + .set = nh_op_set, + .symbol_get = nh_op_symbol_get, +}; + +/* At exit: what the unit itself still holds (pins, unsettled waiters, + * util.inspect), so the runtime's reference audit sees only the program's + * own objects. */ +static void nh_cleanup(void) { + nh_drain_freed(); + if (nh_inspect_fn != NULL) { + scr_dyn_release(nh_inspect_fn); + nh_inspect_fn = NULL; + } + while (nh_npins > 0) { + NhPin pin = nh_pins[--nh_npins]; + nh_pin_release_obj(&pin); + nh_ref_release(pin.ref); + } + while (nh_nwaiters > 0) scr_promise_release(nh_waiters[--nh_nwaiters].promise); + free(nh_pins); + free(nh_waiters); + nh_pins = NULL; + nh_waiters = NULL; + nh_pins_cap = nh_waiters_cap = 0; +} + +void scr_ni_handles_install(void) { + scr_dyn_handle_install(SCR_DYNH_NODE_ISLAND, &nh_ops); + scr_atexit(nh_cleanup); +} + +/* ── NI-4a: libCall entry points of compiled island code ───────────────── + * The compiler (frontend/lowering/lower-node-island.ts) lowers member + * operations on island values to `nodeIsland.*` libCalls, which pass + * strings as ScrStr and argument lists as one dyn array; these adapt them + * to the scr_ni_* API above. Borrowed arguments, +1 results, NULL (or a + * pending exception) when JavaScript threw. */ + +static size_t ni_libcall_args(ScrDyn *args, ScrDyn *const **items) { + if (args == NULL || args->kind != SCR_DYN_ARR) { + *items = NULL; + return 0; + } + *items = args->v.arr.items; + return args->v.arr.len; +} + +ScrDyn *scr_node_island_get(ScrDyn *obj, ScrStr *key) { + return scr_ni_get_str(obj, key->data, key->len); +} + +void scr_node_island_set(ScrDyn *obj, ScrStr *key, ScrDyn *value) { + (void)scr_ni_set_str(obj, key->data, key->len, value); +} + +void scr_node_island_set_key(ScrDyn *obj, ScrDyn *key, ScrDyn *value) { + (void)scr_ni_set(obj, key, value); +} + +ScrDyn *scr_node_island_invoke(ScrDyn *obj, ScrStr *key, ScrDyn *args) { + ScrDyn *const *items; + size_t argc = ni_libcall_args(args, &items); + return scr_ni_call_method(obj, key->data, key->len, items, argc); +} + +ScrDyn *scr_node_island_apply(ScrDyn *fn, ScrDyn *args) { + ScrDyn *const *items; + size_t argc = ni_libcall_args(args, &items); + return scr_ni_call(fn, NULL, items, argc); +} + +ScrDyn *scr_node_island_apply_to(ScrDyn *fn, ScrDyn *this_, ScrDyn *args) { + ScrDyn *const *items; + size_t argc = ni_libcall_args(args, &items); + return scr_ni_call(fn, this_, items, argc); +} + +ScrDyn *scr_node_island_construct(ScrDyn *ctor, ScrDyn *args) { + ScrDyn *const *items; + size_t argc = ni_libcall_args(args, &items); + return scr_ni_construct(ctor, items, argc); +} + +bool scr_node_island_instanceof(ScrDyn *value, ScrDyn *ctor) { + return scr_ni_instanceof(value, ctor) == 1; +} + +#endif /* !_WIN32 && !__wasi__ */ diff --git a/packages/runtime/src/scr_node_island_host.cjs b/packages/runtime/src/scr_node_island_host.cjs index 90c6187ab..bf3ec2b74 100644 --- a/packages/runtime/src/scr_node_island_host.cjs +++ b/packages/runtime/src/scr_node_island_host.cjs @@ -4,16 +4,19 @@ * scr_node_island_host.h embeds this file as a C string; regenerate it * with `node scripts/node-island-host.mjs` after every change. * - * The libnode bridge (scr_node_bridge.cc) installs a native `settle` - * function as globalThis.__scriptcNodeIslandHost before this script runs, - * and takes the dispatcher this script leaves as - * globalThis.__scriptcNodeIsland afterwards; it deletes both globals. + * The libnode bridge (scr_node_bridge.cc) installs its natives (`settle`, + * and for the hosted loop `run` and `listen`) as + * globalThis.__scriptcNodeIslandHost before this script runs, and takes the + * dispatcher this script leaves as globalThis.__scriptcNodeIsland + * afterwards (`call` and the hosted-loop helpers); it deletes both globals. * * A copy of the call semantics lives in packages/node/index.js, which runs * them in-process under Node: both must produce the same results, errors * and messages. Outcomes are "R" (returned; "R" alone is undefined) * or "T" (threw), the encoding of scr_node_bridge.h. */ -const { createRequire, registerHooks } = require("node:module"); +const { createRequire, isBuiltin, registerHooks } = require("node:module"); +const { constants: osConstants } = require("node:os"); +const timers = require("node:timers"); const nativeHost = globalThis.__scriptcNodeIslandHost; @@ -154,6 +157,215 @@ function call(id, sync, file, specifier, name, argsJson) { return undefined; } +// ── island modules (scope-B NI-4) ────────────────────────────────────── +// Island code loaded by absolute path: require (CommonJS, and require(esm) +// for ES modules without top-level await) answers the exports; import +// answers a promise of the namespace. +const { pathToFileURL } = require("node:url"); + +function load(path, esm) { + // A built-in module's name, with or without the node: prefix. + if (isBuiltin(path)) return require(path); + if (esm) return import(pathToFileURL(path).href); + return createRequire(path)(path); +} + +// ── the hosted loop (scope-B NI-3) ───────────────────────────────────── +// The runtime's ticks, timers and immediates are Node's, created here with +// Node's own timers (captured now, so island code replacing the globals +// does not reach them) and run through the native `run`. Handles name the +// Node objects until the runtime clears them. + +const JOB_TICK = 2; +const JOB_TIMER = 3; +const JOB_IMMEDIATE = 4; +const handles = new Map(); +let nextHandle = 1; + +function onTick(token) { + nativeHost.run(JOB_TICK, token); +} + +function onTimer(token) { + nativeHost.run(JOB_TIMER, token); +} + +function onImmediate(handle, token) { + handles.delete(handle); + nativeHost.run(JOB_IMMEDIATE, token); +} + +function setTimer(delay, repeat, ref, token) { + let timer; + if (repeat > 0 && delay === repeat) { + timer = timers.setInterval(onTimer, repeat, token); + } else { + timer = timers.setTimeout(onTimer, delay, token); + // An interval migrated mid-period: its first run comes after the rest + // of the period, later ones every `repeat` (Node re-arms a Timeout + // whose _repeat is set, exactly as for setInterval). + if (repeat > 0) timer._repeat = repeat; + } + if (!ref) timer.unref(); + const handle = nextHandle++; + handles.set(handle, timer); + return handle; +} + +function clearTimer(handle) { + const timer = handles.get(handle); + if (timer === undefined) return; + handles.delete(handle); + timers.clearTimeout(timer); +} + +function refTimer(handle, ref) { + const timer = handles.get(handle); + if (timer !== undefined) { + if (ref) timer.ref(); + else timer.unref(); + } +} + +function refreshTimer(handle) { + const timer = handles.get(handle); + if (timer !== undefined) timer.refresh(); +} + +function setImmediateJob(ref, token) { + const handle = nextHandle++; + const immediate = timers.setImmediate(onImmediate, handle, token); + if (!ref) immediate.unref(); + handles.set(handle, immediate); + return handle; +} + +function clearImmediateJob(handle) { + const immediate = handles.get(handle); + if (immediate === undefined) return; + handles.delete(handle); + timers.clearImmediate(immediate); +} + +function refImmediate(handle, ref) { + refTimer(handle, ref); +} + +function nextTick(token) { + process.nextTick(onTick, token); +} + +// ── process state (scope-B NI-3) ───────────────────────────────────────── +// Signal and 'exit' listeners of island code join the runtime's registries +// at their registration position, so one registry orders both sides and +// the runtime keeps owning every signal disposition: Node's own handlers, +// which would hand a signal to libuv and reset it to SIG_DFL when the last +// listener goes, are taken off. The listeners stay on `process` (listeners +// and listenerCount answer as in Node); the runtime calls them through +// `emit`, and Node's own 'exit' emission is left to the runtime's exit +// sequence. + +const EVENT_SIGNAL = 1; +const EVENT_EXIT = 2; +const signalNames = new Set(Object.keys(osConstants.signals)); +const islandListeners = []; +let nextListener = 1; + +function eventKind(event) { + if (event === "exit") return EVENT_EXIT; + return typeof event === "string" && signalNames.has(event) ? EVENT_SIGNAL : 0; +} + +for (const [event, name] of [ + ["newListener", "startListeningIfSignal"], + ["removeListener", "stopListeningIfSignal"], +]) { + for (const listener of process.listeners(event)) { + if (listener.name === name) process.removeListener(event, listener); + } +} + +process.on("newListener", (event, listener) => { + const kind = eventKind(event); + if (kind === 0) return; + const id = nextListener++; + islandListeners.push({ event, listener, id, raw: undefined }); + nativeHost.listen(kind, event, id, 1, 0); +}); + +function forget(i, kind) { + const entry = islandListeners[i]; + islandListeners.splice(i, 1); + nativeHost.listen(kind, entry.event, entry.id, 0, 0); +} + +process.on("removeListener", (event, listener) => { + const kind = eventKind(event); + if (kind === 0) return; + // Node reports the removed registration as the raw listener it installed (a + // once wrapper removing itself reports the wrapper) or as the listener + // itself. The raw listener each entry recorded separates a once + // registration from an `on` registration of the same function: the removed + // one is the entry whose raw listener is no longer on `process`. + const live = process.rawListeners(event); + let same = -1; + for (let i = islandListeners.length - 1; i >= 0; i--) { + const entry = islandListeners[i]; + if (entry.event !== event) continue; + if (entry.listener !== listener && entry.raw !== listener) continue; + if (entry.raw !== undefined && !live.includes(entry.raw)) { + forget(i, kind); + return; + } + if (same < 0) same = i; + } + // Duplicate registrations of one function share a raw listener; Node + // removes the most recent, which is the last matching entry. + if (same >= 0) forget(same, kind); +}); + +// The raw listener is the function handed to the emitter (a once wrapper +// for process.once, which registers through process.on): tag each entry +// with it as its registration returns. Node emits newListener before it +// installs the listener, so the entry learns its raw only afterwards. +for (const name of ["on", "addListener", "prependListener"]) { + const add = process[name]; + process[name] = function addListener(event, listener) { + const start = islandListeners.length; + const result = Reflect.apply(add, this, arguments); + if (islandListeners.length > start) { + const registered = listener.listener ?? listener; + for (let i = start; i < islandListeners.length; i++) { + const entry = islandListeners[i]; + if (entry.event === event && entry.listener === registered && entry.raw === undefined) { + entry.raw = listener; + break; + } + } + } + return result; + }; +} + +function emit(kind, id, name, arg) { + const entry = islandListeners.find((e) => e.id === id); + if (entry === undefined) return; + // The registered function: the listener itself, or its once wrapper, + // which removes itself before calling it. + const registered = + entry.raw ?? + process.rawListeners(entry.event).find((raw) => (raw.listener ?? raw) === entry.listener); + if (registered === undefined) return; + if (kind === EVENT_SIGNAL) Reflect.apply(registered, process, [name, arg]); + else Reflect.apply(registered, process, [arg]); +} + +const nodeEmit = process.emit; +process.emit = function emit(event, ...args) { + if (event === "exit") return this.listenerCount("exit") > 0; + return Reflect.apply(nodeEmit, this, [event, ...args]); +}; + // Writes reach the shared stdout and stderr before control returns to the // program, as the program's own writes do. for (const stream of [process.stdout, process.stderr]) { @@ -161,13 +373,6 @@ for (const stream of [process.stdout, process.stderr]) { stream._handle.setBlocking(true); } -// The program ends with `code` (the bridge then runs Node's loop to its -// end, which emits 'beforeExit' and 'exit'): Node's process shares it, as -// it shares one process under Node. -function finish(code) { - if (code !== 0) process.exitCode = code; -} - // process.execPath is this program, not a node executable: a fork without // an execPath of its own would start the program again. const childProcess = require("node:child_process"); @@ -185,4 +390,16 @@ childProcess.fork = function fork(modulePath, args, options) { }; require("node:module").syncBuiltinESMExports(); -globalThis.__scriptcNodeIsland = { call, finish }; +globalThis.__scriptcNodeIsland = { + call, + setTimer, + clearTimer, + refTimer, + refreshTimer, + setImmediate: setImmediateJob, + clearImmediate: clearImmediateJob, + refImmediate, + nextTick, + emit, + load, +}; diff --git a/packages/runtime/src/scr_node_island_host.h b/packages/runtime/src/scr_node_island_host.h index ed78c8951..04d96aeb1 100644 --- a/packages/runtime/src/scr_node_island_host.h +++ b/packages/runtime/src/scr_node_island_host.h @@ -10,16 +10,19 @@ static const char scr_node_island_host_js[] = " * scr_node_island_host.h embeds this file as a C string; regenerate it\012" " * with `node scripts/node-island-host.mjs` after every change.\012" " *\012" - " * The libnode bridge (scr_node_bridge.cc) installs a native `settle`\012" - " * function as globalThis.__scriptcNodeIslandHost before this script runs,\012" - " * and takes the dispatcher this script leaves as\012" - " * globalThis.__scriptcNodeIsland afterwards; it deletes both globals.\012" + " * The libnode bridge (scr_node_bridge.cc) installs its natives (`settle`,\012" + " * and for the hosted loop `run` and `listen`) as\012" + " * globalThis.__scriptcNodeIslandHost before this script runs, and takes the\012" + " * dispatcher this script leaves as globalThis.__scriptcNodeIsland\012" + " * afterwards (`call` and the hosted-loop helpers); it deletes both globals.\012" " *\012" " * A copy of the call semantics lives in packages/node/index.js, which runs\012" " * them in-process under Node: both must produce the same results, errors\012" " * and messages. Outcomes are \"R\" (returned; \"R\" alone is undefined)\012" " * or \"T\" (threw), the encoding of scr_node_bridge.h. */\012" - "const { createRequire, registerHooks } = require(\"node:module\");\012" + "const { createRequire, isBuiltin, registerHooks } = require(\"node:module\");\012" + "const { constants: osConstants } = require(\"node:os\");\012" + "const timers = require(\"node:timers\");\012" "\012" "const nativeHost = globalThis.__scriptcNodeIslandHost;\012" "\012" @@ -160,6 +163,215 @@ static const char scr_node_island_host_js[] = " return undefined;\012" "}\012" "\012" + "// \342\224\200\342\224\200 island modules (scope-B NI-4) \342\224\200\342\224\200\342\224\200\342\224\200\342\224\200\342\224\200\342\224\200\342\224\200\342\224\200\342\224\200\342\224\200\342\224\200\342\224\200\342\224\200\342\224\200\342\224\200\342\224\200\342\224\200\342\224\200\342\224\200\342\224\200\342\224\200\342\224\200\342\224\200\342\224\200\342\224\200\342\224\200\342\224\200\342\224\200\342\224\200\342\224\200\342\224\200\342\224\200\342\224\200\342\224\200\342\224\200\342\224\200\342\224\200\012" + "// Island code loaded by absolute path: require (CommonJS, and require(esm)\012" + "// for ES modules without top-level await) answers the exports; import\012" + "// answers a promise of the namespace.\012" + "const { pathToFileURL } = require(\"node:url\");\012" + "\012" + "function load(path, esm) {\012" + " // A built-in module's name, with or without the node: prefix.\012" + " if (isBuiltin(path)) return require(path);\012" + " if (esm) return import(pathToFileURL(path).href);\012" + " return createRequire(path)(path);\012" + "}\012" + "\012" + "// \342\224\200\342\224\200 the hosted loop (scope-B NI-3) \342\224\200\342\224\200\342\224\200\342\224\200\342\224\200\342\224\200\342\224\200\342\224\200\342\224\200\342\224\200\342\224\200\342\224\200\342\224\200\342\224\200\342\224\200\342\224\200\342\224\200\342\224\200\342\224\200\342\224\200\342\224\200\342\224\200\342\224\200\342\224\200\342\224\200\342\224\200\342\224\200\342\224\200\342\224\200\342\224\200\342\224\200\342\224\200\342\224\200\342\224\200\342\224\200\342\224\200\342\224\200\012" + "// The runtime's ticks, timers and immediates are Node's, created here with\012" + "// Node's own timers (captured now, so island code replacing the globals\012" + "// does not reach them) and run through the native `run`. Handles name the\012" + "// Node objects until the runtime clears them.\012" + "\012" + "const JOB_TICK = 2;\012" + "const JOB_TIMER = 3;\012" + "const JOB_IMMEDIATE = 4;\012" + "const handles = new Map();\012" + "let nextHandle = 1;\012" + "\012" + "function onTick(token) {\012" + " nativeHost.run(JOB_TICK, token);\012" + "}\012" + "\012" + "function onTimer(token) {\012" + " nativeHost.run(JOB_TIMER, token);\012" + "}\012" + "\012" + "function onImmediate(handle, token) {\012" + " handles.delete(handle);\012" + " nativeHost.run(JOB_IMMEDIATE, token);\012" + "}\012" + "\012" + "function setTimer(delay, repeat, ref, token) {\012" + " let timer;\012" + " if (repeat > 0 && delay === repeat) {\012" + " timer = timers.setInterval(onTimer, repeat, token);\012" + " } else {\012" + " timer = timers.setTimeout(onTimer, delay, token);\012" + " // An interval migrated mid-period: its first run comes after the rest\012" + " // of the period, later ones every `repeat` (Node re-arms a Timeout\012" + " // whose _repeat is set, exactly as for setInterval).\012" + " if (repeat > 0) timer._repeat = repeat;\012" + " }\012" + " if (!ref) timer.unref();\012" + " const handle = nextHandle++;\012" + " handles.set(handle, timer);\012" + " return handle;\012" + "}\012" + "\012" + "function clearTimer(handle) {\012" + " const timer = handles.get(handle);\012" + " if (timer === undefined) return;\012" + " handles.delete(handle);\012" + " timers.clearTimeout(timer);\012" + "}\012" + "\012" + "function refTimer(handle, ref) {\012" + " const timer = handles.get(handle);\012" + " if (timer !== undefined) {\012" + " if (ref) timer.ref();\012" + " else timer.unref();\012" + " }\012" + "}\012" + "\012" + "function refreshTimer(handle) {\012" + " const timer = handles.get(handle);\012" + " if (timer !== undefined) timer.refresh();\012" + "}\012" + "\012" + "function setImmediateJob(ref, token) {\012" + " const handle = nextHandle++;\012" + " const immediate = timers.setImmediate(onImmediate, handle, token);\012" + " if (!ref) immediate.unref();\012" + " handles.set(handle, immediate);\012" + " return handle;\012" + "}\012" + "\012" + "function clearImmediateJob(handle) {\012" + " const immediate = handles.get(handle);\012" + " if (immediate === undefined) return;\012" + " handles.delete(handle);\012" + " timers.clearImmediate(immediate);\012" + "}\012" + "\012" + "function refImmediate(handle, ref) {\012" + " refTimer(handle, ref);\012" + "}\012" + "\012" + "function nextTick(token) {\012" + " process.nextTick(onTick, token);\012" + "}\012" + "\012" + "// \342\224\200\342\224\200 process state (scope-B NI-3) \342\224\200\342\224\200\342\224\200\342\224\200\342\224\200\342\224\200\342\224\200\342\224\200\342\224\200\342\224\200\342\224\200\342\224\200\342\224\200\342\224\200\342\224\200\342\224\200\342\224\200\342\224\200\342\224\200\342\224\200\342\224\200\342\224\200\342\224\200\342\224\200\342\224\200\342\224\200\342\224\200\342\224\200\342\224\200\342\224\200\342\224\200\342\224\200\342\224\200\342\224\200\342\224\200\342\224\200\342\224\200\342\224\200\342\224\200\342\224\200\342\224\200\012" + "// Signal and 'exit' listeners of island code join the runtime's registries\012" + "// at their registration position, so one registry orders both sides and\012" + "// the runtime keeps owning every signal disposition: Node's own handlers,\012" + "// which would hand a signal to libuv and reset it to SIG_DFL when the last\012" + "// listener goes, are taken off. The listeners stay on `process` (listeners\012" + "// and listenerCount answer as in Node); the runtime calls them through\012" + "// `emit`, and Node's own 'exit' emission is left to the runtime's exit\012" + "// sequence.\012" + "\012" + "const EVENT_SIGNAL = 1;\012" + "const EVENT_EXIT = 2;\012" + "const signalNames = new Set(Object.keys(osConstants.signals));\012" + "const islandListeners = [];\012" + "let nextListener = 1;\012" + "\012" + "function eventKind(event) {\012" + " if (event === \"exit\") return EVENT_EXIT;\012" + " return typeof event === \"string\" && signalNames.has(event) \077 EVENT_SIGNAL : 0;\012" + "}\012" + "\012" + "for (const [event, name] of [\012" + " [\"newListener\", \"startListeningIfSignal\"],\012" + " [\"removeListener\", \"stopListeningIfSignal\"],\012" + "]) {\012" + " for (const listener of process.listeners(event)) {\012" + " if (listener.name === name) process.removeListener(event, listener);\012" + " }\012" + "}\012" + "\012" + "process.on(\"newListener\", (event, listener) => {\012" + " const kind = eventKind(event);\012" + " if (kind === 0) return;\012" + " const id = nextListener++;\012" + " islandListeners.push({ event, listener, id, raw: undefined });\012" + " nativeHost.listen(kind, event, id, 1, 0);\012" + "});\012" + "\012" + "function forget(i, kind) {\012" + " const entry = islandListeners[i];\012" + " islandListeners.splice(i, 1);\012" + " nativeHost.listen(kind, entry.event, entry.id, 0, 0);\012" + "}\012" + "\012" + "process.on(\"removeListener\", (event, listener) => {\012" + " const kind = eventKind(event);\012" + " if (kind === 0) return;\012" + " // Node reports the removed registration as the raw listener it installed (a\012" + " // once wrapper removing itself reports the wrapper) or as the listener\012" + " // itself. The raw listener each entry recorded separates a once\012" + " // registration from an `on` registration of the same function: the removed\012" + " // one is the entry whose raw listener is no longer on `process`.\012" + " const live = process.rawListeners(event);\012" + " let same = -1;\012" + " for (let i = islandListeners.length - 1; i >= 0; i--) {\012" + " const entry = islandListeners[i];\012" + " if (entry.event !== event) continue;\012" + " if (entry.listener !== listener && entry.raw !== listener) continue;\012" + " if (entry.raw !== undefined && !live.includes(entry.raw)) {\012" + " forget(i, kind);\012" + " return;\012" + " }\012" + " if (same < 0) same = i;\012" + " }\012" + " // Duplicate registrations of one function share a raw listener; Node\012" + " // removes the most recent, which is the last matching entry.\012" + " if (same >= 0) forget(same, kind);\012" + "});\012" + "\012" + "// The raw listener is the function handed to the emitter (a once wrapper\012" + "// for process.once, which registers through process.on): tag each entry\012" + "// with it as its registration returns. Node emits newListener before it\012" + "// installs the listener, so the entry learns its raw only afterwards.\012" + "for (const name of [\"on\", \"addListener\", \"prependListener\"]) {\012" + " const add = process[name];\012" + " process[name] = function addListener(event, listener) {\012" + " const start = islandListeners.length;\012" + " const result = Reflect.apply(add, this, arguments);\012" + " if (islandListeners.length > start) {\012" + " const registered = listener.listener \077\077 listener;\012" + " for (let i = start; i < islandListeners.length; i++) {\012" + " const entry = islandListeners[i];\012" + " if (entry.event === event && entry.listener === registered && entry.raw === undefined) {\012" + " entry.raw = listener;\012" + " break;\012" + " }\012" + " }\012" + " }\012" + " return result;\012" + " };\012" + "}\012" + "\012" + "function emit(kind, id, name, arg) {\012" + " const entry = islandListeners.find((e) => e.id === id);\012" + " if (entry === undefined) return;\012" + " // The registered function: the listener itself, or its once wrapper,\012" + " // which removes itself before calling it.\012" + " const registered =\012" + " entry.raw \077\077\012" + " process.rawListeners(entry.event).find((raw) => (raw.listener \077\077 raw) === entry.listener);\012" + " if (registered === undefined) return;\012" + " if (kind === EVENT_SIGNAL) Reflect.apply(registered, process, [name, arg]);\012" + " else Reflect.apply(registered, process, [arg]);\012" + "}\012" + "\012" + "const nodeEmit = process.emit;\012" + "process.emit = function emit(event, ...args) {\012" + " if (event === \"exit\") return this.listenerCount(\"exit\") > 0;\012" + " return Reflect.apply(nodeEmit, this, [event, ...args]);\012" + "};\012" + "\012" "// Writes reach the shared stdout and stderr before control returns to the\012" "// program, as the program's own writes do.\012" "for (const stream of [process.stdout, process.stderr]) {\012" @@ -167,13 +379,6 @@ static const char scr_node_island_host_js[] = " stream._handle.setBlocking(true);\012" "}\012" "\012" - "// The program ends with `code` (the bridge then runs Node's loop to its\012" - "// end, which emits 'beforeExit' and 'exit'): Node's process shares it, as\012" - "// it shares one process under Node.\012" - "function finish(code) {\012" - " if (code !== 0) process.exitCode = code;\012" - "}\012" - "\012" "// process.execPath is this program, not a node executable: a fork without\012" "// an execPath of its own would start the program again.\012" "const childProcess = require(\"node:child_process\");\012" @@ -191,6 +396,18 @@ static const char scr_node_island_host_js[] = "};\012" "require(\"node:module\").syncBuiltinESMExports();\012" "\012" - "globalThis.__scriptcNodeIsland = { call, finish };\012"; + "globalThis.__scriptcNodeIsland = {\012" + " call,\012" + " setTimer,\012" + " clearTimer,\012" + " refTimer,\012" + " refreshTimer,\012" + " setImmediate: setImmediateJob,\012" + " clearImmediate: clearImmediateJob,\012" + " refImmediate,\012" + " nextTick,\012" + " emit,\012" + " load,\012" + "};\012"; #endif /* SCR_NODE_ISLAND_HOST_H */ diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 619c4b7fb..d3284ced5 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -236,6 +236,16 @@ importers: specifier: workspace:* version: link:../threads + packages/libnode-common: {} + + packages/libnode-darwin-arm64: {} + + packages/libnode-darwin-x64: {} + + packages/libnode-linux-arm64-gnu: {} + + packages/libnode-linux-x64-gnu: {} + packages/llvm-darwin-arm64: {} packages/llvm-darwin-x64: {} diff --git a/scripts/sync-versions.mjs b/scripts/sync-versions.mjs index 57d6c4ec0..3f564b15e 100644 --- a/scripts/sync-versions.mjs +++ b/scripts/sync-versions.mjs @@ -48,6 +48,10 @@ for (const pkg of [ "llvm-linux-x64-musl", "llvm-linux-arm64-musl", "llvm-win32-x64-msvc", + "libnode-linux-x64-gnu", + "libnode-linux-arm64-gnu", + "libnode-darwin-arm64", + "libnode-darwin-x64", ]) { const path = manifest(pkg); const json = read(path); diff --git a/tests/corpus-node-islands-order/e01-island-timer-keeps-alive/main.ts b/tests/corpus-node-islands-order/e01-island-timer-keeps-alive/main.ts new file mode 100644 index 000000000..144e2b479 --- /dev/null +++ b/tests/corpus-node-islands-order/e01-island-timer-keeps-alive/main.ts @@ -0,0 +1,20 @@ +// @requires-libnode +// X { T 30 { L i-late } } L h-end +import { callSync } from "@scriptc/node"; +import { readFile, writeSync } from "node:fs"; + +function log(s: string): void { + writeSync(1, `${s}\n`); +} + +function busy(ms: number): void { + const end = Date.now() + ms; + while (Date.now() < end) {} +} + +function island(src: string): void { + callSync("../island.cjs", "run", src); +} + +island("T 30 { L i-late }"); +log("h-end"); diff --git a/tests/corpus-node-islands-order/e02-island-io-keeps-alive/main.ts b/tests/corpus-node-islands-order/e02-island-io-keeps-alive/main.ts new file mode 100644 index 000000000..9ac7c00b1 --- /dev/null +++ b/tests/corpus-node-islands-order/e02-island-io-keeps-alive/main.ts @@ -0,0 +1,20 @@ +// @requires-libnode +// X { F 50 { L i-done } } L h-end +import { callSync } from "@scriptc/node"; +import { readFile, writeSync } from "node:fs"; + +function log(s: string): void { + writeSync(1, `${s}\n`); +} + +function busy(ms: number): void { + const end = Date.now() + ms; + while (Date.now() < end) {} +} + +function island(src: string): void { + callSync("../island.cjs", "run", src); +} + +island("F 50 { L i-done }"); +log("h-end"); diff --git a/tests/corpus-node-islands-order/e03-unref-server-exits/main.ts b/tests/corpus-node-islands-order/e03-unref-server-exits/main.ts new file mode 100644 index 000000000..53c2100cc --- /dev/null +++ b/tests/corpus-node-islands-order/e03-unref-server-exits/main.ts @@ -0,0 +1,20 @@ +// @requires-libnode +// X { O { L i-listening } } L h-end +import { callSync } from "@scriptc/node"; +import { readFile, writeSync } from "node:fs"; + +function log(s: string): void { + writeSync(1, `${s}\n`); +} + +function busy(ms: number): void { + const end = Date.now() + ms; + while (Date.now() < end) {} +} + +function island(src: string): void { + callSync("../island.cjs", "run", src); +} + +island("O { L i-listening }"); +log("h-end"); diff --git a/tests/corpus-node-islands-order/e04-throw-in-io-step/main.ts b/tests/corpus-node-islands-order/e04-throw-in-io-step/main.ts new file mode 100644 index 000000000..8110c3744 --- /dev/null +++ b/tests/corpus-node-islands-order/e04-throw-in-io-step/main.ts @@ -0,0 +1,23 @@ +// @requires-libnode +// @order: race +// X { G R { L i-io E } } B 30 L h-sync T 10 { L h-t10 } +import { callSync } from "@scriptc/node"; +import { readFile, writeSync } from "node:fs"; + +function log(s: string): void { + writeSync(1, `${s}\n`); +} + +function busy(ms: number): void { + const end = Date.now() + ms; + while (Date.now() < end) {} +} + +function island(src: string): void { + callSync("../island.cjs", "run", src); +} + +island("G R { L i-io E }"); +busy(30); +log("h-sync"); +setTimeout(() => log("h-t10"), 10); diff --git a/tests/corpus-node-islands-order/g01-fs-chain/main.ts b/tests/corpus-node-islands-order/g01-fs-chain/main.ts new file mode 100644 index 000000000..0b14df503 --- /dev/null +++ b/tests/corpus-node-islands-order/g01-fs-chain/main.ts @@ -0,0 +1,24 @@ +// @requires-libnode +// @order: race +// X { T 0 { F 20 { L i-fs-chain } } } T 2 { B 80 L h-sync } +import { callSync } from "@scriptc/node"; +import { readFile, writeSync } from "node:fs"; + +function log(s: string): void { + writeSync(1, `${s}\n`); +} + +function busy(ms: number): void { + const end = Date.now() + ms; + while (Date.now() < end) {} +} + +function island(src: string): void { + callSync("../island.cjs", "run", src); +} + +island("T 0 { F 20 { L i-fs-chain } }"); +setTimeout(() => { + busy(80); + log("h-sync"); +}, 2); diff --git a/tests/corpus-node-islands-order/g02-net/main.ts b/tests/corpus-node-islands-order/g02-net/main.ts new file mode 100644 index 000000000..c3aa0159c --- /dev/null +++ b/tests/corpus-node-islands-order/g02-net/main.ts @@ -0,0 +1,23 @@ +// @requires-libnode +// X { T 0 { N { L i-net } } } T 2 { B 80 L h-sync } +import { callSync } from "@scriptc/node"; +import { readFile, writeSync } from "node:fs"; + +function log(s: string): void { + writeSync(1, `${s}\n`); +} + +function busy(ms: number): void { + const end = Date.now() + ms; + while (Date.now() < end) {} +} + +function island(src: string): void { + callSync("../island.cjs", "run", src); +} + +island("T 0 { N { L i-net } }"); +setTimeout(() => { + busy(80); + log("h-sync"); +}, 2); diff --git a/tests/corpus-node-islands-order/g03-child/main.ts b/tests/corpus-node-islands-order/g03-child/main.ts new file mode 100644 index 000000000..2f9f7171e --- /dev/null +++ b/tests/corpus-node-islands-order/g03-child/main.ts @@ -0,0 +1,24 @@ +// @requires-libnode +// @order: race +// X { T 0 { C { L i-child } } } T 2 { B 150 L h-sync } +import { callSync } from "@scriptc/node"; +import { readFile, writeSync } from "node:fs"; + +function log(s: string): void { + writeSync(1, `${s}\n`); +} + +function busy(ms: number): void { + const end = Date.now() + ms; + while (Date.now() < end) {} +} + +function island(src: string): void { + callSync("../island.cjs", "run", src); +} + +island("T 0 { C { L i-child } }"); +setTimeout(() => { + busy(150); + log("h-sync"); +}, 2); diff --git a/tests/corpus-node-islands-order/g04-dns/main.ts b/tests/corpus-node-islands-order/g04-dns/main.ts new file mode 100644 index 000000000..f9bb7cb4b --- /dev/null +++ b/tests/corpus-node-islands-order/g04-dns/main.ts @@ -0,0 +1,24 @@ +// @requires-libnode +// @order: race +// X { T 0 { D { L i-dns } } } T 2 { B 80 L h-sync } +import { callSync } from "@scriptc/node"; +import { readFile, writeSync } from "node:fs"; + +function log(s: string): void { + writeSync(1, `${s}\n`); +} + +function busy(ms: number): void { + const end = Date.now() + ms; + while (Date.now() < end) {} +} + +function island(src: string): void { + callSync("../island.cjs", "run", src); +} + +island("T 0 { D { L i-dns } }"); +setTimeout(() => { + busy(80); + log("h-sync"); +}, 2); diff --git a/tests/corpus-node-islands-order/g05-zlib/main.ts b/tests/corpus-node-islands-order/g05-zlib/main.ts new file mode 100644 index 000000000..14abe51b1 --- /dev/null +++ b/tests/corpus-node-islands-order/g05-zlib/main.ts @@ -0,0 +1,23 @@ +// @requires-libnode +// X { T 0 { Z { L i-zlib } } } T 2 { B 80 L h-sync } +import { callSync } from "@scriptc/node"; +import { readFile, writeSync } from "node:fs"; + +function log(s: string): void { + writeSync(1, `${s}\n`); +} + +function busy(ms: number): void { + const end = Date.now() + ms; + while (Date.now() < end) {} +} + +function island(src: string): void { + callSync("../island.cjs", "run", src); +} + +island("T 0 { Z { L i-zlib } }"); +setTimeout(() => { + busy(80); + log("h-sync"); +}, 2); diff --git a/tests/corpus-node-islands-order/g06-worker/main.ts b/tests/corpus-node-islands-order/g06-worker/main.ts new file mode 100644 index 000000000..5ee47a294 --- /dev/null +++ b/tests/corpus-node-islands-order/g06-worker/main.ts @@ -0,0 +1,23 @@ +// @requires-libnode +// X { T 0 { W { L i-worker } } } T 2 { B 300 L h-sync } +import { callSync } from "@scriptc/node"; +import { readFile, writeSync } from "node:fs"; + +function log(s: string): void { + writeSync(1, `${s}\n`); +} + +function busy(ms: number): void { + const end = Date.now() + ms; + while (Date.now() < end) {} +} + +function island(src: string): void { + callSync("../island.cjs", "run", src); +} + +island("T 0 { W { L i-worker } }"); +setTimeout(() => { + busy(300); + log("h-sync"); +}, 2); diff --git a/tests/corpus-node-islands-order/g07-internal-timer/main.ts b/tests/corpus-node-islands-order/g07-internal-timer/main.ts new file mode 100644 index 000000000..724a57cd6 --- /dev/null +++ b/tests/corpus-node-islands-order/g07-internal-timer/main.ts @@ -0,0 +1,23 @@ +// @requires-libnode +// X { T 0 { S { L i-internal-timer } } } T 2 { B 80 L h-sync } +import { callSync } from "@scriptc/node"; +import { readFile, writeSync } from "node:fs"; + +function log(s: string): void { + writeSync(1, `${s}\n`); +} + +function busy(ms: number): void { + const end = Date.now() + ms; + while (Date.now() < end) {} +} + +function island(src: string): void { + callSync("../island.cjs", "run", src); +} + +island("T 0 { S { L i-internal-timer } }"); +setTimeout(() => { + busy(80); + log("h-sync"); +}, 2); diff --git a/tests/corpus-node-islands-order/island.cjs b/tests/corpus-node-islands-order/island.cjs new file mode 100644 index 000000000..e02e2e857 --- /dev/null +++ b/tests/corpus-node-islands-order/island.cjs @@ -0,0 +1,183 @@ +"use strict"; +// The island half of the Node island ordering programs (scope-B design +// §5.2): `run(src)` interprets a small scheduling language inside Node, +// the same in a scriptc build (Node loaded in-process, one event loop) and +// under Node (@scriptc/node calls the module directly). Each program's +// main.ts is the native half. Ops (bodies in braces): +// L name log a line (synchronous write to fd 1) +// J log the AsyncLocalStorage store +// Y { .. } run the body inside als.run("s", ..) +// T ms { .. } setTimeout I { .. } setImmediate +// M { .. } queueMicrotask K { .. } process.nextTick +// P { .. } Promise.resolve().then(body) +// R { .. } fs.readFile, then the body +// A { .. } fs.promises.readFile, then the body +// F n { .. } n sequential fs.stat callbacks, then the body +// N { .. } a TCP round trip on localhost, the body on client data +// C { .. } child_process.execFile("/bin/true"), the body in its callback +// D { .. } dns.lookup("localhost") +// Z { .. } zlib.gzip +// W { .. } worker_threads: the body on the worker's first message +// S { .. } a socket idle timeout (a Node-internal timer) +// O { .. } an unref'd TCP server, the body on 'listening' +// E throw; G installs an uncaughtException handler that logs +// B ms busy-wait +const fs = require("node:fs"); +const { AsyncLocalStorage } = require("node:async_hooks"); + +const als = new AsyncLocalStorage(); + +function parse(src) { + const toks = src.trim().split(/\s+/).filter(Boolean); + let i = 0; + function body() { + if (toks[i++] !== "{") throw new Error(`expected { at ${i}`); + const start = i; + let depth = 1; + while (depth) { + const t = toks[i++]; + if (t === "{") depth++; + else if (t === "}") depth--; + } + return toks.slice(start, i - 1).join(" "); + } + const out = []; + while (i < toks.length) { + const k = toks[i++]; + if (k === "L") out.push({ k, s: toks[i++] }); + else if (k === "J" || k === "E" || k === "G") out.push({ k }); + else if (k === "B") out.push({ k, ms: +toks[i++] }); + else if (k === "T" || k === "F") { + const ms = +toks[i++]; + out.push({ k, ms, body: body() }); + } else out.push({ k, body: body() }); + } + return out; +} + +const log = (s) => fs.writeSync(1, `${s}\n`); +const busy = (ms) => { + const end = Date.now() + ms; + while (Date.now() < end); +}; + +function exec(src) { + for (const op of parse(src)) { + switch (op.k) { + case "L": + log(op.s); + break; + case "J": + log(`store=${als.getStore()}`); + break; + case "E": + throw new Error("boom"); + case "G": + process.on("uncaughtException", (e) => log(`uncaught-${e.message}`)); + break; + case "Y": + als.run("s", () => exec(op.body)); + break; + case "B": + busy(op.ms); + break; + default: + schedule(op); + } + } +} + +function schedule(op) { + const run = () => exec(op.body); + switch (op.k) { + case "T": + setTimeout(run, op.ms); + break; + case "I": + setImmediate(run); + break; + case "M": + queueMicrotask(run); + break; + case "K": + process.nextTick(run); + break; + case "P": + Promise.resolve().then(run); + break; + case "R": + fs.readFile(__filename, run); + break; + case "A": + fs.promises.readFile(__filename).then(run); + break; + case "F": { + let n = op.ms; + const next = () => (n-- > 0 ? fs.stat(__filename, next) : run()); + next(); + break; + } + case "N": { + const net = require("node:net"); + const srv = net + .createServer((s) => s.end("x")) + .listen(0, "127.0.0.1", () => { + const c = net.connect(srv.address().port, "127.0.0.1"); + c.once("data", () => { + c.destroy(); + srv.close(); + run(); + }); + }); + break; + } + case "C": + require("node:child_process").execFile("/bin/true", run); + break; + case "D": + require("node:dns").lookup("localhost", run); + break; + case "Z": + require("node:zlib").gzip("hello", run); + break; + case "W": { + const { Worker } = require("node:worker_threads"); + const w = new Worker('require("node:worker_threads").parentPort.postMessage(1)', { + eval: true, + }); + w.once("message", () => { + w.terminate(); + run(); + }); + break; + } + case "S": { + const net = require("node:net"); + const srv = net + .createServer((s) => + s.setTimeout(10, () => { + s.destroy(); + srv.close(); + run(); + }), + ) + .listen(0, "127.0.0.1", () => { + const c = net.connect(srv.address().port, "127.0.0.1"); + c.on("error", () => {}); + c.on("close", () => {}); + }); + break; + } + case "O": { + const s = require("node:net").createServer().listen(0, "127.0.0.1", run); + s.unref(); + break; + } + default: + throw new Error(`op ${op.k}`); + } +} + +exports.run = function run(src) { + exec(src); +}; diff --git a/tests/corpus-node-islands-order/p01-timers-interleave/main.ts b/tests/corpus-node-islands-order/p01-timers-interleave/main.ts new file mode 100644 index 000000000..1c002a494 --- /dev/null +++ b/tests/corpus-node-islands-order/p01-timers-interleave/main.ts @@ -0,0 +1,22 @@ +// @requires-libnode +// T 0 { L h1 } X { T 0 { L i1 } } T 0 { L h2 } X { T 0 { L i2 } } +import { callSync } from "@scriptc/node"; +import { readFile, writeSync } from "node:fs"; + +function log(s: string): void { + writeSync(1, `${s}\n`); +} + +function busy(ms: number): void { + const end = Date.now() + ms; + while (Date.now() < end) {} +} + +function island(src: string): void { + callSync("../island.cjs", "run", src); +} + +setTimeout(() => log("h1"), 0); +island("T 0 { L i1 }"); +setTimeout(() => log("h2"), 0); +island("T 0 { L i2 }"); diff --git a/tests/corpus-node-islands-order/p02-tick-micro-immediate/main.ts b/tests/corpus-node-islands-order/p02-tick-micro-immediate/main.ts new file mode 100644 index 000000000..c6492e227 --- /dev/null +++ b/tests/corpus-node-islands-order/p02-tick-micro-immediate/main.ts @@ -0,0 +1,23 @@ +// @requires-libnode +// I { L hI } X { I { L iI } M { L iM } K { L iK } } M { L hM } K { L hK } L sync +import { callSync } from "@scriptc/node"; +import { readFile, writeSync } from "node:fs"; + +function log(s: string): void { + writeSync(1, `${s}\n`); +} + +function busy(ms: number): void { + const end = Date.now() + ms; + while (Date.now() < end) {} +} + +function island(src: string): void { + callSync("../island.cjs", "run", src); +} + +setImmediate(() => log("hI")); +island("I { L iI } M { L iM } K { L iK }"); +queueMicrotask(() => log("hM")); +process.nextTick(() => log("hK")); +log("sync"); diff --git a/tests/corpus-node-islands-order/p03-busy-vs-island-timer/main.ts b/tests/corpus-node-islands-order/p03-busy-vs-island-timer/main.ts new file mode 100644 index 000000000..1f7f81e26 --- /dev/null +++ b/tests/corpus-node-islands-order/p03-busy-vs-island-timer/main.ts @@ -0,0 +1,21 @@ +// @requires-libnode +// X { T 0 { L i-timer } } B 50 L h-after-busy +import { callSync } from "@scriptc/node"; +import { readFile, writeSync } from "node:fs"; + +function log(s: string): void { + writeSync(1, `${s}\n`); +} + +function busy(ms: number): void { + const end = Date.now() + ms; + while (Date.now() < end) {} +} + +function island(src: string): void { + callSync("../island.cjs", "run", src); +} + +island("T 0 { L i-timer }"); +busy(50); +log("h-after-busy"); diff --git a/tests/corpus-node-islands-order/p05-io-and-promises/main.ts b/tests/corpus-node-islands-order/p05-io-and-promises/main.ts new file mode 100644 index 000000000..b8d7f7325 --- /dev/null +++ b/tests/corpus-node-islands-order/p05-io-and-promises/main.ts @@ -0,0 +1,22 @@ +// @requires-libnode +// @order: race +// X { R { L i-io M { L i-io-micro } } } R { L h-io } T 0 { L h-t0 } +import { callSync } from "@scriptc/node"; +import { readFile, writeSync } from "node:fs"; + +function log(s: string): void { + writeSync(1, `${s}\n`); +} + +function busy(ms: number): void { + const end = Date.now() + ms; + while (Date.now() < end) {} +} + +function island(src: string): void { + callSync("../island.cjs", "run", src); +} + +island("R { L i-io M { L i-io-micro } }"); +readFile(import.meta.filename, "utf8", () => log("h-io")); +setTimeout(() => log("h-t0"), 0); diff --git a/tests/corpus-node-islands-order/p07-microtask-fifo/main.ts b/tests/corpus-node-islands-order/p07-microtask-fifo/main.ts new file mode 100644 index 000000000..14de725d4 --- /dev/null +++ b/tests/corpus-node-islands-order/p07-microtask-fifo/main.ts @@ -0,0 +1,23 @@ +// @requires-libnode +// M { L hm1 } X { M { L im1 } } M { L hm2 } X { M { L im2 } } L sync +import { callSync } from "@scriptc/node"; +import { readFile, writeSync } from "node:fs"; + +function log(s: string): void { + writeSync(1, `${s}\n`); +} + +function busy(ms: number): void { + const end = Date.now() + ms; + while (Date.now() < end) {} +} + +function island(src: string): void { + callSync("../island.cjs", "run", src); +} + +queueMicrotask(() => log("hm1")); +island("M { L im1 }"); +queueMicrotask(() => log("hm2")); +island("M { L im2 }"); +log("sync"); diff --git a/tests/corpus-node-islands-order/p08-timer-vs-immediate/main.ts b/tests/corpus-node-islands-order/p08-timer-vs-immediate/main.ts new file mode 100644 index 000000000..f448c8925 --- /dev/null +++ b/tests/corpus-node-islands-order/p08-timer-vs-immediate/main.ts @@ -0,0 +1,22 @@ +// @requires-libnode +// @order: race +// T 1 { L h-t1 } X { T 0 { L i-t0 } I { L i-imm } } I { L h-imm } +import { callSync } from "@scriptc/node"; +import { readFile, writeSync } from "node:fs"; + +function log(s: string): void { + writeSync(1, `${s}\n`); +} + +function busy(ms: number): void { + const end = Date.now() + ms; + while (Date.now() < end) {} +} + +function island(src: string): void { + callSync("../island.cjs", "run", src); +} + +setTimeout(() => log("h-t1"), 1); +island("T 0 { L i-t0 } I { L i-imm }"); +setImmediate(() => log("h-imm")); diff --git a/tests/corpus-node-islands-order/p09-deadline-order-under-busy/main.ts b/tests/corpus-node-islands-order/p09-deadline-order-under-busy/main.ts new file mode 100644 index 000000000..c89284633 --- /dev/null +++ b/tests/corpus-node-islands-order/p09-deadline-order-under-busy/main.ts @@ -0,0 +1,22 @@ +// @requires-libnode +// X { T 10 { L i-t10 } } T 20 { L h-t20 } B 30 L h-sync +import { callSync } from "@scriptc/node"; +import { readFile, writeSync } from "node:fs"; + +function log(s: string): void { + writeSync(1, `${s}\n`); +} + +function busy(ms: number): void { + const end = Date.now() + ms; + while (Date.now() < end) {} +} + +function island(src: string): void { + callSync("../island.cjs", "run", src); +} + +island("T 10 { L i-t10 }"); +setTimeout(() => log("h-t20"), 20); +busy(30); +log("h-sync"); diff --git a/tests/corpus-node-islands-order/p10-island-chain/main.ts b/tests/corpus-node-islands-order/p10-island-chain/main.ts new file mode 100644 index 000000000..a59856f79 --- /dev/null +++ b/tests/corpus-node-islands-order/p10-island-chain/main.ts @@ -0,0 +1,24 @@ +// @requires-libnode +// @order: race +// X { T 0 { L i-a K { L i-tick } M { L i-micro } I { L i-imm } } } T 0 { L h-t0 M { L h-micro } } +import { callSync } from "@scriptc/node"; +import { readFile, writeSync } from "node:fs"; + +function log(s: string): void { + writeSync(1, `${s}\n`); +} + +function busy(ms: number): void { + const end = Date.now() + ms; + while (Date.now() < end) {} +} + +function island(src: string): void { + callSync("../island.cjs", "run", src); +} + +island("T 0 { L i-a K { L i-tick } M { L i-micro } I { L i-imm } }"); +setTimeout(() => { + log("h-t0"); + queueMicrotask(() => log("h-micro")); +}, 0); diff --git a/tests/corpus-node-islands-order/p11-island-real-io/main.ts b/tests/corpus-node-islands-order/p11-island-real-io/main.ts new file mode 100644 index 000000000..6895cb6e0 --- /dev/null +++ b/tests/corpus-node-islands-order/p11-island-real-io/main.ts @@ -0,0 +1,24 @@ +// @requires-libnode +// @order: race +// X { R { L i-io M { L i-io-micro } } } R { L h-io } T 0 { L h-t0 } +import { callSync } from "@scriptc/node"; +import { readFile, writeSync } from "node:fs"; + +function log(s: string): void { + writeSync(1, `${s}\n`); +} + +function busy(ms: number): void { + const end = Date.now() + ms; + while (Date.now() < end) {} +} + +function island(src: string): void { + callSync("../island.cjs", "run", src); +} + +island("R { L i-io M { L i-io-micro } }"); +readFile(import.meta.filename, "utf8", () => { + log("h-io"); +}); +setTimeout(() => log("h-t0"), 0); diff --git a/tests/corpus-node-islands-order/p12-island-io-vs-busy/main.ts b/tests/corpus-node-islands-order/p12-island-io-vs-busy/main.ts new file mode 100644 index 000000000..477175473 --- /dev/null +++ b/tests/corpus-node-islands-order/p12-island-io-vs-busy/main.ts @@ -0,0 +1,23 @@ +// @requires-libnode +// @order: race +// X { R { L i-io } } B 30 L h-sync T 0 { L h-t0 } +import { callSync } from "@scriptc/node"; +import { readFile, writeSync } from "node:fs"; + +function log(s: string): void { + writeSync(1, `${s}\n`); +} + +function busy(ms: number): void { + const end = Date.now() + ms; + while (Date.now() < end) {} +} + +function island(src: string): void { + callSync("../island.cjs", "run", src); +} + +island("R { L i-io }"); +busy(30); +log("h-sync"); +setTimeout(() => log("h-t0"), 0); diff --git a/tests/corpus-node-islands-order/q01-promise-job-fifo/main.ts b/tests/corpus-node-islands-order/q01-promise-job-fifo/main.ts new file mode 100644 index 000000000..36fb5d76b --- /dev/null +++ b/tests/corpus-node-islands-order/q01-promise-job-fifo/main.ts @@ -0,0 +1,23 @@ +// @requires-libnode +// M { L hm1 } X { P { L ip1 } } M { L hm2 } X { P { L ip2 } } L sync +import { callSync } from "@scriptc/node"; +import { readFile, writeSync } from "node:fs"; + +function log(s: string): void { + writeSync(1, `${s}\n`); +} + +function busy(ms: number): void { + const end = Date.now() + ms; + while (Date.now() < end) {} +} + +function island(src: string): void { + callSync("../island.cjs", "run", src); +} + +queueMicrotask(() => log("hm1")); +island("P { L ip1 }"); +queueMicrotask(() => log("hm2")); +island("P { L ip2 }"); +log("sync"); diff --git a/tests/corpus-node-islands-order/q02-promise-batch/main.ts b/tests/corpus-node-islands-order/q02-promise-batch/main.ts new file mode 100644 index 000000000..09f9eb191 --- /dev/null +++ b/tests/corpus-node-islands-order/q02-promise-batch/main.ts @@ -0,0 +1,23 @@ +// @requires-libnode +// X { P { L ip1 P { L ip2 } } } M { L hm1 M { L hm2 } } +import { callSync } from "@scriptc/node"; +import { readFile, writeSync } from "node:fs"; + +function log(s: string): void { + writeSync(1, `${s}\n`); +} + +function busy(ms: number): void { + const end = Date.now() + ms; + while (Date.now() < end) {} +} + +function island(src: string): void { + callSync("../island.cjs", "run", src); +} + +island("P { L ip1 P { L ip2 } }"); +queueMicrotask(() => { + log("hm1"); + queueMicrotask(() => log("hm2")); +}); diff --git a/tests/corpus-node-islands-order/q03-io-callback-queues/main.ts b/tests/corpus-node-islands-order/q03-io-callback-queues/main.ts new file mode 100644 index 000000000..18b973c13 --- /dev/null +++ b/tests/corpus-node-islands-order/q03-io-callback-queues/main.ts @@ -0,0 +1,21 @@ +// @requires-libnode +// X { R { L i-io K { L i-tick } P { L i-p } M { L i-m } I { L i-imm } } } B 40 L h-sync +import { callSync } from "@scriptc/node"; +import { readFile, writeSync } from "node:fs"; + +function log(s: string): void { + writeSync(1, `${s}\n`); +} + +function busy(ms: number): void { + const end = Date.now() + ms; + while (Date.now() < end) {} +} + +function island(src: string): void { + callSync("../island.cjs", "run", src); +} + +island("R { L i-io K { L i-tick } P { L i-p } M { L i-m } I { L i-imm } }"); +busy(40); +log("h-sync"); diff --git a/tests/corpus-node-islands-order/q04-fs-promises/main.ts b/tests/corpus-node-islands-order/q04-fs-promises/main.ts new file mode 100644 index 000000000..d253223f1 --- /dev/null +++ b/tests/corpus-node-islands-order/q04-fs-promises/main.ts @@ -0,0 +1,23 @@ +// @requires-libnode +// @order: race +// X { A { L i-a P { L i-ap } } } B 40 L h-sync T 0 { L h-t0 } +import { callSync } from "@scriptc/node"; +import { readFile, writeSync } from "node:fs"; + +function log(s: string): void { + writeSync(1, `${s}\n`); +} + +function busy(ms: number): void { + const end = Date.now() + ms; + while (Date.now() < end) {} +} + +function island(src: string): void { + callSync("../island.cjs", "run", src); +} + +island("A { L i-a P { L i-ap } }"); +busy(40); +log("h-sync"); +setTimeout(() => log("h-t0"), 0); diff --git a/tests/corpus-node-islands-order/q06-async-context/main.ts b/tests/corpus-node-islands-order/q06-async-context/main.ts new file mode 100644 index 000000000..cd814a4ae --- /dev/null +++ b/tests/corpus-node-islands-order/q06-async-context/main.ts @@ -0,0 +1,19 @@ +// @requires-libnode +// X { Y { T 0 { J } I { J } K { J } M { J } R { J } } } +import { callSync } from "@scriptc/node"; +import { readFile, writeSync } from "node:fs"; + +function log(s: string): void { + writeSync(1, `${s}\n`); +} + +function busy(ms: number): void { + const end = Date.now() + ms; + while (Date.now() < end) {} +} + +function island(src: string): void { + callSync("../island.cjs", "run", src); +} + +island("Y { T 0 { J } I { J } K { J } M { J } R { J } }"); diff --git a/tests/corpus-node-islands-order/q07-timer-from-io-vs-host-timer/main.ts b/tests/corpus-node-islands-order/q07-timer-from-io-vs-host-timer/main.ts new file mode 100644 index 000000000..e4af331aa --- /dev/null +++ b/tests/corpus-node-islands-order/q07-timer-from-io-vs-host-timer/main.ts @@ -0,0 +1,22 @@ +// @requires-libnode +// X { R { T 0 { L i-t0 } } } B 40 T 0 { L h-t0 } L h-sync +import { callSync } from "@scriptc/node"; +import { readFile, writeSync } from "node:fs"; + +function log(s: string): void { + writeSync(1, `${s}\n`); +} + +function busy(ms: number): void { + const end = Date.now() + ms; + while (Date.now() < end) {} +} + +function island(src: string): void { + callSync("../island.cjs", "run", src); +} + +island("R { T 0 { L i-t0 } }"); +busy(40); +setTimeout(() => log("h-t0"), 0); +log("h-sync"); diff --git a/tests/corpus-node-islands-order/q09-native-promise-jobs/main.ts b/tests/corpus-node-islands-order/q09-native-promise-jobs/main.ts new file mode 100644 index 000000000..b6d4b470f --- /dev/null +++ b/tests/corpus-node-islands-order/q09-native-promise-jobs/main.ts @@ -0,0 +1,25 @@ +// @requires-libnode +// P { L hp1 } X { P { L ip1 P { L ip3 } } } P { L hp2 P { L hp3 } } L sync +import { callSync } from "@scriptc/node"; +import { readFile, writeSync } from "node:fs"; + +function log(s: string): void { + writeSync(1, `${s}\n`); +} + +function busy(ms: number): void { + const end = Date.now() + ms; + while (Date.now() < end) {} +} + +function island(src: string): void { + callSync("../island.cjs", "run", src); +} + +Promise.resolve().then(() => log("hp1")); +island("P { L ip1 P { L ip3 } }"); +Promise.resolve().then(() => { + log("hp2"); + Promise.resolve().then(() => log("hp3")); +}); +log("sync"); diff --git a/tests/corpus-node-islands-order/q10-async-await-vs-island-jobs/main.ts b/tests/corpus-node-islands-order/q10-async-await-vs-island-jobs/main.ts new file mode 100644 index 000000000..7e813acb1 --- /dev/null +++ b/tests/corpus-node-islands-order/q10-async-await-vs-island-jobs/main.ts @@ -0,0 +1,29 @@ +// @requires-libnode +// async f: await, ha1, await, ha2, await, ha3 || X { P { L ip1 P { L ip2 P { L ip3 } } } } +import { callSync } from "@scriptc/node"; +import { readFile, writeSync } from "node:fs"; + +function log(s: string): void { + writeSync(1, `${s}\n`); +} + +function busy(ms: number): void { + const end = Date.now() + ms; + while (Date.now() < end) {} +} + +function island(src: string): void { + callSync("../island.cjs", "run", src); +} + +async function steps(): Promise { + await null; + log("ha1"); + await null; + log("ha2"); + await Promise.resolve(); + log("ha3"); +} +void steps(); +island("P { L ip1 P { L ip2 P { L ip3 } } }"); +log("sync"); diff --git a/tests/corpus-node-islands-order/q11-ticks-vs-island-ticks/main.ts b/tests/corpus-node-islands-order/q11-ticks-vs-island-ticks/main.ts new file mode 100644 index 000000000..12e9647db --- /dev/null +++ b/tests/corpus-node-islands-order/q11-ticks-vs-island-ticks/main.ts @@ -0,0 +1,25 @@ +// @requires-libnode +// K { L hk1 K { L hk3 } } X { K { L ik1 M { L im1 } } } K { L hk2 } M { L hm1 } +import { callSync } from "@scriptc/node"; +import { readFile, writeSync } from "node:fs"; + +function log(s: string): void { + writeSync(1, `${s}\n`); +} + +function busy(ms: number): void { + const end = Date.now() + ms; + while (Date.now() < end) {} +} + +function island(src: string): void { + callSync("../island.cjs", "run", src); +} + +process.nextTick(() => { + log("hk1"); + process.nextTick(() => log("hk3")); +}); +island("K { L ik1 M { L im1 } }"); +process.nextTick(() => log("hk2")); +queueMicrotask(() => log("hm1")); diff --git a/tests/corpus-node-islands-order/s01-pending-in-main-body/main.ts b/tests/corpus-node-islands-order/s01-pending-in-main-body/main.ts new file mode 100644 index 000000000..e7992afa5 --- /dev/null +++ b/tests/corpus-node-islands-order/s01-pending-in-main-body/main.ts @@ -0,0 +1,38 @@ +// @requires-libnode +// @order: race +// Switch: work of every kind is pending in the program's queues when Node loads +// during the main body: timers (one an interval), an immediate, a tick, a +// microtask and a promise job (fs callbacks: s06). +import { callSync } from "@scriptc/node"; +import { writeSync } from "node:fs"; + +function log(s: string): void { + writeSync(1, `${s}\n`); +} + +function busy(ms: number): void { + const end = Date.now() + ms; + while (Date.now() < end) {} +} + +function island(src: string): void { + callSync("../island.cjs", "run", src); +} + +let fired = 0; +setTimeout(() => log("h-t5"), 5); +setTimeout(() => log("h-t0"), 0); +const interval = setInterval(() => { + log("h-iv"); + if (++fired === 3) clearInterval(interval); +}, 3); +setImmediate(() => log("h-imm")); +queueMicrotask(() => log("h-m")); +process.nextTick(() => log("h-k")); +void Promise.resolve().then(() => log("h-p")); +island("T 0 { L i-t0 } T 5 { L i-t5 } I { L i-imm } K { L i-k } M { L i-m } P { L i-p }"); +setTimeout(() => log("h-t0b"), 0); +setImmediate(() => log("h-imm2")); +queueMicrotask(() => log("h-m2")); +process.nextTick(() => log("h-k2")); +log("sync"); diff --git a/tests/corpus-node-islands-order/s02-load-in-timer-callback/main.ts b/tests/corpus-node-islands-order/s02-load-in-timer-callback/main.ts new file mode 100644 index 000000000..fff8853d5 --- /dev/null +++ b/tests/corpus-node-islands-order/s02-load-in-timer-callback/main.ts @@ -0,0 +1,35 @@ +// @requires-libnode +// @order: race +// Switch: Node loads inside a timer callback while timers of the same and later +// deadlines, an immediate and an interval are pending. +import { callSync } from "@scriptc/node"; +import { readFile, writeSync } from "node:fs"; + +function log(s: string): void { + writeSync(1, `${s}\n`); +} + +function busy(ms: number): void { + const end = Date.now() + ms; + while (Date.now() < end) {} +} + +function island(src: string): void { + callSync("../island.cjs", "run", src); +} + +let fired = 0; +const interval = setInterval(() => { + log("h-iv"); + if (++fired === 2) clearInterval(interval); +}, 1); +setTimeout(() => { + log("h-a"); + island("T 0 { L i-t0 } I { L i-imm } K { L i-k } M { L i-m }"); + queueMicrotask(() => log("h-a-m")); + process.nextTick(() => log("h-a-k")); + log("h-a-end"); +}, 1); +setTimeout(() => log("h-b"), 1); +setTimeout(() => log("h-c"), 4); +setImmediate(() => log("h-imm")); diff --git a/tests/corpus-node-islands-order/s03-load-in-immediate/main.ts b/tests/corpus-node-islands-order/s03-load-in-immediate/main.ts new file mode 100644 index 000000000..f4e453ab4 --- /dev/null +++ b/tests/corpus-node-islands-order/s03-load-in-immediate/main.ts @@ -0,0 +1,29 @@ +// @requires-libnode +// @order: race +// Switch: Node loads inside an immediate while later immediates of the same check +// phase, and timers, are pending. +import { callSync } from "@scriptc/node"; +import { readFile, writeSync } from "node:fs"; + +function log(s: string): void { + writeSync(1, `${s}\n`); +} + +function busy(ms: number): void { + const end = Date.now() + ms; + while (Date.now() < end) {} +} + +function island(src: string): void { + callSync("../island.cjs", "run", src); +} + +setImmediate(() => { + log("h-i1"); + island("I { L i-imm } T 0 { L i-t0 } K { L i-k }"); + setImmediate(() => log("h-i1-imm")); + process.nextTick(() => log("h-i1-k")); +}); +setImmediate(() => log("h-i2")); +setImmediate(() => log("h-i3")); +setTimeout(() => log("h-t0"), 0); diff --git a/tests/corpus-node-islands-order/s04-load-in-promise-job/main.ts b/tests/corpus-node-islands-order/s04-load-in-promise-job/main.ts new file mode 100644 index 000000000..fa6f30876 --- /dev/null +++ b/tests/corpus-node-islands-order/s04-load-in-promise-job/main.ts @@ -0,0 +1,35 @@ +// @requires-libnode +// Switch: Node loads in an async function's continuation while more promise +// jobs, ticks and timers are pending. +import { callSync } from "@scriptc/node"; +import { readFile, writeSync } from "node:fs"; + +function log(s: string): void { + writeSync(1, `${s}\n`); +} + +function busy(ms: number): void { + const end = Date.now() + ms; + while (Date.now() < end) {} +} + +function island(src: string): void { + callSync("../island.cjs", "run", src); +} + +async function main(): Promise { + log("h-start"); + await null; + log("h-after-await"); + queueMicrotask(() => log("h-m-before")); + island("P { L i-p } M { L i-m } K { L i-k } T 0 { L i-t0 }"); + queueMicrotask(() => log("h-m-after")); + await null; + log("h-after-await2"); + process.nextTick(() => log("h-k")); + await new Promise((resolve) => setTimeout(resolve, 2)); + log("h-after-timer"); +} +void Promise.resolve().then(() => log("h-p")); +setTimeout(() => log("h-t0"), 0); +void main(); diff --git a/tests/corpus-node-islands-order/s05-load-with-open-socket/main.ts b/tests/corpus-node-islands-order/s05-load-with-open-socket/main.ts new file mode 100644 index 000000000..fe8181754 --- /dev/null +++ b/tests/corpus-node-islands-order/s05-load-with-open-socket/main.ts @@ -0,0 +1,41 @@ +// @requires-libnode +// Switch: a TCP server and a client connection of the program are open when Node +// loads: their I/O continues on Node's loop. +import { callSync } from "@scriptc/node"; +import { readFile, writeSync } from "node:fs"; +import { createConnection, createServer, type Socket } from "node:net"; + +function log(s: string): void { + writeSync(1, `${s}\n`); +} + +function busy(ms: number): void { + const end = Date.now() + ms; + while (Date.now() < end) {} +} + +function island(src: string): void { + callSync("../island.cjs", "run", src); +} + +const server = createServer((socket: Socket) => { + socket.on("data", (data: Buffer) => { + log(`h-server-data ${data.toString()}`); + socket.end("pong"); + }); +}); +server.listen(0, "127.0.0.1", () => { + const address = server.address(); + const port = typeof address === "object" && address !== null ? address.port : 0; + log("h-listening"); + const client = createConnection(port, "127.0.0.1", () => { + log("h-connected"); + island("K { L i-k } M { L i-m } T 200 { L i-late }"); + client.write("ping"); + }); + client.on("data", (data: Buffer) => log(`h-client-data ${data.toString()}`)); + client.on("end", () => { + log("h-client-end"); + server.close(() => log("h-closed")); + }); +}); diff --git a/tests/corpus-node-islands-order/s06-load-with-pending-fs-and-unref/main.ts b/tests/corpus-node-islands-order/s06-load-with-pending-fs-and-unref/main.ts new file mode 100644 index 000000000..5ee2a96aa --- /dev/null +++ b/tests/corpus-node-islands-order/s06-load-with-pending-fs-and-unref/main.ts @@ -0,0 +1,30 @@ +// @requires-libnode +// @order: race +// Switch: fs callbacks and an unref'd timer are pending when Node loads; the +// unref'd timer must not keep the program alive, the refreshed one must. +import { callSync } from "@scriptc/node"; +import { readFile, writeSync } from "node:fs"; + +function log(s: string): void { + writeSync(1, `${s}\n`); +} + +function busy(ms: number): void { + const end = Date.now() + ms; + while (Date.now() < end) {} +} + +function island(src: string): void { + callSync("../island.cjs", "run", src); +} + +const unrefd = setTimeout(() => log("h-unref-never"), 50); +unrefd.unref(); +const refreshed = setTimeout(() => log("h-refreshed"), 5); +readFile(import.meta.filename, "utf8", () => { + log("h-io-1"); + readFile(import.meta.filename, "utf8", () => log("h-io-2")); +}); +island("T 2 { L i-t2 }"); +refreshed.refresh(); +log(`has-ref ${String(unrefd.hasRef())} ${String(refreshed.hasRef())}`); diff --git a/tests/corpus/6570-npm-builtin-function-bind/main.ts b/tests/corpus/6570-npm-builtin-function-bind/main.ts index f04dc2053..0318058e3 100644 --- a/tests/corpus/6570-npm-builtin-function-bind/main.ts +++ b/tests/corpus/6570-npm-builtin-function-bind/main.ts @@ -5,6 +5,10 @@ // fs.realpath 1.0.0 (ISC) and value-forms read builtin functions as values: // fs.realpathSync stored and compared, Buffer.alloc aliased, and a String // method read for a feature probe. +// package.json keeps bound-builtins native: a fence in its module code +// would run it in a Node island, and this program iterates the array its +// report() returns, which native code cannot iterate across the island +// boundary yet. import truncate from 'truncate-utf8-bytes'; import { report, badLength } from 'bound-builtins'; import realpath from 'fs.realpath'; diff --git a/tests/corpus/6570-npm-builtin-function-bind/package.json b/tests/corpus/6570-npm-builtin-function-bind/package.json new file mode 100644 index 000000000..84d59206f --- /dev/null +++ b/tests/corpus/6570-npm-builtin-function-bind/package.json @@ -0,0 +1 @@ +{ "type": "module", "scriptc": { "packages": { "bound-builtins": "native" } } } diff --git a/tests/corpus/6804-node-island-signals/main.mjs b/tests/corpus/6804-node-island-signals/main.mjs new file mode 100644 index 000000000..c6d91ad7b --- /dev/null +++ b/tests/corpus/6804-node-island-signals/main.mjs @@ -0,0 +1,64 @@ +// @requires-libnode +// Signal listeners that island code adds and removes join the program's own +// signal registry at their registration position: every listener of both +// sides runs in registration order, a once listener runs once, and removing +// an island listener leaves the program's handler for the same signal in +// place (libuv never owns the signal, so the removal does not reset it to +// its default disposition and the next signal does not kill the process). +// JavaScript, for the signal names the typed declarations do not cover. +import { callSync } from "@scriptc/node"; + +function island(name, signal, label) { + callSync("./process.cjs", name, signal, label); +} + +function later(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +async function main() { + const native = (name) => console.log(`native ${name}`); + process.on("SIGUSR2", native); + island("listen", "SIGUSR2", "b"); + island("listenOnce", "SIGUSR2", "once"); + const second = () => console.log("native second"); + process.on("SIGUSR2", second); + + process.kill(process.pid, "SIGUSR2"); + await later(50); + console.log("-- island removes b"); + island("unlisten", "SIGUSR2", "b"); + process.kill(process.pid, "SIGUSR2"); + await later(50); + + console.log("-- island adds and removes a SIGHUP listener"); + const hup = () => console.log("native SIGHUP"); + process.on("SIGHUP", hup); + island("listen", "SIGHUP", "hup"); + island("unlisten", "SIGHUP", "hup"); + process.kill(process.pid, "SIGHUP"); + await later(50); + + console.log("-- island listens to a signal the program does not"); + island("listen", "SIGUSR1", "usr1"); + process.kill(process.pid, "SIGUSR1"); + await later(50); + island("unlisten", "SIGUSR1", "usr1"); + + process.removeListener("SIGUSR2", native); + process.removeListener("SIGUSR2", second); + process.removeListener("SIGHUP", hup); + + console.log("-- one function registered once and on"); + island("listenOnce", "SIGUSR2", "dup"); + island("listen", "SIGUSR2", "dup"); + process.kill(process.pid, "SIGUSR2"); + await later(50); + console.log("-- the once registration is gone"); + process.kill(process.pid, "SIGUSR2"); + await later(50); + island("unlisten", "SIGUSR2", "dup"); + console.log("done"); +} + +void main(); diff --git a/tests/corpus/6804-node-island-signals/process.cjs b/tests/corpus/6804-node-island-signals/process.cjs new file mode 100644 index 000000000..cb9c9f03c --- /dev/null +++ b/tests/corpus/6804-node-island-signals/process.cjs @@ -0,0 +1,23 @@ +// Process events from island code (scope-B NI-3). +const listeners = new Map(); + +function listener(label) { + let fn = listeners.get(label); + if (fn === undefined) { + fn = (name, number) => console.log(`island ${label} ${name} ${typeof number}`); + listeners.set(label, fn); + } + return fn; +} + +exports.listen = (signal, label) => { + process.on(signal, listener(label)); +}; + +exports.listenOnce = (signal, label) => { + process.once(signal, listener(label)); +}; + +exports.unlisten = (signal, label) => { + process.removeListener(signal, listener(label)); +}; diff --git a/tests/corpus/6805-node-island-exit-code/main.ts b/tests/corpus/6805-node-island-exit-code/main.ts new file mode 100644 index 000000000..f57740850 --- /dev/null +++ b/tests/corpus/6805-node-island-exit-code/main.ts @@ -0,0 +1,15 @@ +// @exit: 3 +// @requires-libnode +// The exit code island code leaves in process.exitCode is the program's, +// and 'exit' listeners of both sides run in registration order with it. +import { callSync } from "@scriptc/node"; + +process.on("exit", (code) => console.log("native exit listener 1", code)); +callSync("./process.cjs", "onExit", "a"); +process.on("exit", (code) => console.log("native exit listener 2", code)); +callSync("./process.cjs", "onExit", "b"); +setTimeout(() => { + callSync("./process.cjs", "setExitCode", 3); + console.log("timer done"); +}, 5); +console.log("main done"); diff --git a/tests/corpus/6805-node-island-exit-code/process.cjs b/tests/corpus/6805-node-island-exit-code/process.cjs new file mode 100644 index 000000000..bf1d67e56 --- /dev/null +++ b/tests/corpus/6805-node-island-exit-code/process.cjs @@ -0,0 +1,7 @@ +exports.setExitCode = (code) => { + process.exitCode = code; +}; + +exports.onExit = (label) => { + process.on("exit", (code) => console.log(`island exit listener ${label} ${code}`)); +}; diff --git a/tests/corpus/6806-node-island-exit-in-call/main.ts b/tests/corpus/6806-node-island-exit-in-call/main.ts new file mode 100644 index 000000000..9fde1d86b --- /dev/null +++ b/tests/corpus/6806-node-island-exit-in-call/main.ts @@ -0,0 +1,22 @@ +// @exit: 4 +// @requires-libnode +// process.exit in island code, called from a timer while more timers and +// an interval are pending: the program ends at once with the code, after +// the 'exit' listeners of both sides ran in registration order. +import { callSync } from "@scriptc/node"; + +process.on("exit", (code) => console.log("native exit listener", code)); +callSync("./process.cjs", "onExit", "a"); +let intervals = 0; +const interval = setInterval(() => { + intervals++; +}, 1); +setTimeout(() => { + console.log("timer", intervals > 0); + callSync("./process.cjs", "exitNow", 4); + console.log("not reached"); +}, 10); +setTimeout(() => { + clearInterval(interval); + console.log("never"); +}, 50); diff --git a/tests/corpus/6806-node-island-exit-in-call/process.cjs b/tests/corpus/6806-node-island-exit-in-call/process.cjs new file mode 100644 index 000000000..4d8a24dc0 --- /dev/null +++ b/tests/corpus/6806-node-island-exit-in-call/process.cjs @@ -0,0 +1,8 @@ +exports.onExit = (label) => { + process.on("exit", (code) => console.log(`island exit listener ${label} ${code}`)); +}; + +exports.exitNow = (code) => { + console.log("island exits"); + process.exit(code); +}; diff --git a/tests/corpus/6807-node-island-exit-in-island-timer/main.ts b/tests/corpus/6807-node-island-exit-in-island-timer/main.ts new file mode 100644 index 000000000..ec8d5300b --- /dev/null +++ b/tests/corpus/6807-node-island-exit-in-island-timer/main.ts @@ -0,0 +1,17 @@ +// @exit: 5 +// @requires-libnode +// An island timer calls process.exit while the program's own timers keep +// the loop busy: the code and the exit listeners are the program's. +import { callSync } from "@scriptc/node"; + +process.on("exit", (code) => console.log("native exit listener", code)); +callSync("./process.cjs", "exitLater", 5, 20); +let ticks = 0; +const interval = setInterval(() => { + ticks++; +}, 2); +setTimeout(() => { + clearInterval(interval); + console.log("never"); +}, 200); +setTimeout(() => console.log("native timer before", ticks > 0), 10); diff --git a/tests/corpus/6807-node-island-exit-in-island-timer/process.cjs b/tests/corpus/6807-node-island-exit-in-island-timer/process.cjs new file mode 100644 index 000000000..321a0a7f1 --- /dev/null +++ b/tests/corpus/6807-node-island-exit-in-island-timer/process.cjs @@ -0,0 +1,7 @@ +exports.exitLater = (code, ms) => { + process.on("exit", (exitCode) => console.log(`island exit listener ${exitCode}`)); + setTimeout(() => { + console.log("island timer exits"); + process.exit(code); + }, ms); +}; diff --git a/tests/corpus/6808-node-island-exit-from-program/main.ts b/tests/corpus/6808-node-island-exit-from-program/main.ts new file mode 100644 index 000000000..bcd8e1b3b --- /dev/null +++ b/tests/corpus/6808-node-island-exit-from-program/main.ts @@ -0,0 +1,15 @@ +// @exit: 6 +// @requires-libnode +// The program's process.exit runs the 'exit' listeners island code +// registered, at their position, with its code; pending island work is +// dropped, as is anything the listeners schedule. +import { callSync } from "@scriptc/node"; + +callSync("./process.cjs", "onExit", "a"); +process.on("exit", (code) => console.log("native exit listener", code)); +callSync("./process.cjs", "onExit", "b"); +callSync("./process.cjs", "pending"); +setTimeout(() => { + console.log("exiting"); + process.exit(6); +}, 5); diff --git a/tests/corpus/6808-node-island-exit-from-program/process.cjs b/tests/corpus/6808-node-island-exit-from-program/process.cjs new file mode 100644 index 000000000..c6f0f242b --- /dev/null +++ b/tests/corpus/6808-node-island-exit-from-program/process.cjs @@ -0,0 +1,10 @@ +exports.onExit = (label) => { + process.on("exit", (code) => { + console.log(`island exit listener ${label} ${code}`); + process.nextTick(() => console.log("tick from an exit listener never runs")); + }); +}; + +exports.pending = () => { + setTimeout(() => console.log("island timer never fires"), 100); +}; diff --git a/tests/corpus/6809-node-island-stdio/main.ts b/tests/corpus/6809-node-island-stdio/main.ts new file mode 100644 index 000000000..90af5e253 --- /dev/null +++ b/tests/corpus/6809-node-island-stdio/main.ts @@ -0,0 +1,23 @@ +// @requires-libnode +// stdout and stderr are the same descriptors on both sides, and every +// write is complete when it returns, so the lines of both sides interleave +// in program order, large writes included. +import { callSync } from "@scriptc/node"; + +console.log("native log 1"); +callSync("./stdio.cjs", "write", "1"); +process.stdout.write("native write 1\n"); +console.error("native error 1"); +callSync("./stdio.cjs", "big", 200000); +process.stdout.write(`${"n".repeat(200000)}\n`); +// The island timer (50 ms) and the native timer (100 ms) are due well +// after this script and the immediates finish, on a loaded host too. +callSync("./stdio.cjs", "later", 50); +setTimeout(() => { + console.log("native timer log"); + callSync("./stdio.cjs", "write", "2"); + console.error("native timer error"); +}, 100); +setImmediate(() => console.log("native immediate log")); +process.on("exit", () => console.log("native exit listener log")); +console.log("native log 2"); diff --git a/tests/corpus/6809-node-island-stdio/stdio.cjs b/tests/corpus/6809-node-island-stdio/stdio.cjs new file mode 100644 index 000000000..87adbb12d --- /dev/null +++ b/tests/corpus/6809-node-island-stdio/stdio.cjs @@ -0,0 +1,21 @@ +exports.write = (label) => { + console.log(`island log ${label}`); + process.stdout.write(`island write ${label}\n`); + console.error(`island error ${label}`); +}; + +exports.big = (n) => { + process.stdout.write(`${"i".repeat(n)}\n`); +}; + +exports.later = (ms) => { + setTimeout(() => { + console.log("island timer log"); + console.error("island timer error"); + }, ms); + setImmediate(() => process.stdout.write("island immediate write\n")); + process.on("exit", () => { + console.log("island exit listener log"); + console.error("island exit listener error"); + }); +}; diff --git a/tests/corpus/6810-node-island-children/children.cjs b/tests/corpus/6810-node-island-children/children.cjs new file mode 100644 index 000000000..6d6579db6 --- /dev/null +++ b/tests/corpus/6810-node-island-children/children.cjs @@ -0,0 +1,9 @@ +const { execFile } = require("node:child_process"); + +exports.run = () => + new Promise((resolve, reject) => { + execFile("/bin/echo", ["island child"], (error, stdout) => { + if (error) reject(error); + else resolve(stdout.trim()); + }); + }); diff --git a/tests/corpus/6810-node-island-children/main.ts b/tests/corpus/6810-node-island-children/main.ts new file mode 100644 index 000000000..d59a63fb6 --- /dev/null +++ b/tests/corpus/6810-node-island-children/main.ts @@ -0,0 +1,27 @@ +// @requires-libnode +// Children of both sides: island code's first spawn installs libuv's SIGCHLD +// handler, and the program's own children are still reaped (scriptc waits +// for its pids itself, libuv only reaps its own). +import { call } from "@scriptc/node"; +import { spawn, spawnSync } from "node:child_process"; + +function nativeChild(): Promise { + return new Promise((resolve) => { + const child = spawn("/bin/sh", ["-c", "echo native child; exit 3"], { + stdio: ["ignore", "inherit", "inherit"], + }); + child.on("close", (code: number | null) => resolve(code ?? -1)); + }); +} + +async function main(): Promise { + console.log("island:", await call("./children.cjs", "run")); + console.log("native exit", await nativeChild()); + console.log("island again:", await call("./children.cjs", "run")); + const sync = spawnSync("/bin/sh", ["-c", "exit 4"]); + console.log("spawnSync status", sync.status); + const [a, b] = await Promise.all([nativeChild(), call("./children.cjs", "run")]); + console.log("together", a, b); +} + +void main(); diff --git a/tests/corpus/6811-node-island-uncaught-listener/island.cjs b/tests/corpus/6811-node-island-uncaught-listener/island.cjs new file mode 100644 index 000000000..650c7f3f3 --- /dev/null +++ b/tests/corpus/6811-node-island-uncaught-listener/island.cjs @@ -0,0 +1,4 @@ +exports.timers = () => { + setTimeout(() => console.log("island timer 1"), 15); + setTimeout(() => console.log("island timer 2"), 40); +}; diff --git a/tests/corpus/6811-node-island-uncaught-listener/main.mjs b/tests/corpus/6811-node-island-uncaught-listener/main.mjs new file mode 100644 index 000000000..9195e7baa --- /dev/null +++ b/tests/corpus/6811-node-island-uncaught-listener/main.mjs @@ -0,0 +1,20 @@ +// @requires-libnode +// The program's uncaughtException listener handles a throw from a native +// callback after Node loaded, and the loop goes on with the work of both +// sides. JavaScript: the typed declarations have no uncaughtException. +import { callSync } from "@scriptc/node"; + +process.on("uncaughtException", (error) => { + console.log("caught", error.message); +}); +callSync("./island.cjs", "timers"); +setTimeout(() => { + console.log("native timer throws"); + throw new Error("native boom"); +}, 5); +setTimeout(() => console.log("native timer after"), 10); +setImmediate(() => { + queueMicrotask(() => { + throw new Error("from a microtask"); + }); +}); diff --git a/tests/corpus/6812-node-island-handles/esm.mjs b/tests/corpus/6812-node-island-handles/esm.mjs new file mode 100644 index 000000000..0cb1a7d0e --- /dev/null +++ b/tests/corpus/6812-node-island-handles/esm.mjs @@ -0,0 +1,6 @@ +export const kind = "esm"; +export let counter = 0; +export function bump() { + return ++counter; +} +export default { name: "default export" }; diff --git a/tests/corpus/6812-node-island-handles/lib.cjs b/tests/corpus/6812-node-island-handles/lib.cjs new file mode 100644 index 000000000..073e93272 --- /dev/null +++ b/tests/corpus/6812-node-island-handles/lib.cjs @@ -0,0 +1,30 @@ +"use strict"; +const shared = { count: 0, items: [] }; + +class Point { + constructor(x, y) { + this.x = x; + this.y = y; + } + norm() { + return Math.hypot(this.x, this.y); + } +} + +exports.shared = shared; +exports.Point = Point; +exports.getShared = () => shared; +exports.bump = () => ++shared.count; +exports.read = (o, k) => o[k]; +exports.same = (a, b) => a === b; +exports.makeCounter = () => { + let n = 0; + return { inc: () => ++n, get value() { return n; } }; +}; +exports.add = (a, b) => a + b; +exports.numbers = () => [-0, NaN, Infinity, 2 ** 53 + 2, 10n ** 30n, -7n]; +exports.isNegZero = (x) => Object.is(x, -0); +exports.strings = () => ["plain", "wörld ☃ 🌍", "lone \ud800 high", "\udc00 lone low"]; +exports.codeUnits = (s) => Array.from(s, (c) => c.charCodeAt(0).toString(16)).join(" "); +exports.describe = (v) => `${typeof v} ${Object.prototype.toString.call(v)}`; +exports.keysOf = (o) => Object.keys(o).join(","); diff --git a/tests/corpus/6812-node-island-handles/main.mjs b/tests/corpus/6812-node-island-handles/main.mjs new file mode 100644 index 000000000..c999997fc --- /dev/null +++ b/tests/corpus/6812-node-island-handles/main.mjs @@ -0,0 +1,62 @@ +// @requires-libnode +// Island objects held by reference (scope-B NI-4): one handle per Node +// object, so identity, aliasing and mutation are shared with Node, while +// primitives, strings and BigInt cross as exact copies. +import { requireModule } from "@scriptc/node"; + +const lib = requireModule("./lib.cjs"); + +// Identity: the same Node object is the same value on both sides. +const a = lib.getShared(); +const b = lib.shared; +console.log("same handle", a === b, lib.same(a, b), lib.same(a, lib.getShared())); +console.log("distinct", lib.makeCounter() === lib.makeCounter()); + +// Mutation is visible on both sides. +a.count = 41; +console.log("island sees", lib.read(b, "count"), "bump", lib.bump(), "native sees", a.count); +a.items.push("x"); +lib.shared.items.push("y"); +console.log("items", a.items.length, b.items.join("+"), lib.read(lib.shared.items, 1)); +a.extra = { nested: true }; +console.log("keys", lib.keysOf(a), Object.keys(b).join(",")); +console.log("in", "extra" in a, "missing" in a); +delete a.extra; +console.log("deleted", "extra" in b, lib.keysOf(a)); + +// Closures and getters live in Node. +const counter = lib.makeCounter(); +counter.inc(); +counter.inc(); +console.log("counter", counter.inc(), counter.value); + +// Functions as values, typeof, classes, instanceof. +const add = lib.add; +console.log("call", add(2, 3), typeof add, typeof a, typeof lib.Point); +const p = new lib.Point(3, 4); +console.log("point", p.x, p.y, p.norm(), p instanceof lib.Point, a instanceof lib.Point); +console.log("describe", lib.describe(p), lib.describe(add)); + +// Exact numbers and BigInt. +const nums = lib.numbers(); +const negZero = nums[0]; +console.log("numbers", Object.is(negZero, -0), lib.isNegZero(negZero), Number.isNaN(nums[1]), nums[2], nums[3]); +console.log("bigint", nums[4], typeof nums[4], nums[4] + 1n, lib.add(nums[5], 2n)); +console.log("negzero in", lib.isNegZero(-0), lib.isNegZero(0)); + +// Strings, lone surrogates included. +const strs = lib.strings(); +for (let i = 0; i < 4; i++) { + const s = strs[i]; + console.log("string", s.length, lib.codeUnits(s) === lib.codeUnits(strs[i]), lib.same(s, strs[i])); +} +console.log("native string", lib.codeUnits("a\ud800b"), lib.add("x", "é")); + +// util.inspect of island objects is Node's own. +console.log(p); +console.log(counter, [1, 2]); +console.log("inspect", lib.makeCounter()); + +// ES modules load through require(esm): the namespace is a live handle. +const esm = requireModule("./esm.mjs"); +console.log("esm", esm.kind, esm.default.name, esm.bump(), esm.bump(), esm.counter); diff --git a/tests/corpus/6813-node-island-handle-errors/lib.cjs b/tests/corpus/6813-node-island-handle-errors/lib.cjs new file mode 100644 index 000000000..f613c33f5 --- /dev/null +++ b/tests/corpus/6813-node-island-handle-errors/lib.cjs @@ -0,0 +1,30 @@ +"use strict"; +class NotFound extends Error { + constructor(what) { + super(`${what} not found`); + this.name = "NotFound"; + this.code = "E_NOT_FOUND"; + } +} +let last; +exports.NotFound = NotFound; +exports.fail = (kind) => { + if (kind === "type") { + const e = new TypeError("bad type"); + e.code = "ERR_BAD"; + last = e; + throw e; + } + if (kind === "range") throw (last = new RangeError("out of range")); + if (kind === "custom") throw (last = new NotFound("thing")); + if (kind === "string") throw "just a string"; + if (kind === "object") throw (last = { reason: "plain object" }); + return "no throw"; +}; +exports.isLast = (e) => e === last; +exports.inspectError = (e) => + `${e instanceof Error} ${e.constructor.name} ${e.name} ${JSON.stringify(e.message)} ${e.code} ${typeof e.stack}`; +exports.rethrow = (e) => { + throw e; +}; +exports.call = (o, m) => o[m](); diff --git a/tests/corpus/6813-node-island-handle-errors/main.mjs b/tests/corpus/6813-node-island-handle-errors/main.mjs new file mode 100644 index 000000000..36a7e71fa --- /dev/null +++ b/tests/corpus/6813-node-island-handle-errors/main.mjs @@ -0,0 +1,61 @@ +// @requires-libnode +// Errors across the island boundary (scope-B NI-4): an island Error caught +// in the program keeps its class, name, message, code and stack, and is the +// same Node object when it crosses back; a program error passed into Node +// becomes a Node error with the same name, message and code. +import { requireModule } from "@scriptc/node"; + +const lib = requireModule("./lib.cjs"); + +for (const kind of ["type", "range", "custom"]) { + try { + lib.fail(kind); + } catch (e) { + console.log(kind, e instanceof Error, e instanceof TypeError, e instanceof RangeError); + console.log(" ", e.name, e.message, e.code); + console.log(" stack head", String(e.stack).split("\n")[0]); + console.log(" stack has island frame", String(e.stack).includes("lib.cjs")); + console.log(" same object in Node", lib.isLast(e), lib.inspectError(e)); + try { + lib.rethrow(e); + } catch (again) { + console.log(" rethrown", again === e, again.message); + } + } +} + +try { + lib.fail("string"); +} catch (e) { + console.log("string", typeof e, e); +} +try { + lib.fail("object"); +} catch (e) { + console.log("object", typeof e, e.reason, lib.isLast(e)); +} +console.log(lib.fail("none")); + +// Program errors crossing into Node. +const mine = new RangeError("from the program"); +console.log("native", lib.inspectError(mine)); +const coded = new Error("with code"); +coded.code = "E_MINE"; +console.log("native coded", lib.inspectError(coded)); +try { + lib.rethrow(new TypeError("round trip")); +} catch (e) { + console.log("round trip", e instanceof TypeError, e.message); +} + +// Errors from calling missing members. +try { + lib.call({}, "nope"); +} catch (e) { + console.log("missing method", e instanceof TypeError, e.message); +} +try { + lib.nope(); +} catch (e) { + console.log("missing export", e instanceof TypeError, e.message); +} diff --git a/tests/corpus/6814-node-island-handle-bytes/lib.cjs b/tests/corpus/6814-node-island-handle-bytes/lib.cjs new file mode 100644 index 000000000..04e516851 --- /dev/null +++ b/tests/corpus/6814-node-island-handle-bytes/lib.cjs @@ -0,0 +1,12 @@ +"use strict"; +let kept; +exports.makeBuffer = () => (kept = Buffer.from("hello")); +exports.kept = () => kept; +exports.read = (b) => `${b.constructor.name} ${Buffer.isBuffer(b)} [${Array.from(b).join(",")}]`; +exports.fill = (b, v) => { + b.fill(v); + return b.length; +}; +exports.same = (a, b) => a === b; +exports.floats = () => new Float64Array([1.5, -0, NaN]); +exports.sum = (arr) => arr.reduce((a, b) => a + b, 0); diff --git a/tests/corpus/6814-node-island-handle-bytes/main.mjs b/tests/corpus/6814-node-island-handle-bytes/main.mjs new file mode 100644 index 000000000..3decd1b12 --- /dev/null +++ b/tests/corpus/6814-node-island-handle-bytes/main.mjs @@ -0,0 +1,31 @@ +// @requires-libnode +// Byte stores across the island boundary (scope-B NI-4): a Buffer or typed +// array crossing either way shares its memory, so writes on one side are +// visible on the other without copies. +import { requireModule } from "@scriptc/node"; + +const lib = requireModule("./lib.cjs"); + +const buf = lib.makeBuffer(); +console.log("from island", buf.length, buf.toString(), Buffer.isBuffer(buf)); +buf[0] = 72 - 32 + 32; // 'H' +buf[1] = 69; +console.log("island sees native write", lib.read(lib.kept())); +lib.fill(lib.kept(), 120); +console.log("native sees island write", buf.toString(), buf[4]); +console.log("same buffer back", lib.same(buf, lib.kept()), buf === lib.kept()); + +const mine = new Uint8Array([1, 2, 3]); +console.log("into island", lib.read(mine)); +lib.fill(mine, 9); +console.log("native sees fill", mine.join(",")); + +const nb = Buffer.from("abc"); +lib.fill(nb, 122); +console.log("native buffer", nb.toString(), lib.read(nb)); + +const floats = lib.floats(); +console.log("floats", floats.length, floats[0], Object.is(floats[1], -0), Number.isNaN(floats[2])); +floats[0] = 2.5; +floats[2] = 4; +console.log("float sum in island", lib.sum(floats)); diff --git a/tests/corpus/6815-node-island-handle-promises/lib.cjs b/tests/corpus/6815-node-island-handle-promises/lib.cjs new file mode 100644 index 000000000..9c7d3707d --- /dev/null +++ b/tests/corpus/6815-node-island-handle-promises/lib.cjs @@ -0,0 +1,10 @@ +"use strict"; +let pending; +exports.later = (v, ms) => new Promise((resolve) => setTimeout(() => resolve(v), ms)); +exports.fails = async (m) => { + await null; + throw new RangeError(m); +}; +exports.object = async () => ({ answer: 42 }); +exports.keep = () => (pending = Promise.resolve("kept")); +exports.isKept = (p) => p === pending; diff --git a/tests/corpus/6815-node-island-handle-promises/main.mjs b/tests/corpus/6815-node-island-handle-promises/main.mjs new file mode 100644 index 000000000..8aa49850a --- /dev/null +++ b/tests/corpus/6815-node-island-handle-promises/main.mjs @@ -0,0 +1,24 @@ +// @requires-libnode +// Island promises as program promises (scope-B NI-4): awaiting one settles +// through a V8 reaction on the one hosted loop; rejections keep their +// error; the same Node promise is the same program promise. +import { requireModule } from "@scriptc/node"; + +const lib = requireModule("./lib.cjs"); + +async function main() { + console.log("later", await lib.later("value", 5)); + try { + await lib.fails("nope"); + } catch (e) { + console.log("rejected", e instanceof RangeError, e.message); + } + const o = await lib.object(); + console.log("object", o.answer, typeof o); + const p = lib.keep(); + console.log("promise identity", lib.isKept(p), await p); + const all = await Promise.all([lib.later(1, 3), lib.later(2, 1)]); + console.log("all", all.join(",")); +} + +main().then(() => console.log("done")); diff --git a/tests/corpus/6816-node-island-objects/main.ts b/tests/corpus/6816-node-island-objects/main.ts new file mode 100644 index 000000000..002613b33 --- /dev/null +++ b/tests/corpus/6816-node-island-objects/main.ts @@ -0,0 +1,84 @@ +// @requires-libnode +// An npm package scriptc refuses natively (its loader uses a computed +// require() specifier) runs in a Node island: its objects cross as +// handles, so the same object returned twice is ===, and writes through +// one reference are visible through every other, on both sides. +import { Store, createStore, describe, VERSION, makePoint, addPoints, ping, makeTricky, type Tricky } from "islandkit"; +import * as kit from "islandkit"; + +console.log(VERSION); +const s = createStore("fruits"); +const apple = s.add("apple", 3); +const banana = s.add("banana", 5); +console.log(s.name, s.size, s.total()); + +const again = s.get("apple"); +console.log("identity", apple === again, apple !== banana, again === s.first()); +console.log("owner", apple.owner === s, banana.owner === s); + +apple.value = 10; +console.log("alias", s.get("apple")!.value, s.total()); + +s.rename(apple, "apricot"); +console.log(apple.name, describe(apple), s.get("apricot") === apple); + +const missing = s.get("cherry"); +console.log("missing", missing === undefined); + +const veg = new Store("veg"); +veg.add("kale", 1); +console.log(veg.name, veg.size, veg instanceof Store, s instanceof kit.Store); +console.log(veg.first() !== null, new Store("empty").first() === null); + +const p = addPoints(makePoint(1, 2), { x: 3, y: 4 }); +console.log(p.x, p.y); +p.x = 100; +console.log(p.x + p.y); + +// JavaScript reads `tricky.method` before it evaluates `+swapper`, whose +// valueOf replaces the method: the call must reach the original. +class Swapper { + target: Tricky; + constructor(target: Tricky) { + this.target = target; + } + valueOf(): number { + this.target.method = this.target.other; + return 1; + } +} +const tricky = makeTricky(); +const swapper = new Swapper(tricky); +console.log("unary", tricky.method(+swapper), tricky.method(1)); + +console.log(ping(), ping(), kit.ping()); + +try { + s.add("", 1); +} catch (e) { + console.log("caught", (e as Error).message); +} + +function heaviest(store: Store, names: string[]): string { + let best = ""; + let max = -Infinity; + for (const n of names) { + const e = store.get(n); + if (e !== undefined && e.value > max) { + max = e.value; + best = e.name; + } + } + return best; +} +console.log("heaviest", heaviest(s, ["apricot", "banana", "cherry"])); + +// Native web streams and Node islands in one program select the async +// runtime build that defines both (SCR_NATIVE_REACTIONS and +// SCR_NODE_ISLANDS); without it the hosted-loop symbols are undefined. +const stream = new ReadableStream({ + start(controller) { + controller.close(); + }, +}); +console.log("stream", stream instanceof ReadableStream, stream.locked); diff --git a/tests/corpus/6816-node-island-objects/node_modules/islandkit/index.d.ts b/tests/corpus/6816-node-island-objects/node_modules/islandkit/index.d.ts new file mode 100644 index 000000000..7f1e9cf1c --- /dev/null +++ b/tests/corpus/6816-node-island-objects/node_modules/islandkit/index.d.ts @@ -0,0 +1,31 @@ +export interface Entry { + name: string; + value: number; + readonly owner: Store; +} +export interface Point { + x: number; + y: number; +} +export declare class Store { + constructor(name: string); + readonly name: string; + readonly size: number; + add(name: string, value: number): Entry; + get(name: string): Entry | undefined; + total(): number; + rename(entry: Entry, name: string): void; + first(): Entry | null; +} +export declare const VERSION: string; +export declare function createStore(name: string): Store; +export declare function describe(entry: Entry): string; +export declare function makePoint(x: number, y: number): Point; +export declare function addPoints(a: Point, b: Point): Point; +export declare let loads: number; +export declare function ping(): number; +export interface Tricky { + method(value: number): string; + other(): string; +} +export declare function makeTricky(): Tricky; diff --git a/tests/corpus/6816-node-island-objects/node_modules/islandkit/index.js b/tests/corpus/6816-node-island-objects/node_modules/islandkit/index.js new file mode 100644 index 000000000..613181936 --- /dev/null +++ b/tests/corpus/6816-node-island-objects/node_modules/islandkit/index.js @@ -0,0 +1,26 @@ +"use strict"; +// A small object API whose loader picks its parts with a computed +// require() specifier: scriptc cannot compile that natively (R1.2), so the +// package runs in a Node island. Its objects keep their identity across +// the boundary. +const { createRequire } = require("node:module"); +const load = createRequire(__filename); +const parts = ["./lib/format", "./lib/store"].map((p) => load(p)); +const [format, store] = parts; + +exports.VERSION = "islandkit " + require("./package.json").version; +exports.Store = store.Store; +exports.createStore = (name) => new store.Store(name); +exports.describe = format.describe; +exports.makePoint = (x, y) => ({ x, y }); +exports.addPoints = (a, b) => ({ x: a.x + b.x, y: a.y + b.y }); +exports.loads = 0; +exports.ping = () => ++exports.loads; +exports.makeTricky = () => ({ + method() { + return "original"; + }, + other() { + return "replaced"; + }, +}); diff --git a/tests/corpus/6816-node-island-objects/node_modules/islandkit/lib/format.js b/tests/corpus/6816-node-island-objects/node_modules/islandkit/lib/format.js new file mode 100644 index 000000000..1deca0e36 --- /dev/null +++ b/tests/corpus/6816-node-island-objects/node_modules/islandkit/lib/format.js @@ -0,0 +1,2 @@ +"use strict"; +exports.describe = (entry) => `${entry.name}=${entry.value} in ${entry.owner.name}`; diff --git a/tests/corpus/6816-node-island-objects/node_modules/islandkit/lib/store.js b/tests/corpus/6816-node-island-objects/node_modules/islandkit/lib/store.js new file mode 100644 index 000000000..e0aa6c897 --- /dev/null +++ b/tests/corpus/6816-node-island-objects/node_modules/islandkit/lib/store.js @@ -0,0 +1,33 @@ +"use strict"; +class Store { + constructor(name) { + this.name = name; + this.entries = new Map(); + } + get size() { + return this.entries.size; + } + add(name, value) { + if (name === "") throw new TypeError("entry names must not be empty"); + const entry = { name, value, owner: this }; + this.entries.set(name, entry); + return entry; + } + get(name) { + return this.entries.get(name); + } + total() { + let sum = 0; + for (const e of this.entries.values()) sum += e.value; + return sum; + } + rename(entry, name) { + this.entries.delete(entry.name); + entry.name = name; + this.entries.set(name, entry); + } + first() { + return this.entries.values().next().value ?? null; + } +} +exports.Store = Store; diff --git a/tests/corpus/6816-node-island-objects/node_modules/islandkit/package.json b/tests/corpus/6816-node-island-objects/node_modules/islandkit/package.json new file mode 100644 index 000000000..e0ee16f46 --- /dev/null +++ b/tests/corpus/6816-node-island-objects/node_modules/islandkit/package.json @@ -0,0 +1,6 @@ +{ + "name": "islandkit", + "version": "1.2.0", + "main": "index.js", + "types": "index.d.ts" +} diff --git a/tests/corpus/6817-node-island-esm/main.ts b/tests/corpus/6817-node-island-esm/main.ts new file mode 100644 index 000000000..ffe3a6122 --- /dev/null +++ b/tests/corpus/6817-node-island-esm/main.ts @@ -0,0 +1,27 @@ +// @requires-libnode +// An ES module package scriptc refuses natively (it imports its parts +// with a computed specifier) runs in a Node island: named imports are live +// bindings, the default export and the namespace are the module's own, and +// a later import() answers the same module instance. +import greet, { count, bump, Counter, loadPart, shared, hit } from "esmisland"; +import * as ns from "esmisland"; + +console.log(greet("island"), count); +bump(); +bump(); +console.log("live", count, ns.count); + +const c = new Counter(5); +c.inc(); +console.log(c.value, c instanceof Counter, c instanceof ns.Counter); +console.log(hit() === shared, shared.hits, ns.shared === shared); + +async function main(): Promise { + const label = await loadPart("a"); + console.log(label); + const again = await import("esmisland"); + console.log(again.count, again.shared === shared, again.Counter === Counter); + again.bump(); + console.log("after", count); +} +void main(); diff --git a/tests/corpus/6817-node-island-esm/node_modules/esmisland/index.d.ts b/tests/corpus/6817-node-island-esm/node_modules/esmisland/index.d.ts new file mode 100644 index 000000000..0dba93d26 --- /dev/null +++ b/tests/corpus/6817-node-island-esm/node_modules/esmisland/index.d.ts @@ -0,0 +1,14 @@ +export declare let count: number; +export declare function bump(): number; +export declare class Counter { + constructor(start: number); + value: number; + inc(): number; +} +export default function greet(name: string): string; +export declare function loadPart(name: string): Promise; +export interface Shared { + hits: number; +} +export declare const shared: Shared; +export declare function hit(): Shared; diff --git a/tests/corpus/6817-node-island-esm/node_modules/esmisland/index.js b/tests/corpus/6817-node-island-esm/node_modules/esmisland/index.js new file mode 100644 index 000000000..f9cfab3f2 --- /dev/null +++ b/tests/corpus/6817-node-island-esm/node_modules/esmisland/index.js @@ -0,0 +1,29 @@ +// An ES module whose parts load through import() with a computed +// specifier: scriptc cannot compile that natively (R1.2), so it runs in a +// Node island. Its live bindings, classes and shared objects keep their +// meaning across the boundary. +export let count = 0; +export function bump() { + count++; + return count; +} +export class Counter { + constructor(start) { + this.value = start; + } + inc() { + return ++this.value; + } +} +export default function greet(name) { + return `hello ${name}`; +} +export async function loadPart(name) { + const part = await import(`./parts/${name}.js`); + return part.label; +} +export const shared = { hits: 0 }; +export function hit() { + shared.hits++; + return shared; +} diff --git a/tests/corpus/6817-node-island-esm/node_modules/esmisland/package.json b/tests/corpus/6817-node-island-esm/node_modules/esmisland/package.json new file mode 100644 index 000000000..6e2b2d9ec --- /dev/null +++ b/tests/corpus/6817-node-island-esm/node_modules/esmisland/package.json @@ -0,0 +1,11 @@ +{ + "name": "esmisland", + "version": "0.3.0", + "type": "module", + "exports": { + ".": { + "types": "./index.d.ts", + "default": "./index.js" + } + } +} diff --git a/tests/corpus/6817-node-island-esm/node_modules/esmisland/parts/a.js b/tests/corpus/6817-node-island-esm/node_modules/esmisland/parts/a.js new file mode 100644 index 000000000..10721ccaa --- /dev/null +++ b/tests/corpus/6817-node-island-esm/node_modules/esmisland/parts/a.js @@ -0,0 +1 @@ +export const label = "part a"; diff --git a/tests/corpus/6822-node-island-gc/alloc.cjs b/tests/corpus/6822-node-island-gc/alloc.cjs new file mode 100644 index 000000000..cfa1be734 --- /dev/null +++ b/tests/corpus/6822-node-island-gc/alloc.cjs @@ -0,0 +1,20 @@ +// Allocates short-lived objects in rounds and keeps one per round alive +// across calls, so collections run inside the calls and must keep it. +const keep = []; + +exports.churn = function (rounds) { + let count = 0; + for (let r = 0; r < rounds; r++) { + const batch = []; + for (let i = 0; i < 200000; i++) batch.push({ i, s: "x" + i }); + count += batch.length; + keep.push(batch[r]); + } + return count; +}; + +exports.kept = function () { + let sum = 0; + for (const item of keep) sum += item.i; + return keep.length * 1000000 + sum; +}; diff --git a/tests/corpus/6822-node-island-gc/main.ts b/tests/corpus/6822-node-island-gc/main.ts new file mode 100644 index 000000000..22d984a59 --- /dev/null +++ b/tests/corpus/6822-node-island-gc/main.ts @@ -0,0 +1,24 @@ +// @requires-libnode +// Garbage collections inside calls into Node.js: calls that allocate enough +// for V8 to collect while JavaScript runs, from the main program and from an +// async function that a timer on Node's loop resumes. +import { call, callSync } from "@scriptc/node"; + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +async function main(): Promise { + console.log("sync", callSync("./alloc.cjs", "churn", 20)); + console.log("async", await call("./alloc.cjs", "churn", 10)); + for (let i = 0; i < 3; i++) { + await sleep(1); + console.log("after timer", i, callSync("./alloc.cjs", "churn", 20)); + } + const a = call("./alloc.cjs", "churn", 5); + const b = call("./alloc.cjs", "churn", 5); + console.log("both", await a, await b); + console.log("kept", callSync("./alloc.cjs", "kept")); +} + +main(); diff --git a/tests/corpus/6823-node-island-nested-stack/echo.cjs b/tests/corpus/6823-node-island-nested-stack/echo.cjs new file mode 100644 index 000000000..f74a40e4e --- /dev/null +++ b/tests/corpus/6823-node-island-nested-stack/echo.cjs @@ -0,0 +1,8 @@ +// Returns its argument after using some of Node's stack. +function down(n, depth) { + return depth === 0 ? n : down(n, depth - 1); +} + +exports.echo = function (n) { + return down(n, 500); +}; diff --git a/tests/corpus/6823-node-island-nested-stack/main.ts b/tests/corpus/6823-node-island-nested-stack/main.ts new file mode 100644 index 000000000..f9539187f --- /dev/null +++ b/tests/corpus/6823-node-island-nested-stack/main.ts @@ -0,0 +1,26 @@ +// @requires-libnode +// A timer on Node's loop runs runtime code deep below the callback's own +// frame, and an async function it starts calls into Node.js before its +// first await: that call's frames go below the deepest live runtime frame +// on Node's stack, so the frames above it return intact. +import { callSync } from "@scriptc/node"; + +async function viaNode(n: number): Promise { + const echoed = callSync("./echo.cjs", "echo", n); + await null; + return echoed; +} + +const started: Promise[] = []; +function deep(depth: number, n: number): number { + const local = [depth, n, depth * n]; + if (depth > 0) return deep(depth - 1, n) + local[2]!; + started.push(viaNode(n)); + return local[0]! + local[1]!; +} + +console.log("loaded", callSync("./echo.cjs", "echo", 1)); +setTimeout(() => { + console.log("deep", deep(200, 3)); + void Promise.all(started).then((values) => console.log("echoed", values.join(","))); +}, 0); diff --git a/tests/corpus/6839-node-island-path-mapped/main.ts b/tests/corpus/6839-node-island-path-mapped/main.ts new file mode 100644 index 000000000..603392d02 --- /dev/null +++ b/tests/corpus/6839-node-island-path-mapped/main.ts @@ -0,0 +1,21 @@ +// @requires-libnode +// tsconfig `paths` maps an island package to its installed directory (a +// monorepo pinning one copy): its import resolves to the package's +// declarations, which type the program, and the package loads in Node. +import { Paths, heavier, route, type Route } from "pathkit"; + +interface Table { + title: string; + rows: T[]; +} + +function render(table: Table): string { + return `${table.title}: ${table.rows.map((r) => r.label()).join(", ")}`; +} + +const routes: Route[] = [route("/a", 1), route("/b", 3), route("/c", 2)]; +const table: Table = { title: "routes", rows: routes }; +console.log(render(table)); +const top = heavier(routes[0]!, routes[1]!); +console.log(top.rank, top.label(), top === routes[1], routes.length); +console.log(Paths.join("api", "v1")); diff --git a/tests/corpus/6839-node-island-path-mapped/node_modules/pathkit/build/index.d.mts b/tests/corpus/6839-node-island-path-mapped/node_modules/pathkit/build/index.d.mts new file mode 100644 index 000000000..aa3c28f80 --- /dev/null +++ b/tests/corpus/6839-node-island-path-mapped/node_modules/pathkit/build/index.d.mts @@ -0,0 +1,10 @@ +export interface Route { + readonly path: string; + readonly weight: number; + label(): string; +} +export type Weighted = T & { readonly rank: number }; +export declare function route(path: string, weight: number): Route; +export declare function heavier(a: T, b: T): Weighted; +import * as Paths from "./paths.mjs"; +export { Paths }; diff --git a/tests/corpus/6839-node-island-path-mapped/node_modules/pathkit/build/index.mjs b/tests/corpus/6839-node-island-path-mapped/node_modules/pathkit/build/index.mjs new file mode 100644 index 000000000..04b3bb548 --- /dev/null +++ b/tests/corpus/6839-node-island-path-mapped/node_modules/pathkit/build/index.mjs @@ -0,0 +1,13 @@ +// The loader requires its part with spread arguments, which scriptc does +// not compile natively: the package runs in a Node island. +import { createRequire } from "node:module"; +const load = createRequire(import.meta.url); +const { Route } = load(...["./route.cjs"]); +export function route(path, weight) { + return new Route(path, weight); +} +export function heavier(a, b) { + return Object.assign(b.weight > a.weight ? b : a, { rank: 1 }); +} +import * as Paths from "./paths.mjs"; +export { Paths }; diff --git a/tests/corpus/6839-node-island-path-mapped/node_modules/pathkit/build/paths.d.mts b/tests/corpus/6839-node-island-path-mapped/node_modules/pathkit/build/paths.d.mts new file mode 100644 index 000000000..9d19dad8e --- /dev/null +++ b/tests/corpus/6839-node-island-path-mapped/node_modules/pathkit/build/paths.d.mts @@ -0,0 +1 @@ +export declare function join(...parts: string[]): string; diff --git a/tests/corpus/6839-node-island-path-mapped/node_modules/pathkit/build/paths.mjs b/tests/corpus/6839-node-island-path-mapped/node_modules/pathkit/build/paths.mjs new file mode 100644 index 000000000..e343ff3d2 --- /dev/null +++ b/tests/corpus/6839-node-island-path-mapped/node_modules/pathkit/build/paths.mjs @@ -0,0 +1,3 @@ +export function join(...parts) { + return "/" + parts.join("/"); +} diff --git a/tests/corpus/6839-node-island-path-mapped/node_modules/pathkit/build/route.cjs b/tests/corpus/6839-node-island-path-mapped/node_modules/pathkit/build/route.cjs new file mode 100644 index 000000000..0d53a5a65 --- /dev/null +++ b/tests/corpus/6839-node-island-path-mapped/node_modules/pathkit/build/route.cjs @@ -0,0 +1,11 @@ +"use strict"; +class Route { + constructor(path, weight) { + this.path = path; + this.weight = weight; + } + label() { + return this.path + "@" + this.weight; + } +} +exports.Route = Route; diff --git a/tests/corpus/6839-node-island-path-mapped/node_modules/pathkit/package.json b/tests/corpus/6839-node-island-path-mapped/node_modules/pathkit/package.json new file mode 100644 index 000000000..f39f36eb8 --- /dev/null +++ b/tests/corpus/6839-node-island-path-mapped/node_modules/pathkit/package.json @@ -0,0 +1,7 @@ +{ + "name": "pathkit", + "version": "2.0.0", + "type": "module", + "types": "./build/index.d.mts", + "exports": { ".": { "import": "./build/index.mjs", "default": "./build/index.mjs" } } +} diff --git a/tests/corpus/6839-node-island-path-mapped/tsconfig.json b/tests/corpus/6839-node-island-path-mapped/tsconfig.json new file mode 100644 index 000000000..aa3b6fe0b --- /dev/null +++ b/tests/corpus/6839-node-island-path-mapped/tsconfig.json @@ -0,0 +1,8 @@ +{ + "compilerOptions": { + "strict": true, + "paths": { + "pathkit": ["./node_modules/pathkit"] + } + } +} diff --git a/tests/corpus/6890-node-island-timer-refresh/echo.cjs b/tests/corpus/6890-node-island-timer-refresh/echo.cjs new file mode 100644 index 000000000..76d5cde1d --- /dev/null +++ b/tests/corpus/6890-node-island-timer-refresh/echo.cjs @@ -0,0 +1,3 @@ +exports.echo = function (n) { + return n; +}; diff --git a/tests/corpus/6890-node-island-timer-refresh/main.ts b/tests/corpus/6890-node-island-timer-refresh/main.ts new file mode 100644 index 000000000..3f5b68ee5 --- /dev/null +++ b/tests/corpus/6890-node-island-timer-refresh/main.ts @@ -0,0 +1,15 @@ +// @requires-libnode +// refresh() of a timer armed before Node loads restarts it with the delay +// it was created with, not the remaining delay it had at handoff: a due +// timer it refreshes does not run before Node's loop starts. The timer +// armed after the refresh fires first. +import { callSync } from "@scriptc/node"; + +const slow = setTimeout(() => console.log("slow"), 1000); +// Most of the timer's delay passes before Node loads, so its runtime entry +// reaches the hosted loop with little time left (or already due). +const start = Date.now(); +while (Date.now() - start < 800) {} +console.log("loaded", callSync("./echo.cjs", "echo", 1)); +slow.refresh(); +setTimeout(() => console.log("fast"), 400); diff --git a/tests/corpus/992-fs-roundtrip.ts b/tests/corpus/992-fs-roundtrip.ts index 92b527fb1..6ac7398b4 100644 --- a/tests/corpus/992-fs-roundtrip.ts +++ b/tests/corpus/992-fs-roundtrip.ts @@ -2,7 +2,9 @@ // inside a scratch directory derived from THIS process's own identity: the // harness runs Node and the native binary CONCURRENTLY in one cwd, and // argv[1]'s trailing segment differs between the two (script name vs binary -// name), so the sides never touch the same paths. +// name), so the sides never touch the same paths. The pid keeps two Node +// oracles apart too: the differential and LLVM differential suites can run +// the same program at once in one shard. import { appendFileSync, existsSync, diff --git a/tests/corpus/generic-conditional-member-bindings.ts b/tests/corpus/generic-conditional-member-bindings.ts new file mode 100644 index 000000000..60631e191 --- /dev/null +++ b/tests/corpus/generic-conditional-member-bindings.ts @@ -0,0 +1,100 @@ +// Members typed by a distributive conditional over a generic parameter +// (`opts?: T extends "alpha" ? AlphaOpts : ...`), read inside the generic +// body: the instance's binding decides the layout, member by member of a +// literal union, and an all-never binding leaves an optional member that +// holds only undefined, as the concrete instantiation's member does. + +type KnownApi = "alpha" | "beta" | "gamma"; +type Api = KnownApi | (string & {}); + +interface AlphaOpts { + fast?: boolean; +} +interface BetaOpts { + depth?: number; +} + +interface Base { + id: string; + api: TApi; +} + +interface Item extends Base { + kind?: "item"; + opts?: TApi extends "alpha" ? AlphaOpts : TApi extends "beta" ? BetaOpts : never; +} + +interface Other extends Base { + kind: "other"; + size: number; +} + +type AnyItem = Item | Other; + +interface Options { + name: string; + items: readonly AnyItem[]; +} + +interface Built { + name: string; + all: () => readonly AnyItem[]; + chat: () => Item[]; + add: (item: AnyItem) => void; +} + +function describe(item: Item): string { + return `${item.id}:${item.api}:${item.opts === undefined ? "-" : JSON.stringify(item.opts)}`; +} + +function build(input: Options): Built { + const baseline = input.items; + let dynamic: readonly AnyItem[] = []; + const current = (): readonly AnyItem[] => { + const merged = [...baseline]; + for (const item of dynamic) merged.push(item); + return merged; + }; + return { + name: input.name, + all: current, + chat: () => current().filter((item): item is Item => item.kind !== "other"), + add: (item) => { + dynamic = [...dynamic, item]; + }, + }; +} + +function report(built: Built): void { + console.log(`${built.name}: ${built.all().length} items, ${built.chat().length} chat`); + for (const item of built.chat()) console.log(` ${describe(item)}`); +} + +const mixed: Built<"alpha" | "gamma"> = build({ + name: "mixed", + items: [ + { id: "a1", api: "alpha", opts: { fast: true } }, + { id: "g1", api: "gamma" }, + ], +}); +mixed.add({ id: "a2", api: "alpha" }); +report(mixed); + +const beta: Built<"beta"> = build({ + name: "beta", + items: [ + { id: "b1", api: "beta", opts: { depth: 2 } }, + { id: "o1", api: "other", kind: "other", size: 1 }, + ], +}); +beta.add({ id: "o2", api: "other", kind: "other", size: 3 }); +report(beta); + +const custom = build({ name: "custom", items: [{ id: "c1", api: "custom" }] }); +custom.add({ id: "c2", api: "custom" }); +report(custom); + +const open = build({ name: "open", items: [{ id: "x1", api: "external" }] }); +report(open); +const first = open.chat()[0]; +console.log(first === undefined ? "none" : first.opts === undefined ? "no opts" : "opts"); diff --git a/tests/corpus/generic-conditional-member-units.ts b/tests/corpus/generic-conditional-member-units.ts new file mode 100644 index 000000000..5899f6d35 --- /dev/null +++ b/tests/corpus/generic-conditional-member-units.ts @@ -0,0 +1,75 @@ +// Members typed by a distributive conditional over a generic parameter whose +// binding selects `unknown`, `{}`, `undefined` or `null` for some members, +// read inside the generic body. The checker's union of the results decides the +// layout: `unknown` and `{}` absorb the other results, and `undefined` and +// `null` are the union's unit arms, as in the concrete instantiation. + +type Kind = "alpha" | "beta" | "gamma" | (string & {}); + +interface BetaOpts { + depth: number; +} + +interface Loose { + kind: K; + meta: K extends "alpha" ? unknown : K extends "beta" ? BetaOpts : never; +} + +interface Empty { + kind: K; + meta: K extends "alpha" ? {} : K extends "beta" ? BetaOpts : never; +} + +interface Units { + kind: K; + meta: K extends "alpha" ? undefined : K extends "beta" ? BetaOpts : K extends "gamma" ? null : never; +} + +function looseMetas(items: readonly Loose[]): string { + let out = ""; + for (const item of items) out += `${item.kind}:${JSON.stringify(item.meta)} `; + return out; +} + +function emptyMetas(items: readonly Empty[]): string { + let out = ""; + for (const item of items) { + // The local holds the member's type, so the `{}` arm's mapping is + // emitted (a bare read would convert to dyn at the use site). + const meta = item.meta; + out += `${item.kind}:${JSON.stringify(meta)} `; + } + return out; +} + +function unitMetas(items: readonly Units[]): string { + let out = ""; + for (const item of items) { + const meta = item.meta; + out += `${item.kind}:${meta === undefined ? "u" : meta === null ? "n" : meta.depth} `; + } + return out; +} + +const loose: Loose<"alpha" | "beta">[] = [ + { kind: "alpha", meta: "free" }, + { kind: "alpha", meta: [1, 2] }, + { kind: "beta", meta: { depth: 2 } }, +]; +console.log(looseMetas(loose)); + +const empty: Empty<"alpha" | "beta">[] = [ + { kind: "alpha", meta: { free: true } }, + { kind: "alpha", meta: "text" }, + { kind: "beta", meta: { depth: 4 } }, +]; +console.log(emptyMetas(empty)); +console.log(JSON.stringify(empty)); + +const units: Units<"alpha" | "beta" | "gamma">[] = [ + { kind: "alpha", meta: undefined }, + { kind: "beta", meta: { depth: 3 } }, + { kind: "gamma", meta: null }, +]; +console.log(unitMetas(units)); +console.log(JSON.stringify(units)); diff --git a/tests/fixtures/node-island-process/island-rejection.ts b/tests/fixtures/node-island-process/island-rejection.ts new file mode 100644 index 000000000..0d809aae9 --- /dev/null +++ b/tests/fixtures/node-island-process/island-rejection.ts @@ -0,0 +1,6 @@ +// An island promise rejects unhandled: exit 1 with Node's report. +import { callSync } from "@scriptc/node"; + +process.on("exit", (code) => console.log("native exit listener", code)); +callSync("./island.cjs", "rejectLater", 5); +setTimeout(() => console.log("native timer never"), 50); diff --git a/tests/fixtures/node-island-process/island-uncaught.ts b/tests/fixtures/node-island-process/island-uncaught.ts new file mode 100644 index 000000000..ae54321a5 --- /dev/null +++ b/tests/fixtures/node-island-process/island-uncaught.ts @@ -0,0 +1,7 @@ +// An island timer throws: Node's uncaught report, exit 1, and the program's +// 'exit' listener with the code. +import { callSync } from "@scriptc/node"; + +process.on("exit", (code) => console.log("native exit listener", code)); +callSync("./island.cjs", "throwLater", 5); +setTimeout(() => console.log("native timer never"), 50); diff --git a/tests/fixtures/node-island-process/island.cjs b/tests/fixtures/node-island-process/island.cjs new file mode 100644 index 000000000..4de8186b6 --- /dev/null +++ b/tests/fixtures/node-island-process/island.cjs @@ -0,0 +1,20 @@ +// The island half of the process-integration fixtures (scope-B NI-3). +exports.onExit = (label) => { + process.on("exit", (code) => console.log(`island exit listener ${label} ${code}`)); +}; + +exports.throwLater = (ms) => { + setTimeout(() => { + throw new Error("island boom"); + }, ms); +}; + +exports.rejectLater = (ms) => { + setTimeout(() => { + Promise.reject(new Error("island rejected")); + }, ms); +}; + +exports.timer = (ms, label) => { + setTimeout(() => console.log(`island timer ${label}`), ms); +}; diff --git a/tests/fixtures/node-island-process/native-rejection.ts b/tests/fixtures/node-island-process/native-rejection.ts new file mode 100644 index 000000000..e320309ed --- /dev/null +++ b/tests/fixtures/node-island-process/native-rejection.ts @@ -0,0 +1,10 @@ +// A native promise rejects unhandled after Node loaded: exit 1 with the +// report, the island's 'exit' listener runs. +import { callSync } from "@scriptc/node"; + +callSync("./island.cjs", "onExit", "a"); +setTimeout(() => { + console.log("native timer"); + void Promise.reject(new Error("native rejected")); +}, 5); +setTimeout(() => console.log("native timer never"), 50); diff --git a/tests/fixtures/node-island-process/native-uncaught.ts b/tests/fixtures/node-island-process/native-uncaught.ts new file mode 100644 index 000000000..2f6053e55 --- /dev/null +++ b/tests/fixtures/node-island-process/native-uncaught.ts @@ -0,0 +1,10 @@ +// A native timer throws after Node loaded: exit 1, the uncaught report, and +// the island's 'exit' listener with the code. +import { callSync } from "@scriptc/node"; + +callSync("./island.cjs", "onExit", "a"); +callSync("./island.cjs", "timer", 50, "never"); +setTimeout(() => { + console.log("native timer"); + throw new Error("native boom"); +}, 5); diff --git a/tests/fixtures/node-island/builtins.ts b/tests/fixtures/node-island/builtins.ts new file mode 100644 index 000000000..d83eea696 --- /dev/null +++ b/tests/fixtures/node-island/builtins.ts @@ -0,0 +1,7 @@ +// requireModule() of Node's built-in modules, named with and without the +// node: prefix. +import { requireModule } from "@scriptc/node"; + +const path = requireModule("path"); +const os = requireModule("node:os"); +console.log(path.join("a", "b"), typeof os.platform()); diff --git a/tests/fixtures/node-island/handles-leak.cjs b/tests/fixtures/node-island/handles-leak.cjs new file mode 100644 index 000000000..9c99f90c9 --- /dev/null +++ b/tests/fixtures/node-island/handles-leak.cjs @@ -0,0 +1,7 @@ +"use strict"; +exports.make = (i) => ({ i, payload: new Array(16).fill(i), tag: `object ${i}` }); +exports.buffer = (i) => Buffer.alloc(64, i & 0xff); +exports.fail = (i) => { + throw new Error(`failure ${i}`); +}; +exports.later = async (i) => i; diff --git a/tests/fixtures/node-island/handles-leak.mjs b/tests/fixtures/node-island/handles-leak.mjs new file mode 100644 index 000000000..1a732614f --- /dev/null +++ b/tests/fixtures/node-island/handles-leak.mjs @@ -0,0 +1,25 @@ +// Island objects the program drops are released (scope-B NI-4, acyclic +// garbage): after many iterations that each take handles to fresh Node +// objects, buffers, errors and promises, the import table holds only what +// the program still references. node-island-handles.test.ts reads the +// count SCRIPTC_NODE_ISLANDS_TRACE=handles prints at exit. +import { requireModule } from "@scriptc/node"; + +const lib = requireModule("./handles-leak.cjs"); +const n = Number(process.argv[2] ?? "20000"); +let sum = 0; +for (let i = 0; i < n; i++) { + const o = lib.make(i); + sum += o.i + o.payload.length; + const b = lib.buffer(i); + sum += b[0]; + try { + lib.fail(i); + } catch (e) { + sum += e.message.length; + } +} +let awaited = 0; +for (let i = 0; i < 1000; i++) awaited += await lib.later(i); +const kept = lib.make(-1); +console.log("sum", sum, awaited, kept.tag); diff --git a/tests/fixtures/node-island/order/main.ts b/tests/fixtures/node-island/order/main.ts new file mode 100644 index 000000000..3a58de977 --- /dev/null +++ b/tests/fixtures/node-island/order/main.ts @@ -0,0 +1,26 @@ +// JavaScript's evaluation order on island objects: a method call reads the +// method before its arguments run, and an assignment evaluates the receiver +// and the key before the value. +import { make, type Thing } from "orderkit"; + +const o = make(); +// The argument replaces the method; the call still runs the one it read. +console.log((o.m as (...args: unknown[]) => string)((o.m = 0), 1)); + +const trace: string[] = []; +const target = make(); +function receiver(): Thing { + trace.push("receiver"); + return target; +} +function key(): string { + trace.push("key"); + return "k"; +} +function value(): number { + trace.push("value"); + return 1; +} +receiver()[key()] = value(); +receiver().p = value(); +console.log(trace.join(","), target["k"], target["p"]); diff --git a/tests/fixtures/node-island/order/node_modules/orderkit/index.d.ts b/tests/fixtures/node-island/order/node_modules/orderkit/index.d.ts new file mode 100644 index 000000000..aa7722ec0 --- /dev/null +++ b/tests/fixtures/node-island/order/node_modules/orderkit/index.d.ts @@ -0,0 +1,5 @@ +export declare class Thing { + m: ((...args: unknown[]) => string) | number; + [key: string]: unknown; +} +export declare function make(): Thing; diff --git a/tests/fixtures/node-island/order/node_modules/orderkit/index.js b/tests/fixtures/node-island/order/node_modules/orderkit/index.js new file mode 100644 index 000000000..9d4bfa8f4 --- /dev/null +++ b/tests/fixtures/node-island/order/node_modules/orderkit/index.js @@ -0,0 +1,6 @@ +"use strict"; +// Loads its part through a computed specifier when it loads, so it runs in +// a Node island. +const { createRequire } = require("node:module"); +const parts = [createRequire(__filename)(["./thing"][0])]; +exports.make = () => new parts[0].Thing(); diff --git a/tests/fixtures/node-island/order/node_modules/orderkit/package.json b/tests/fixtures/node-island/order/node_modules/orderkit/package.json new file mode 100644 index 000000000..ee88f4830 --- /dev/null +++ b/tests/fixtures/node-island/order/node_modules/orderkit/package.json @@ -0,0 +1 @@ +{ "name": "orderkit", "version": "1.0.0", "main": "index.js", "types": "index.d.ts" } diff --git a/tests/fixtures/node-island/order/node_modules/orderkit/thing.js b/tests/fixtures/node-island/order/node_modules/orderkit/thing.js new file mode 100644 index 000000000..7d3a40845 --- /dev/null +++ b/tests/fixtures/node-island/order/node_modules/orderkit/thing.js @@ -0,0 +1,7 @@ +"use strict"; +class Thing { + m() { + return "original"; + } +} +exports.Thing = Thing; diff --git a/tests/fixtures/node-island/path-mapped-reexport.ts b/tests/fixtures/node-island/path-mapped-reexport.ts new file mode 100644 index 000000000..7bb102298 --- /dev/null +++ b/tests/fixtures/node-island/path-mapped-reexport.ts @@ -0,0 +1,7 @@ +// Re-exports from an island package that tsconfig `paths` maps to its +// installed directory: the type-only one is erased, the value one reports +// the island boundary, and the package's declarations never compile as a +// program module. The test builds it beside corpus 6839's pathkit package. +export type { Route } from "pathkit"; +export { route } from "pathkit"; +console.log("loaded"); diff --git a/tests/fixtures/npm/cases/unloaded-dynamic-import/dep.ts b/tests/fixtures/npm/cases/unloaded-dynamic-import/dep.ts new file mode 100644 index 000000000..6fb394385 --- /dev/null +++ b/tests/fixtures/npm/cases/unloaded-dynamic-import/dep.ts @@ -0,0 +1,9 @@ +// A dependency of the import()-only `lazy.ts` graph below. Its own static +// import names a package the project marks unloaded, so the graph rejects +// when `./lazy.ts` is imported — and equally when `./dep.ts` is imported +// directly. +import { heavy } from "gtunloadedheavy"; + +export function helper(text: string): string { + return heavy(text); +} diff --git a/tests/fixtures/npm/cases/unloaded-dynamic-import/lazy.ts b/tests/fixtures/npm/cases/unloaded-dynamic-import/lazy.ts new file mode 100644 index 000000000..28a364f82 --- /dev/null +++ b/tests/fixtures/npm/cases/unloaded-dynamic-import/lazy.ts @@ -0,0 +1,8 @@ +import { heavy } from "gtunloadedheavy"; +import { helper } from "./dep.ts"; + +console.log("lazy evaluated"); + +export function run(text: string): string { + return helper(text) + heavy(text); +} diff --git a/tests/fixtures/npm/cases/unloaded-dynamic-import/main.ts b/tests/fixtures/npm/cases/unloaded-dynamic-import/main.ts new file mode 100644 index 000000000..53e8a90ca --- /dev/null +++ b/tests/fixtures/npm/cases/unloaded-dynamic-import/main.ts @@ -0,0 +1,41 @@ +/// +// Packages the project marks never loaded +// (package.json: "scriptc": { "packages": { ...: "unloaded" } }) behind +// import(): a workspace package imported on demand, and a module only +// import() loads whose static import names such a package. Each import() +// rejects with Node's ERR_MODULE_NOT_FOUND, exactly as when the package is +// not installed, and no module of the refused graph evaluates. +// tests/harness/package-shims.test.ts compiles this program with both +// packages installed and checks the output is unchanged. +function report(what: string, err: unknown): void { + const e = err as { code?: string; message: string }; + const [reason, importer] = e.message.split(" imported from "); + console.log(what, e.code, reason, importer?.slice(importer.lastIndexOf("/") + 1)); +} + +async function main(): Promise { + console.log("start"); + try { + const { parse } = await import("wsunloadedparser"); + console.log("parsed", parse("a")); + } catch (err) { + report("parser:", err); + } + try { + const lazy = await import("./lazy.ts"); + console.log(lazy.run("b")); + } catch (err) { + report("lazy:", err); + } + // `dep.ts` is a dependency of the already-rejected `./lazy.ts` graph, but + // this literal import() targets it directly: its own graph rejects too, so + // it must reject at run time instead of joining the compiled graph. + try { + const dep = await import("./dep.ts"); + console.log(dep.helper("c")); + } catch (err) { + report("dep:", err); + } + console.log("end"); +} +main(); diff --git a/tests/fixtures/npm/cases/unloaded-dynamic-import/package.json b/tests/fixtures/npm/cases/unloaded-dynamic-import/package.json new file mode 100644 index 000000000..8dae8adc4 --- /dev/null +++ b/tests/fixtures/npm/cases/unloaded-dynamic-import/package.json @@ -0,0 +1,10 @@ +{ + "private": true, + "type": "module", + "scriptc": { + "packages": { + "wsunloadedparser": "unloaded", + "gtunloadedheavy": "unloaded" + } + } +} diff --git a/tests/fixtures/npm/cases/unloaded-dynamic-import/types.d.ts b/tests/fixtures/npm/cases/unloaded-dynamic-import/types.d.ts new file mode 100644 index 000000000..67102c5bd --- /dev/null +++ b/tests/fixtures/npm/cases/unloaded-dynamic-import/types.d.ts @@ -0,0 +1,8 @@ +// Types for the two packages the project marks never loaded. Neither is +// installed here; tests/harness/package-shims.test.ts installs both. +declare module "wsunloadedparser" { + export function parse(text: string): string; +} +declare module "gtunloadedheavy" { + export function heavy(text: string): string; +} diff --git a/tests/harness/corpus-costs/6812-node-island-handles/main.mjs.json b/tests/harness/corpus-costs/6812-node-island-handles/main.mjs.json new file mode 100644 index 000000000..16841f64d --- /dev/null +++ b/tests/harness/corpus-costs/6812-node-island-handles/main.mjs.json @@ -0,0 +1 @@ +{ "plain": 2000, "san": 30000 } diff --git a/tests/harness/corpus-costs/6813-node-island-handle-errors/main.mjs.json b/tests/harness/corpus-costs/6813-node-island-handle-errors/main.mjs.json new file mode 100644 index 000000000..16841f64d --- /dev/null +++ b/tests/harness/corpus-costs/6813-node-island-handle-errors/main.mjs.json @@ -0,0 +1 @@ +{ "plain": 2000, "san": 30000 } diff --git a/tests/harness/corpus-costs/6814-node-island-handle-bytes/main.mjs.json b/tests/harness/corpus-costs/6814-node-island-handle-bytes/main.mjs.json new file mode 100644 index 000000000..16841f64d --- /dev/null +++ b/tests/harness/corpus-costs/6814-node-island-handle-bytes/main.mjs.json @@ -0,0 +1 @@ +{ "plain": 2000, "san": 30000 } diff --git a/tests/harness/corpus-costs/6815-node-island-handle-promises/main.mjs.json b/tests/harness/corpus-costs/6815-node-island-handle-promises/main.mjs.json new file mode 100644 index 000000000..16841f64d --- /dev/null +++ b/tests/harness/corpus-costs/6815-node-island-handle-promises/main.mjs.json @@ -0,0 +1 @@ +{ "plain": 2000, "san": 30000 } diff --git a/tests/harness/corpus-costs/6822-node-island-gc/main.ts.json b/tests/harness/corpus-costs/6822-node-island-gc/main.ts.json new file mode 100644 index 000000000..8affc810d --- /dev/null +++ b/tests/harness/corpus-costs/6822-node-island-gc/main.ts.json @@ -0,0 +1 @@ +{ "plain": 3300, "san": 23000 } diff --git a/tests/harness/corpus-costs/6823-node-island-nested-stack/main.ts.json b/tests/harness/corpus-costs/6823-node-island-nested-stack/main.ts.json new file mode 100644 index 000000000..5a61731bd --- /dev/null +++ b/tests/harness/corpus-costs/6823-node-island-nested-stack/main.ts.json @@ -0,0 +1 @@ +{ "plain": 3000, "san": 20000 } diff --git a/tests/harness/corpus-costs/6839-node-island-path-mapped/main.ts.json b/tests/harness/corpus-costs/6839-node-island-path-mapped/main.ts.json new file mode 100644 index 000000000..f9a85c4e9 --- /dev/null +++ b/tests/harness/corpus-costs/6839-node-island-path-mapped/main.ts.json @@ -0,0 +1 @@ +{ "plain": 3300, "san": 30000 } diff --git a/tests/harness/corpus-costs/6890-node-island-timer-refresh/main.ts.json b/tests/harness/corpus-costs/6890-node-island-timer-refresh/main.ts.json new file mode 100644 index 000000000..8affc810d --- /dev/null +++ b/tests/harness/corpus-costs/6890-node-island-timer-refresh/main.ts.json @@ -0,0 +1 @@ +{ "plain": 3300, "san": 23000 } diff --git a/tests/harness/corpus-costs/generic-conditional-member-bindings.ts.json b/tests/harness/corpus-costs/generic-conditional-member-bindings.ts.json new file mode 100644 index 000000000..f46d03f09 --- /dev/null +++ b/tests/harness/corpus-costs/generic-conditional-member-bindings.ts.json @@ -0,0 +1 @@ +{ "plain": 14775, "san": 16991 } diff --git a/tests/harness/corpus-costs/generic-conditional-member-units.ts.json b/tests/harness/corpus-costs/generic-conditional-member-units.ts.json new file mode 100644 index 000000000..2336c02d9 --- /dev/null +++ b/tests/harness/corpus-costs/generic-conditional-member-units.ts.json @@ -0,0 +1 @@ +{ "plain": 3600, "san": 5500 } diff --git a/tests/harness/libnode.ts b/tests/harness/libnode.ts index d3941631e..355f21799 100644 --- a/tests/harness/libnode.ts +++ b/tests/harness/libnode.ts @@ -1,7 +1,7 @@ /* Node island tests need a libnode pack (SCRIPTC_LIBNODE). The Linux x64 and - * macOS CI test lanes provide one (.github/actions/libnode-dev-pack) and - * must run these tests, so there a missing pack is a failure, never a skip. - * Other platforms have no pack yet (NI-13), and a local run without a pack + * macOS CI test lanes provide one (.github/actions/libnode) and must run + * these tests, so there a missing pack is a failure, never a skip. Other + * CI platforms do not provision a pack yet, and a local run without a pack * skips. */ import { readFileSync } from "node:fs"; @@ -25,7 +25,7 @@ export function libnodePack(): string | null { if (pack !== undefined && pack !== "") return pack; if (libnodeRequired) throw new Error( - "Node island tests need a libnode pack in this CI lane, and SCRIPTC_LIBNODE is not set (.github/actions/libnode-dev-pack provides it)", + "Node island tests need a libnode pack in this CI lane, and SCRIPTC_LIBNODE is not set (.github/actions/libnode provides it)", ); return null; } diff --git a/tests/harness/node-island-handles.test.ts b/tests/harness/node-island-handles.test.ts new file mode 100644 index 000000000..3bbe1865f --- /dev/null +++ b/tests/harness/node-island-handles.test.ts @@ -0,0 +1,217 @@ +/* Island object handles (scope-B NI-4) where a differential corpus program + * cannot show it: handles to Node objects the program dropped are released, + * so the import table does not grow with the work a program does (acyclic + * garbage; cycles across both heaps wait for NI-8). The corpus (6812-6815) + * pins handle semantics against Node. An island package ships with the + * program, so the executable runs without the tree it was built from. + * Needs a libnode pack in SCRIPTC_LIBNODE (libnode.ts). */ +import { execFile } from "node:child_process"; +import { cpSync, mkdirSync, readdirSync, readFileSync, renameSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import { promisify } from "node:util"; +import { describe, expect, test } from "vitest"; +import { compile } from "@scriptc/compiler"; +import { libnodePack } from "./libnode.js"; + +const execFileAsync = promisify(execFile); +const repoRoot = join(import.meta.dirname, "../.."); +const outDir = join(repoRoot, "node_modules/.cache/scriptc-tests/node-island-handles"); +const sanitize = process.env["SCRIPTC_SAN"] === "1"; +const skip = libnodePack() === null; + +describe.skipIf(skip)("Node island handles", () => { + test("handles the program dropped are released", async () => { + mkdirSync(outDir, { recursive: true }); + const result = await compile(join(repoRoot, "tests/fixtures/node-island/handles-leak.mjs"), { + outPath: join(outDir, "handles-leak"), + outDir, + sanitize, + }); + if (!result.ok) + throw new Error(result.diagnostics.map((d) => `${d.code}: ${d.message}`).join("\n")); + const env = { ...process.env, SCRIPTC_NODE_ISLANDS_TRACE: "handles" }; + const counts: number[] = []; + for (const n of ["2000", "20000"]) { + const { stdout, stderr } = await execFileAsync(result.binaryPath, [n], { + env, + encoding: "utf8", + }); + expect(stdout).toMatch(/^sum \d+ 499500 object -1\n$/); + const live = /\[node-islands\] live handles (\d+)/.exec(stderr); + expect(live).not.toBeNull(); + counts.push(Number(live![1])); + } + // The module and the kept object (with util.inspect, at most a few + // more), independent of the number of iterations. + expect(counts[0]).toBe(counts[1]); + expect(counts[1]).toBeLessThan(8); + }, 300_000); + + async function runFixture(name: string, entry: string): Promise { + const dir = join(outDir, name); + mkdirSync(dir, { recursive: true }); + const result = await compile(join(repoRoot, "tests/fixtures/node-island", entry), { + outPath: join(dir, name), + outDir: dir, + sanitize, + }); + if (!result.ok) + throw new Error(result.diagnostics.map((d) => `${d.code}: ${d.message}`).join("\n")); + const env = { ...process.env, XDG_CACHE_HOME: join(dir, "cache") }; + return (await execFileAsync(result.binaryPath, [], { env, encoding: "utf8" })).stdout; + } + + test("island member calls and assignments evaluate in JavaScript's order", async () => { + expect(await runFixture("order", "order/main.ts")).toBe( + "original\nreceiver,key,value,receiver,value 1 1\n", + ); + }, 300_000); + + test("requireModule() loads Node's built-in modules", async () => { + expect(await runFixture("builtins", "builtins.ts")).toBe("a/b string\n"); + }, 300_000); + + /** Runs a binary that refuses at run time: its stderr. */ + async function refusal(binary: string): Promise { + const run = await execFileAsync(binary, [], { encoding: "utf8" }).catch( + (e: { stderr?: string }) => e, + ); + return String((run as { stderr?: string }).stderr ?? ""); + } + + test("a package marked native never runs in a Node island", async () => { + // esmbridge's native build has deferred refusals (its CommonJS exports + // object read as a value, an Object.keys form), so by default it runs in + // a Node island; marked native, it compiles natively and refuses at run + // time. + const dir = join(outDir, "native-policy"); + mkdirSync(dir, { recursive: true }); + const entry = join(repoRoot, "tests/fixtures/npm/cases/cjs-esm-named/main.ts"); + const islanded = await compile(entry, { outPath: join(dir, "main"), outDir: dir, sanitize }); + // The SC8001 notes that name esmbridge. + const islandNotes = (r: typeof islanded): string[] => + r.ok + ? (r.warnings ?? []) + .filter((d) => d.code === "SC8001" && d.message.includes("'esmbridge@")) + .map((d) => d.message) + : []; + expect(islanded.ok).toBe(true); + expect(islandNotes(islanded)).toHaveLength(1); + const marked = await compile(entry, { + outPath: join(dir, "marked"), + outDir: dir, + sanitize, + nodeIslands: { native: ["esmbridge"] }, + }); + expect(marked.ok).toBe(true); + expect(islandNotes(marked)).toEqual([]); + if (marked.ok) + expect(await refusal(marked.binaryPath)).toMatch(/\[SC\d{4} at [^\]]*\/esmbridge\//); + }, 300_000); + + /** The SC8001 notes of a build that name `pkg`. */ + async function islandNotesOf(entry: string, name: string, pkg: string): Promise { + const dir = join(outDir, name); + mkdirSync(dir, { recursive: true }); + const result = await compile(join(repoRoot, entry), { + outPath: join(dir, name), + outDir: dir, + sanitize, + }); + expect(result.ok, name).toBe(true); + if (!result.ok) return []; + return (result.warnings ?? []) + .filter((d) => d.code === "SC8001" && d.message.includes(`'${pkg}@`)) + .map((d) => d.message); + } + + test("a package whose fences may never run stays native", async () => { + // jose's run-time fences are inside function bodies, on paths a program + // may never call: only fences that surely fire make a Node island. + expect( + await islandNotesOf("tests/corpus/6331-jose-root-entry/main.ts", "jose-native", "jose"), + ).toEqual([]); + }, 300_000); + + test("a fence in module code makes a Node island, with the ES modules importing it", async () => { + // cjszoo's module code sets its exports object's prototype (R2.2), so + // its fence fires when it loads; esmbridge's ES module imports names + // from it, which only Node's CommonJS facade gives as Node does. + const entry = "tests/fixtures/npm/cases/cjs-esm-named/main.ts"; + const zoo = await islandNotesOf(entry, "fence-sure", "cjszoo"); + expect(zoo).toHaveLength(1); + expect(zoo[0]).toContain("R2.2"); + const bridge = await islandNotesOf(entry, "fence-sure", "esmbridge"); + expect(bridge).toEqual([ + "package 'esmbridge@1.0.0' runs in a Node island: packaging: an ES module of the package imports the CommonJS island package 'cjszoo'", + ]); + }, 300_000); + + test("without islands, a package's deferred refusals refuse at run time", async () => { + // With --node-islands=off, packages whose native builds have deferred + // refusals compile natively and report them when they run: orderkit's + // createRequire call with a computed specifier (SC2020), and the first + // one cjs-esm-named reaches in its packages (esmbridge, cjszoo). + const cases = [ + ["order-off", "tests/fixtures/node-island/order/main.ts", /\[SC2020 at [^\]]*\/orderkit\//], + [ + "esm-named-off", + "tests/fixtures/npm/cases/cjs-esm-named/main.ts", + /\[SC\d{4} at [^\]]*\/node_modules\/(?:esmbridge|cjszoo)\//, + ], + ] as const; + for (const [name, entry, code] of cases) { + const dir = join(outDir, name); + mkdirSync(dir, { recursive: true }); + const result = await compile(join(repoRoot, entry), { + outPath: join(dir, name), + outDir: dir, + sanitize, + nodeIslands: { mode: "off" }, + }); + expect(result.ok, name).toBe(true); + if (!result.ok) continue; + expect((result.warnings ?? []).map((d) => d.code)).not.toContain("SC8001"); + expect(await refusal(result.binaryPath), name).toMatch(code); + } + }, 300_000); + + test("an island package ships with the program", async () => { + const root = join(outDir, "ships"); + const src = join(root, "src"); + rmSync(root, { recursive: true, force: true }); + cpSync(join(repoRoot, "tests/corpus/6816-node-island-objects"), src, { recursive: true }); + const env = { ...process.env, XDG_CACHE_HOME: join(root, "cache") }; + const built: Record = {}; + for (const libnode of ["inside", "beside"]) { + const dir = join(root, libnode); + mkdirSync(dir, { recursive: true }); + const result = await compile(join(src, "main.ts"), { + outPath: join(dir, "main"), + outDir: dir, + sanitize, + ...(libnode === "beside" ? { libnode } : {}), + }); + if (!result.ok) + throw new Error(result.diagnostics.map((d) => `${d.code}: ${d.message}`).join("\n")); + built[libnode] = result.binaryPath; + } + const before = await execFileAsync(built["inside"]!, [], { env, encoding: "utf8" }); + expect(before.stdout).toMatch(/^islandkit \d/); + // The executable names its island package by its path in the shipped + // tree, never by the directory it was built from. + expect(readFileSync(built["inside"]!).toString("latin1").includes(src)).toBe(false); + // Beside the executable, the tree holds the package's files. + const index = readFileSync(join(src, "node_modules/islandkit/index.js"), "utf8"); + const modules = join(root, "beside/main.libnode/modules"); + const files = readdirSync(modules, { recursive: true, encoding: "utf8" }); + expect( + files.some((f) => f.endsWith("index.js") && readFileSync(join(modules, f), "utf8") === index), + ).toBe(true); + renameSync(src, join(root, "moved")); + for (const binary of Object.values(built)) { + const after = await execFileAsync(binary, [], { env, encoding: "utf8" }); + expect(after.stdout).toBe(before.stdout); + } + }, 300_000); +}); diff --git a/tests/harness/node-island-order.test.ts b/tests/harness/node-island-order.test.ts new file mode 100644 index 000000000..5d4aa96a1 --- /dev/null +++ b/tests/harness/node-island-order.test.ts @@ -0,0 +1,124 @@ +/* Ordering fidelity of the hosted event loop (scope-B milestone NI-3, + * design §5.2). Once a scriptc program loads Node, its ticks, promise + * jobs, timers, immediates and I/O run on Node's loop together with island + * code's, so the interleaving of both sides must be one Node itself + * produces. Each program in tests/corpus-node-islands-order/ (the native + * half in main.ts, the island half interpreted by island.cjs) runs M times + * under Node, which collects the set of orders Node produces, and N times + * as a compiled binary; every native run must produce an output in Node's + * set. Programs whose order is fully determined by the queues produce one + * output; the I/O races (fs callbacks against busy timers) may produce + * several, and the native runs must stay within them. + * + * A program whose head says `// @order: race` races a timer against I/O + * whose duration differs between the sides (scriptc's own fs callbacks + * complete in one turn where Node's readFile takes several; module loading + * on the first call into Node takes longer than in a Node process), or two + * equal timers armed on either side of an island call, which a millisecond + * boundary between them (likely in a sanitized build) puts in different + * loop turns, so Node permits orders it rarely produces in M runs. Its + * native runs must + * produce the lines of one of Node's outputs, in any order; the report + * still records whether the order itself was among Node's. + * + * The s* programs pin the one-time switch: work already pending in the + * program's queues (timers, immediates, ticks, microtasks, promise jobs, + * fs callbacks, an open socket) at the moment Node loads, in the main + * body, a timer, an immediate and a promise job. + * + * Needs a libnode pack in SCRIPTC_LIBNODE whose Node version is the `node` + * on PATH. NODE_ISLAND_ORDER_NODE_RUNS (default 20) and + * NODE_ISLAND_ORDER_NATIVE_RUNS (default 3) set M and N; + * NODE_ISLAND_ORDER_REPORT names a JSON file for the per-program counts. */ +import { execFile } from "node:child_process"; +import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { promisify } from "node:util"; +import { afterAll, describe, expect, test } from "vitest"; +import { compile } from "@scriptc/compiler"; + +const execFileAsync = promisify(execFile); +const repoRoot = join(import.meta.dirname, "../.."); +const programsDir = join(repoRoot, "tests/corpus-node-islands-order"); +const outDir = join(repoRoot, "node_modules/.cache/scriptc-tests/node-island-order"); +const pack = process.env["SCRIPTC_LIBNODE"]; +const skip = (process.platform !== "linux" && process.platform !== "darwin") || !pack; +const nodeRuns = Number(process.env["NODE_ISLAND_ORDER_NODE_RUNS"] ?? 20); +const nativeRuns = Number(process.env["NODE_ISLAND_ORDER_NATIVE_RUNS"] ?? 3); +const sanitize = process.env["SCRIPTC_SAN"] === "1"; + +const programs = existsSync(programsDir) + ? readdirSync(programsDir, { withFileTypes: true }) + .filter( + (entry) => entry.isDirectory() && existsSync(join(programsDir, entry.name, "main.ts")), + ) + .map((entry) => entry.name) + .sort() + : []; + +/** Linux ASan's once-per-process warning at the first context switch (the + * runtime's fibers and Node's stack): not program output. */ +const ASAN_SWAP = /^==\d+==WARNING: ASan doesn't fully support makecontext\/swapcontext.*\n/gm; + +/** One run's observable outcome: stdout, stderr and the exit status. */ +async function outcome(cmd: string, args: string[]): Promise { + try { + const { stdout, stderr: raw } = await execFileAsync(cmd, args, { + cwd: repoRoot, + encoding: "utf8", + timeout: 20_000, + env: { ...process.env, NODE_NO_WARNINGS: "1" }, + }); + const stderr = raw.replace(ASAN_SWAP, ""); + return `${stdout}${stderr === "" ? "" : `[stderr]\n${stderr}`}[exit 0]`; + } catch (err) { + const e = err as { stdout?: string; stderr?: string; code?: unknown; signal?: unknown }; + if (typeof e.code !== "number") return `[failed ${String(e.signal ?? e.code)}]`; + const stderr = (e.stderr ?? "").replace(ASAN_SWAP, ""); + return `${e.stdout ?? ""}${stderr ? `[stderr]\n${stderr}` : ""}[exit ${e.code}]`; + } +} + +const report: Record; native: Record }> = {}; + +afterAll(() => { + const path = process.env["NODE_ISLAND_ORDER_REPORT"]; + if (path) writeFileSync(path, `${JSON.stringify(report, null, 2)}\n`); +}); + +describe.skipIf(skip)(`Node island ordering (${programs.length} programs)`, () => { + test.for(programs)("%s", { timeout: 600_000 }, async (name) => { + const main = join(programsDir, name, "main.ts"); + const dir = join(outDir, name); + mkdirSync(dir, { recursive: true }); + const built = await compile(main, { outPath: join(dir, "program"), outDir: dir, sanitize }); + if (!built.ok) + throw new Error(built.diagnostics.map((d) => `${d.code}: ${d.message}`).join("\n")); + const node: Record = {}; + for (let i = 0; i < nodeRuns; i++) { + const out = await outcome("node", [main]); + node[out] = (node[out] ?? 0) + 1; + } + const native: Record = {}; + for (let i = 0; i < nativeRuns; i++) { + const out = await outcome(built.binaryPath, []); + native[out] = (native[out] ?? 0) + 1; + } + report[name] = { node, native }; + const race = readFileSync(main, "utf8").split("\n", 3).includes("// @order: race"); + const lines = (out: string): string => out.split("\n").sort().join("\n"); + if (race) { + const nodeLines = Object.keys(node).map(lines); + for (const out of Object.keys(native)) + expect(nodeLines, `native lines not produced by Node in ${nodeRuns} runs`).toContain( + lines(out), + ); + return; + } + for (const out of Object.keys(native)) { + expect(Object.keys(node), `native output not produced by Node in ${nodeRuns} runs`).toContain( + out, + ); + } + }); +}); diff --git a/tests/harness/node-island-process.test.ts b/tests/harness/node-island-process.test.ts new file mode 100644 index 000000000..5b42400d9 --- /dev/null +++ b/tests/harness/node-island-process.test.ts @@ -0,0 +1,72 @@ +/* Process integration of in-process Node islands on the hosted loop + * (scope-B milestone NI-3) where the corpus cannot compare byte for byte: + * an uncaught exception or an unhandled rejection on either side once Node + * is loaded. Each side prints its own report format (scriptc's for native + * code, Node's for island code), and Node's report quotes source lines and + * stack frames the two runs do not share, so stderr is checked for the + * error, while stdout ('exit' listeners of both sides, with the code) and + * the exit status must match Node exactly. The deterministic cases (signals, + * exit codes, process.exit from either side, stdio) are corpus programs + * 6804-6811. Needs a libnode pack in SCRIPTC_LIBNODE. */ +import { execFile } from "node:child_process"; +import { mkdirSync } from "node:fs"; +import { join } from "node:path"; +import { promisify } from "node:util"; +import { describe, expect, test } from "vitest"; +import { compile } from "@scriptc/compiler"; + +const execFileAsync = promisify(execFile); +const repoRoot = join(import.meta.dirname, "../.."); +const fixtures = join(repoRoot, "tests/fixtures/node-island-process"); +const outDir = join(repoRoot, "node_modules/.cache/scriptc-tests/node-island-process"); +const pack = process.env["SCRIPTC_LIBNODE"]; +const skip = (process.platform !== "linux" && process.platform !== "darwin") || !pack; +const sanitize = process.env["SCRIPTC_SAN"] === "1"; + +interface Run { + stdout: string; + stderr: string; + code: number | null; +} + +async function run(cmd: string, args: string[]): Promise { + try { + const { stdout, stderr } = await execFileAsync(cmd, args, { + cwd: repoRoot, + encoding: "utf8", + timeout: 20_000, + }); + return { stdout, stderr, code: 0 }; + } catch (err) { + const e = err as { stdout?: string; stderr?: string; code?: unknown }; + if (typeof e.code !== "number") throw err; + return { stdout: e.stdout ?? "", stderr: e.stderr ?? "", code: e.code }; + } +} + +async function both(name: string): Promise<{ node: Run; native: Run }> { + const main = join(fixtures, `${name}.ts`); + const dir = join(outDir, name); + mkdirSync(dir, { recursive: true }); + const built = await compile(main, { outPath: join(dir, "program"), outDir: dir, sanitize }); + if (!built.ok) + throw new Error(built.diagnostics.map((d) => `${d.code}: ${d.message}`).join("\n")); + const [node, native] = await Promise.all([run("node", [main]), run(built.binaryPath, [])]); + return { node, native }; +} + +describe.skipIf(skip)("Node island process integration", () => { + test.for([ + ["native-uncaught", "Error: native boom"], + ["island-uncaught", "Error: island boom"], + ["native-rejection", "native rejected"], + ["island-rejection", "island rejected"], + ] as const)("%s", { timeout: 300_000 }, async ([name, error]) => { + const { node, native } = await both(name); + expect(node.code).toBe(1); + expect(native.code).toBe(node.code); + expect(native.stdout).toBe(node.stdout); + expect(node.stderr).toContain(error); + expect(native.stderr).toContain(error); + }); +}); diff --git a/tests/harness/node-island.test.ts b/tests/harness/node-island.test.ts index aed4fbb61..442869468 100644 --- a/tests/harness/node-island.test.ts +++ b/tests/harness/node-island.test.ts @@ -308,7 +308,7 @@ describe.skipIf(skip)("Node islands in-process", () => { }); expect(result.code).toBe(1); expect(result.stderr).toBe( - `scriptc: the libnode pack at ${dir} has an incompatible bridge (ABI 999; this program needs 3)\n`, + `scriptc: the libnode pack at ${dir} has an incompatible bridge (ABI 999; this program needs 5)\n`, ); }); @@ -354,3 +354,22 @@ test("recursive JSON types cross into Node", async () => { const ir = JSON.stringify(deserializeModule(readFileSync(outPath, "utf8"))); expect(ir).toContain("nodeIsland.callSync"); }, 300_000); + +// Needs no libnode pack: the build fails before linking. +test("re-exporting a path-mapped island package reports the island boundary", async () => { + const dir = join(outDir, "path-mapped-reexport"); + rmSync(dir, { recursive: true, force: true }); + const corpus = join(repoRoot, "tests/corpus/6839-node-island-path-mapped"); + cpSync(join(corpus, "node_modules"), join(dir, "node_modules"), { recursive: true }); + copyFileSync(join(corpus, "tsconfig.json"), join(dir, "tsconfig.json")); + copyFileSync( + join(repoRoot, "tests/fixtures/node-island/path-mapped-reexport.ts"), + join(dir, "main.ts"), + ); + const result = await compile(join(dir, "main.ts"), { outPath: join(dir, "main"), outDir: dir }); + expect(result.ok).toBe(false); + if (result.ok) return; + expect(result.diagnostics.map((d) => `${d.code} ${d.loc.file.slice(dir.length + 1)}`)).toEqual([ + "SC8010 main.ts", + ]); +}, 300_000); diff --git a/tests/harness/npm-static.test.ts b/tests/harness/npm-static.test.ts index 368d4257f..5951c3d59 100644 --- a/tests/harness/npm-static.test.ts +++ b/tests/harness/npm-static.test.ts @@ -15,6 +15,7 @@ import { promisify } from "node:util"; import { beforeEach, describe, expect, test } from "vitest"; import { analyze, compile } from "@scriptc/compiler"; import { skipUnownedTest } from "./shard.js"; +import { libnodePack } from "./libnode.js"; const execFileAsync = promisify(execFile); const repoRoot = join(import.meta.dirname, "../.."); @@ -632,7 +633,10 @@ describe(`npm-static pilots${sanitize ? " (sanitized)" : ""}`, () => { } }); - test("a word segmenter in a package constructs, and its segment() throws SC2020", async () => { + // The package builds its segmenter at module load, a run-time fence that + // surely fires, so the package runs in its Node island and segments as + // Node does. + test("a word segmenter a package builds at load runs in its Node island", async () => { const entry = join(pilotRoot, "word-segments-cli.ts"); const { coverage } = analyze(entry, { npmStatic: "auto" }); expect(coverage.npmStatic).toEqual([{ package: "word-segments", status: "static" }]); @@ -645,12 +649,15 @@ describe(`npm-static pilots${sanitize ? " (sanitized)" : ""}`, () => { outDir, outPath: join(outDir, "program"), }); + if (libnodePack() === null) { + expect(result.ok, "no libnode pack: the compile stops").toBe(false); + if (!result.ok) expect(result.diagnostics.map((d) => d.code)).toContain("SC8005"); + return; + } if (!result.ok) throw new Error(result.diagnostics.map((d) => `${d.code}: ${d.message}`).join("\n")); const nativeRes = await runBinary(result.binaryPath, []); - expect(nativeRes.stdout.toString()).toMatch( - /^SC2020: Intl\.Segmenter segmentation with granularity 'word' is not supported yet \[SC2020 at .*index\.js:1\]\n$/, - ); + expect(nativeRes.stdout.toString()).toBe("two| |words\n"); expect(nativeRes.exitCode).toBe(0); }); diff --git a/tests/harness/npm.test.ts b/tests/harness/npm.test.ts index b1b705db1..61526b930 100644 --- a/tests/harness/npm.test.ts +++ b/tests/harness/npm.test.ts @@ -9,7 +9,10 @@ * (a refusal deferred to the operation that reached it), for every argv * list or for the listed `argvs`, while the other lists match Node; * - divergence: it runs but does not match Node yet. These run as expected - * failures, so a fix that makes one match fails here until KNOWN drops it. + * failures, so a fix that makes one match fails here until KNOWN drops it; + * - island: a package it imports runs in a Node island (scope-B NI-4): with + * a libnode pack (SCRIPTC_LIBNODE) it matches Node; without one the build + * stops with SC8005 (no pack to load Node from). * * The fixture node_modules are committed test data. Binaries compile the * package sources at build time and read nothing from node_modules at run @@ -24,6 +27,7 @@ import { describe, expect, test } from "vitest"; import { compile } from "@scriptc/compiler"; import { npmCases, type NpmCase } from "./npm-cases.js"; import { shardSelect, shardSuffix } from "./shard.js"; +import { libnodePack } from "./libnode.js"; const execFileAsync = promisify(execFile); const repoRoot = join(import.meta.dirname, "../.."); @@ -31,22 +35,26 @@ const fixturesRoot = join(repoRoot, "tests/fixtures"); const cacheDir = join(repoRoot, "node_modules/.cache/scriptc-tests"); const sanitize = process.env["SCRIPTC_SAN"] === "1"; -type Known = { build: string } | { run: string; argvs?: string[][] } | { divergence: string }; +type Known = + | { build: string } + | { run: string; argvs?: string[][] } + | { divergence: string } + | { island: true }; const KNOWN: Record = { "assert-shims": { run: "SC1043" }, "buffer-shims": { run: "SC2020" }, "bytes-exit": { build: "SC2004" }, - "cjs-esm-named": { run: "SC1090" }, - "cjs-lexer-aligned": { run: "SC1090" }, + "cjs-esm-named": { island: true }, + "cjs-lexer-aligned": { island: true }, "commander-calc": { run: "SC1090", argvs: [["--help"]] }, "commander-calc-async": { run: "SC1090", argvs: [["--help"]] }, "create-require": { run: "SC1090" }, - "dotfolder": { run: "SC1090" }, - "dual-package-kinds": { build: "SC2013" }, - "dynamic-import": { build: "SC2012" }, + "dotfolder": { island: true }, + "dual-package-kinds": { island: true }, + "dynamic-import": { island: true }, "dynamic-import-local": { build: "SC2012" }, - "esbuild-require": { build: "SC2013" }, + "esbuild-require": { island: true }, "esm-missing-default": { build: "SC1090" }, "esm-missing-export": { build: "SC1090" }, "fs-shims": { run: "SC1101" }, @@ -58,19 +66,25 @@ const KNOWN: Record = { }, // require() inside a function. "island-web-plumbing": { run: "SC1090" }, - "lazy-traps": { build: "SC2013" }, + "lazy-traps": { + divergence: + "an island module loaded by path lists itself twice in MODULE_NOT_FOUND require stacks", + }, "misc-shims": { run: "SC1090" }, - "namespace": { build: "SC1013" }, + namespace: { island: true }, "node24-async-events": { run: "SC2020" }, - "node24-commonjs-metadata": { run: "SC2020" }, + "node24-commonjs-metadata": { + divergence: + "a Node island (its require.cache use needs the runtime) loads its entry with itself as the CJS parent, so require stacks list it twice", + }, "node24-crypto-kdf": { run: "SC2020" }, "node24-deep-equality": { run: "SC2020" }, "node24-diagnostics-async": { run: "SC1090" }, "node24-diagnostics-tracing": { run: "SC1090" }, "node24-env": { run: "SC1090" }, - "node24-esm-metadata": { run: "SC1090" }, + "node24-esm-metadata": { island: true }, "node24-event-emitter": { run: "SC2020" }, - "node24-file-url-conversion": { build: "SC1090" }, + "node24-file-url-conversion": { island: true }, "node24-fs-descriptors": { run: "SC2020" }, "node24-mime": { run: "SC2020" }, "node24-mime-errors": { run: "SC2020" }, @@ -87,16 +101,16 @@ const KNOWN: Record = { "node24-url-mutation": { run: "SC1090" }, "node24-util-adapters": { run: "Object.create with this value is not supported yet" }, "node24-util-types": { run: "SC1090" }, - "package-imports": { build: "SC2013" }, + "package-imports": { island: true }, "promise-bridge": { build: "SC1090" }, "self-name": { build: "SC2013" }, "stream-shims": { run: "SC2020" }, "util-shims": { run: "SC2020" }, // The package redeclares the var it exported: Node copies the first value. - "var-export-redeclared": { run: "SC1090" }, + "var-export-redeclared": { island: true }, // `class Upper extends TransformStream`: extending a builtin class from JavaScript. "web-streams": { run: "SC1090" }, - "workspace-copied": { build: "SC2013" }, + "workspace-copied": { island: true }, "workspace-unclean": { build: "SC2013" }, }; @@ -222,6 +236,19 @@ describe(`npm native (${cases.length} programs${sanitize ? ", sanitized" : ""}${ }, 120_000); continue; } + if (known !== undefined && "island" in known) { + test(`${c.name} (Node island)`, async () => { + const built = await build(c.entry); + if (libnodePack() === null) { + expect(built.ok, "no libnode pack: the compile stops").toBe(false); + if (!built.ok) expect(built.codes, built.text).toContain("SC8005"); + return; + } + if (!built.ok) throw new Error(`npm fixture failed to compile:\n${built.text}`); + await expectMatchesNode(c, built.binary); + }, 120_000); + continue; + } test( c.name, async () => { diff --git a/tests/harness/package-shims.test.ts b/tests/harness/package-shims.test.ts index a21c4e768..2a3f2ae2d 100644 --- a/tests/harness/package-shims.test.ts +++ b/tests/harness/package-shims.test.ts @@ -4,6 +4,9 @@ * - a package the project marks unloaded stays unloaded when it IS * installed: the native program prints what Node prints without the * package, and the package's code never runs; + * - the same behind import(): an unloaded workspace package, and a module + * only import() loads that imports an unloaded package, reject with + * Node's ERR_MODULE_NOT_FOUND; * - a polyfill package with a builtin alias that the project marks unloaded * stays unloaded: its require throws MODULE_NOT_FOUND; * - a shimmed package that is not installed, or whose major version has no @@ -22,7 +25,15 @@ * Node running the real package. */ import { execFile, spawn } from "node:child_process"; -import { copyFileSync, cpSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { + copyFileSync, + cpSync, + mkdirSync, + mkdtempSync, + rmSync, + symlinkSync, + writeFileSync, +} from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { promisify } from "node:util"; @@ -116,6 +127,57 @@ describe("package shims", () => { expect(await run(result.binaryPath, [], dir)).toBe(expected); }, 180_000); + test("unloaded packages behind import() stay unloaded, even when installed", async () => { + const original = join(casesRoot, "unloaded-dynamic-import"); + const expected = await run(process.execPath, [join(original, "main.ts")], original); + expect(expected).toContain("parser: ERR_MODULE_NOT_FOUND"); + expect(expected).toContain("lazy: ERR_MODULE_NOT_FOUND"); + expect(expected).toContain("dep: ERR_MODULE_NOT_FOUND"); + + const dir = copyCase("unloaded-dynamic-import", "unloaded-dynamic-installed"); + const write = (path: string, text: string) => { + mkdirSync(join(dir, path, ".."), { recursive: true }); + writeFileSync(join(dir, path), text); + }; + // A workspace package (linked into node_modules from outside it) and an + // installed ES module package. + write( + "workspace/parser/package.json", + '{ "name": "wsunloadedparser", "version": "1.0.0", "type": "module", "main": "./index.ts" }\n', + ); + write( + "workspace/parser/index.ts", + 'console.log("parser loaded");\nexport function parse(text: string): string {\n return `parsed:${text}`;\n}\n', + ); + mkdirSync(join(dir, "node_modules"), { recursive: true }); + symlinkSync("../workspace/parser", join(dir, "node_modules/wsunloadedparser")); + write( + "node_modules/gtunloadedheavy/package.json", + '{ "name": "gtunloadedheavy", "version": "1.0.0", "type": "module", "main": "index.js", "types": "index.d.ts" }\n', + ); + write( + "node_modules/gtunloadedheavy/index.js", + 'console.log("heavy loaded");\nexport function heavy(text) {\n return `heavy:${text}`;\n}\n', + ); + write( + "node_modules/gtunloadedheavy/index.d.ts", + "export declare function heavy(text: string): string;\n", + ); + // Control: Node loads both. + const loaded = await run(process.execPath, [join(dir, "main.ts")], dir); + expect(loaded).toContain("parsed parsed:a"); + expect(loaded).toContain("heavy:b"); + + const out = join(dir, ".scriptc"); + const result = await compile(join(dir, "main.ts"), { + outPath: join(out, "program"), + outDir: out, + sanitize, + }); + if (!result.ok) throw new Error(result.diagnostics.map((d) => d.message).join("\n")); + expect(await run(result.binaryPath, [], dir)).toBe(expected); + }, 180_000); + test("a CommonJS package requiring fs-extra compiles the shim", async () => { const dir = join(scratch, "fs-extra-require"); cpSync(fsExtraModules, join(dir, "node_modules"), { recursive: true }); diff --git a/tests/harness/static-only.test.ts b/tests/harness/static-only.test.ts index d34cd66b9..62f31ac8c 100644 --- a/tests/harness/static-only.test.ts +++ b/tests/harness/static-only.test.ts @@ -81,9 +81,12 @@ console.log(slash("a\\\\b"), /^a/.test("abc"), value + 1, path.basename("a/b")); test("an ineligible package fails with its package-specific native refusal", async () => { const directory = mkdtempSync(join(tmpdir(), "scriptc-package-refusal-")); try { + // Node islands off: the native refusal itself (with islands on, the + // package would run in a Node island instead). const result = await compile(join(root, "tests/fixtures/npm-static/typegap-mix.ts"), { outDir: directory, outPath: join(directory, "program"), + nodeIslands: { mode: "off" }, }); expect(result.ok).toBe(false); if (!result.ok) diff --git a/tests/harness/surface-manifest.test.ts b/tests/harness/surface-manifest.test.ts index 8a838aac6..d854db3e0 100644 --- a/tests/harness/surface-manifest.test.ts +++ b/tests/harness/surface-manifest.test.ts @@ -62,6 +62,9 @@ describe("surface manifest generation", () => { expect(readJson("packages/llvm-linux-x64-musl/package.json").version).toBe(releaseVersion); expect(readJson("packages/llvm-linux-arm64-musl/package.json").version).toBe(releaseVersion); expect(readJson("packages/llvm-win32-x64-msvc/package.json").version).toBe(releaseVersion); + for (const platform of ["linux-x64-gnu", "linux-arm64-gnu", "darwin-arm64", "darwin-x64"]) { + expect(readJson(`packages/libnode-${platform}/package.json`).version).toBe(releaseVersion); + } }); test("schema: ids unique and sorted, enums valid, codes on every non-static entry", () => {