From 1562d086091c56471fff8948ee81f4d2be9748fd Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 23 Oct 2026 13:09:08 -0500 Subject: [PATCH] Merge the loader branch, key the build by the libnode pack, and pick musl packs --- .github/workflows/ci.yml | 6 +- .github/workflows/libnode-packs.yml | 4 +- docs/content/docs/dependencies.mdx | 2 +- .../builtin-apis/streams-and-readline.mdx | 9 +- packages/cli/src/bootstrap.ts | 5 +- packages/cli/test/native-link-info.test.ts | 4 +- packages/compiler/src/backend/llvm/dyn.ts | 39 +- .../compiler/src/backend/llvm/emit-module.ts | 38 +- packages/compiler/src/backend/llvm/emitter.ts | 5 + .../src/backend/llvm/expr-stream-callbacks.ts | 11 + .../compiler/src/backend/llvm/lib-shared.ts | 3 + .../compiler/src/backend/llvm/lib-stream.ts | 3 + packages/compiler/src/backend/runtime-abi.ts | 4 +- .../src/backend/runtime-pack-targets.test.ts | 2 +- .../compiler/src/backend/runtime-pack.test.ts | 2 +- packages/compiler/src/cli/usage.ts | 6 +- .../compiler/src/diagnostics/diagnostic.ts | 15 +- .../src/executable/executable-cache.ts | 17 +- .../compiler/src/frontend/input-tracker.ts | 27 + .../frontend/lowering/builtins/readline.ts | 51 +- .../lowering/containers/string-replacement.ts | 64 +- .../frontend/lowering/expressions/binary.ts | 18 + .../lowering/expressions/narrowing.ts | 3 + .../src/frontend/lowering/lower-calls.ts | 27 + .../src/frontend/lowering/lower-classes.ts | 6 +- .../frontend/lowering/lower-node-island.ts | 39 +- .../src/frontend/lowering/lower-stream.ts | 329 +++++++- .../compiler/src/frontend/lowering/lowerer.ts | 20 +- .../src/frontend/lowering/var-hoisting.ts | 60 +- .../npm-static-bundled-chunks.test.ts | 11 +- .../src/frontend/npm-static-bundled-chunks.ts | 20 +- .../src/frontend/npm-static-rewrite-syntax.ts | 14 +- packages/compiler/src/frontend/npm-static.ts | 25 +- packages/compiler/src/frontend/program.ts | 16 +- packages/compiler/src/index.ts | 57 +- packages/compiler/src/ir/builtin-effects.ts | 1 + .../compiler/src/ir/builtin-signatures.ts | 6 + .../src/ir/function-references.test.ts | 6 + packages/compiler/src/ir/ir.ts | 26 +- packages/compiler/src/ir/runtime-features.ts | 2 + packages/compiler/src/ir/validate.ts | 3 + .../src/node-islands/libnode-pack.test.ts | 26 +- .../compiler/src/node-islands/libnode.test.ts | 49 +- packages/compiler/src/node-islands/libnode.ts | 131 +++- .../compiler/src/node-islands/modules.test.ts | 188 +++++ packages/compiler/src/node-islands/modules.ts | 627 ++++++++++++++++ packages/compiler/src/startup-cache.ts | 1 + .../test/native-codegen-integration.test.ts | 4 +- .../7447-union-record-arm-width.mjs.json | 6 + .../7448-union-record-arm-width.ts.json | 6 + .../corpus/7460-transform-writev.mjs.json | 6 + .../corpus/7461-transform-writev.cjs.json | 6 + .../corpus/7462-transform-writev.ts.json | 6 + .../corpus/7463-stream-option-values.ts.json | 6 + .../corpus/7464-stream-option-values.mjs.json | 6 + .../7465-transform-callback-values.ts.json | 6 + .../7466-stream-construct-assign.mjs.json | 6 + .../7500-pipeline-callback-undefined.ts.json | 6 + .../7501-pipeline-callback-undefined.mjs.json | 6 + .../7502-readline-cursor-undefined.ts.json | 6 + .../7543-var-list-declarator-order.js.json | 6 + ...545-string-regex-checked-arguments.js.json | 6 + .../main.cjs.json | 7 + .../main.cjs.json | 8 + ...7553-var-for-in-of-zero-iterations.js.json | 6 + .../7620-js-class-view-descriptors.js.json | 6 + .../7661-node-call-modules/main.ts.json | 6 + .../7662-node-call-exit-events/main.ts.json | 6 + .../stream-runtime-option-order.ts.json | 6 + .../7546-minified-esm-rules/main.ts.json | 6 + .../7547-minified-cjs-chunks/main.ts.json | 6 + .../libnode-common/bridge/scr_node_bridge.cc | 97 ++- packages/libnode-common/scripts/smoke.mjs | 7 +- packages/libnode-common/test/bridge.test.ts | 2 +- packages/libnode-linux-arm64-gnu/package.json | 3 + packages/libnode-linux-x64-gnu/package.json | 3 + packages/node/README.md | 2 + packages/node/index.d.ts | 2 + .../runtime-darwin-arm64/scripts/build.mjs | 2 +- .../runtime-pack-common/scripts/build.mjs | 2 +- packages/runtime/src/scr_console.c | 2 +- packages/runtime/src/scr_core.h | 2 +- packages/runtime/src/scr_dyn_builtin_stream.c | 9 +- packages/runtime/src/scr_fetch.c | 4 + packages/runtime/src/scr_json.c | 73 +- packages/runtime/src/scr_node_bridge.h | 12 +- packages/runtime/src/scr_node_island.c | 702 +++++++++++++++++- packages/runtime/src/scr_node_island.h | 9 + packages/runtime/src/scr_node_island_host.cjs | 66 +- packages/runtime/src/scr_node_island_host.h | 66 +- packages/runtime/src/scr_stream.c | 90 ++- packages/runtime/src/scr_stream.h | 12 + packages/runtime/test/test_json.c | 86 +++ tests/corpus/7447-union-record-arm-width.mjs | 37 + tests/corpus/7448-union-record-arm-width.ts | 21 + tests/corpus/7460-transform-writev.mjs | 33 + tests/corpus/7461-transform-writev.cjs | 33 + tests/corpus/7462-transform-writev.ts | 109 +++ tests/corpus/7463-stream-option-values.ts | 58 ++ tests/corpus/7464-stream-option-values.mjs | 28 + .../corpus/7465-transform-callback-values.ts | 41 + tests/corpus/7466-stream-construct-assign.mjs | 46 ++ .../7500-pipeline-callback-undefined.ts | 120 +++ .../7501-pipeline-callback-undefined.mjs | 22 + .../corpus/7502-readline-cursor-undefined.ts | 28 + .../corpus/7543-var-list-declarator-order.js | 66 ++ .../7545-string-regex-checked-arguments.js | 89 +++ .../lib.cjs | 10 + .../main.cjs | 19 + .../7549-cjs-minified-esmodule-stamp/main.cjs | 7 + .../module-stamped.cjs | 3 + .../stamped.cjs | 5 + .../7553-var-for-in-of-zero-iterations.js | 18 + .../corpus/7620-js-class-view-descriptors.js | 91 +++ .../7661-node-call-modules/lib/entry.cjs | 14 + .../corpus/7661-node-call-modules/lib/esm.mjs | 5 + .../7661-node-call-modules/lib/factor.mjs | 1 + .../corpus/7661-node-call-modules/lib/util.js | 5 + tests/corpus/7661-node-call-modules/main.ts | 12 + .../node_modules/greeter/data/message.txt | 1 + .../node_modules/greeter/index.js | 8 + .../node_modules/greeter/package.json | 1 + .../node_modules/shout/index.js | 2 + .../node_modules/shout/package.json | 1 + .../7661-node-call-modules/settings.json | 1 + .../7662-node-call-exit-events/listeners.cjs | 13 + .../corpus/7662-node-call-exit-events/main.ts | 8 + .../stream-runtime-option-order.ts | 42 ++ tests/fixtures/node-island/gc-stress.ts | 24 + tests/fixtures/node-island/gc.cjs | 41 + tests/fixtures/node-island/helper.cjs | 20 + tests/fixtures/node-island/lifecycle.ts | 8 + tests/fixtures/node-island/worker.ts | 18 + .../npm/cases/7546-minified-esm-rules/main.ts | 22 + .../cases/7547-minified-cjs-chunks/main.ts | 12 + .../node_modules/gtminicache/dist/index.d.ts | 7 + .../node_modules/gtminicache/dist/index.js | 1 + .../npm/node_modules/gtminicache/package.json | 7 + .../node_modules/gtminimark/dist/index.d.ts | 1 + .../npm/node_modules/gtminimark/dist/index.js | 1 + .../npm/node_modules/gtminimark/package.json | 8 + .../stream-runtime-option-order.ts.txt | 26 + tests/harness/ci-stack-scope.test.ts | 16 + .../7447-union-record-arm-width.mjs.json | 1 + .../7448-union-record-arm-width.ts.json | 1 + .../7460-transform-writev.mjs.json | 1 + .../7461-transform-writev.cjs.json | 1 + .../7462-transform-writev.ts.json | 1 + .../7463-stream-option-values.ts.json | 1 + .../7464-stream-option-values.mjs.json | 1 + .../7465-transform-callback-values.ts.json | 1 + .../7466-stream-construct-assign.mjs.json | 1 + .../7500-pipeline-callback-undefined.ts.json | 1 + .../7501-pipeline-callback-undefined.mjs.json | 1 + .../7502-readline-cursor-undefined.ts.json | 1 + .../7543-var-list-declarator-order.js.json | 1 + ...545-string-regex-checked-arguments.js.json | 1 + .../main.cjs.json | 1 + .../main.cjs.json | 1 + ...7553-var-for-in-of-zero-iterations.js.json | 1 + .../7620-js-class-view-descriptors.js.json | 1 + .../7661-node-call-modules/main.ts.json | 1 + .../7662-node-call-exit-events/main.ts.json | 1 + tests/harness/library-mode.test.ts | 26 +- tests/harness/library-static-types.test.ts | 23 +- tests/harness/node-island.test.ts | 161 +++- tests/harness/npm-static.test.ts | 18 + .../self-hosting-native-toolchain.test.ts | 2 +- tests/library-mode/npm-refuse/lib-minified.ts | 4 +- tests/library-mode/npm-static/lib.ts | 2 +- .../npm-static/node_modules/mathkit/index.js | 2 +- 171 files changed, 4583 insertions(+), 311 deletions(-) create mode 100644 packages/compiler/src/node-islands/modules.test.ts create mode 100644 packages/compiler/src/node-islands/modules.ts create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7447-union-record-arm-width.mjs.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7448-union-record-arm-width.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7460-transform-writev.mjs.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7461-transform-writev.cjs.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7462-transform-writev.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7463-stream-option-values.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7464-stream-option-values.mjs.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7465-transform-callback-values.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7466-stream-construct-assign.mjs.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7500-pipeline-callback-undefined.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7501-pipeline-callback-undefined.mjs.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7502-readline-cursor-undefined.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7543-var-list-declarator-order.js.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7545-string-regex-checked-arguments.js.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7548-cjs-receiverless-builtin-calls/main.cjs.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7549-cjs-minified-esmodule-stamp/main.cjs.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7553-var-for-in-of-zero-iterations.js.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7620-js-class-view-descriptors.js.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7661-node-call-modules/main.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7662-node-call-exit-events/main.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/diagnostics/stream-runtime-option-order.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/7546-minified-esm-rules/main.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/7547-minified-cjs-chunks/main.ts.json create mode 100644 tests/corpus/7447-union-record-arm-width.mjs create mode 100644 tests/corpus/7448-union-record-arm-width.ts create mode 100644 tests/corpus/7460-transform-writev.mjs create mode 100644 tests/corpus/7461-transform-writev.cjs create mode 100644 tests/corpus/7462-transform-writev.ts create mode 100644 tests/corpus/7463-stream-option-values.ts create mode 100644 tests/corpus/7464-stream-option-values.mjs create mode 100644 tests/corpus/7465-transform-callback-values.ts create mode 100644 tests/corpus/7466-stream-construct-assign.mjs create mode 100644 tests/corpus/7500-pipeline-callback-undefined.ts create mode 100644 tests/corpus/7501-pipeline-callback-undefined.mjs create mode 100644 tests/corpus/7502-readline-cursor-undefined.ts create mode 100644 tests/corpus/7543-var-list-declarator-order.js create mode 100644 tests/corpus/7545-string-regex-checked-arguments.js create mode 100644 tests/corpus/7548-cjs-receiverless-builtin-calls/lib.cjs create mode 100644 tests/corpus/7548-cjs-receiverless-builtin-calls/main.cjs create mode 100644 tests/corpus/7549-cjs-minified-esmodule-stamp/main.cjs create mode 100644 tests/corpus/7549-cjs-minified-esmodule-stamp/module-stamped.cjs create mode 100644 tests/corpus/7549-cjs-minified-esmodule-stamp/stamped.cjs create mode 100644 tests/corpus/7553-var-for-in-of-zero-iterations.js create mode 100644 tests/corpus/7620-js-class-view-descriptors.js create mode 100644 tests/corpus/7661-node-call-modules/lib/entry.cjs create mode 100644 tests/corpus/7661-node-call-modules/lib/esm.mjs create mode 100644 tests/corpus/7661-node-call-modules/lib/factor.mjs create mode 100644 tests/corpus/7661-node-call-modules/lib/util.js create mode 100644 tests/corpus/7661-node-call-modules/main.ts create mode 100644 tests/corpus/7661-node-call-modules/node_modules/greeter/data/message.txt create mode 100644 tests/corpus/7661-node-call-modules/node_modules/greeter/index.js create mode 100644 tests/corpus/7661-node-call-modules/node_modules/greeter/package.json create mode 100644 tests/corpus/7661-node-call-modules/node_modules/shout/index.js create mode 100644 tests/corpus/7661-node-call-modules/node_modules/shout/package.json create mode 100644 tests/corpus/7661-node-call-modules/settings.json create mode 100644 tests/corpus/7662-node-call-exit-events/listeners.cjs create mode 100644 tests/corpus/7662-node-call-exit-events/main.ts create mode 100644 tests/diagnostics/stream-runtime-option-order.ts create mode 100644 tests/fixtures/node-island/gc-stress.ts create mode 100644 tests/fixtures/node-island/gc.cjs create mode 100644 tests/fixtures/node-island/worker.ts create mode 100644 tests/fixtures/npm/cases/7546-minified-esm-rules/main.ts create mode 100644 tests/fixtures/npm/cases/7547-minified-cjs-chunks/main.ts create mode 100644 tests/fixtures/npm/node_modules/gtminicache/dist/index.d.ts create mode 100644 tests/fixtures/npm/node_modules/gtminicache/dist/index.js create mode 100644 tests/fixtures/npm/node_modules/gtminicache/package.json create mode 100644 tests/fixtures/npm/node_modules/gtminimark/dist/index.d.ts create mode 100644 tests/fixtures/npm/node_modules/gtminimark/dist/index.js create mode 100644 tests/fixtures/npm/node_modules/gtminimark/package.json create mode 100644 tests/harness/__snapshots__/stream-runtime-option-order.ts.txt create mode 100644 tests/harness/corpus-costs/7447-union-record-arm-width.mjs.json create mode 100644 tests/harness/corpus-costs/7448-union-record-arm-width.ts.json create mode 100644 tests/harness/corpus-costs/7460-transform-writev.mjs.json create mode 100644 tests/harness/corpus-costs/7461-transform-writev.cjs.json create mode 100644 tests/harness/corpus-costs/7462-transform-writev.ts.json create mode 100644 tests/harness/corpus-costs/7463-stream-option-values.ts.json create mode 100644 tests/harness/corpus-costs/7464-stream-option-values.mjs.json create mode 100644 tests/harness/corpus-costs/7465-transform-callback-values.ts.json create mode 100644 tests/harness/corpus-costs/7466-stream-construct-assign.mjs.json create mode 100644 tests/harness/corpus-costs/7500-pipeline-callback-undefined.ts.json create mode 100644 tests/harness/corpus-costs/7501-pipeline-callback-undefined.mjs.json create mode 100644 tests/harness/corpus-costs/7502-readline-cursor-undefined.ts.json create mode 100644 tests/harness/corpus-costs/7543-var-list-declarator-order.js.json create mode 100644 tests/harness/corpus-costs/7545-string-regex-checked-arguments.js.json create mode 100644 tests/harness/corpus-costs/7548-cjs-receiverless-builtin-calls/main.cjs.json create mode 100644 tests/harness/corpus-costs/7549-cjs-minified-esmodule-stamp/main.cjs.json create mode 100644 tests/harness/corpus-costs/7553-var-for-in-of-zero-iterations.js.json create mode 100644 tests/harness/corpus-costs/7620-js-class-view-descriptors.js.json create mode 100644 tests/harness/corpus-costs/7661-node-call-modules/main.ts.json create mode 100644 tests/harness/corpus-costs/7662-node-call-exit-events/main.ts.json diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4ffecc8e2..57e79148a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -52,9 +52,13 @@ jobs: const stackBase = pr?.stack?.base?.ref; const prBase = pr?.base?.ref; // Position 1 can already be merged; compare the current base branches. + // A PR into the default branch is the lowest unmerged layer of any + // stack, whatever the stack metadata names as its base. + const defaultBranch = event.repository?.default_branch; const upperLayer = process.env.GITHUB_EVENT_NAME === 'pull_request' && sameRepository && trustedAuthor && typeof stackBase === 'string' && stackBase.length > 0 && - typeof prBase === 'string' && prBase.length > 0 && stackBase !== prBase; + typeof prBase === 'string' && prBase.length > 0 && stackBase !== prBase && + prBase !== defaultBranch; const full = !upperLayer; appendFileSync(process.env.GITHUB_OUTPUT, `full=${full}\n`); console.log(full ? 'Run full CI' : 'Defer full CI until this PR becomes the lowest unmerged layer'); diff --git a/.github/workflows/libnode-packs.yml b/.github/workflows/libnode-packs.yml index ebe928145..fab3b630f 100644 --- a/.github/workflows/libnode-packs.yml +++ b/.github/workflows/libnode-packs.yml @@ -32,7 +32,9 @@ on: - ".github/workflows/libnode-packs.yml" concurrency: - group: ${{ github.workflow }}-${{ github.ref }} + # A fixed name: under workflow_call, github.workflow is the caller's, and + # sharing the caller's group would cancel the run as a deadlock. + group: libnode-packs-${{ github.ref }} cancel-in-progress: ${{ github.event_name == 'pull_request' }} permissions: diff --git a/docs/content/docs/dependencies.mdx b/docs/content/docs/dependencies.mdx index 14130b234..a9c5ae138 100644 --- a/docs/content/docs/dependencies.mdx +++ b/docs/content/docs/dependencies.mdx @@ -53,7 +53,7 @@ Type-only imports follow Node.js's type stripping. `import type` and `export typ scriptc compiles a package automatically when it: - ships its own type declarations, or is typed by its DefinitelyTyped `@types/` package, or is a dependency of another package that has no declarations at all, and -- ships readable JavaScript: unminified, or minified by a bundler whose helpers keep their names. +- ships JavaScript that isn't a webpack bundle. Minified JavaScript compiles like unminified JavaScript. The declarations check your program; the package's JavaScript compiles with the types scriptc infers from it. A program that doesn't typecheck against the inferred types, for example because an `@types/` package describes another release, fails the build at that use. diff --git a/docs/content/limitations/builtin-apis/streams-and-readline.mdx b/docs/content/limitations/builtin-apis/streams-and-readline.mdx index fe6bb2dfb..45a784742 100644 --- a/docs/content/limitations/builtin-apis/streams-and-readline.mdx +++ b/docs/content/limitations/builtin-apis/streams-and-readline.mdx @@ -18,7 +18,7 @@ Stages may take the `{ signal }` argument. With a function destination, `pipeline()` returns an object-mode PassThrough that receives an async generator destination's values, or an async function destination's value unless it is `null` or `undefined`. `@types/node` types that result as `NodeJS.WritableStream`, so `for await` over it needs the `PassThrough` type (`pipeline(...) as PassThrough`, or a JSDoc `@type`). - In typed code, the callback of a pipeline with an async function destination receives `null` instead of `undefined` when there is no error. + In typed code, a pipeline callback with an async function or async generator destination receives `null` for no error when its error parameter cannot hold `undefined` (@types/node declares it `NodeJS.ErrnoException | null`), and `undefined` when it can, as in Node. `Readable.from` accepts `objectMode` and `highWaterMark` options. The `Readable` operators (`map`, `filter`, `forEach`, `toArray`, `some`, `find`, `every`, `flatMap`, `drop`, `take`, and `reduce`) accept `concurrency`, `highWaterMark`, and `signal` options. Their callbacks may take the `{ signal }` argument, `map` may return any value, and `flatMap` may return any iterable, async iterable, or stream. @@ -32,9 +32,10 @@ `readable.compose(stage, { signal })` is `stream.compose(readable, stage)` with the signal added; the options are an object literal whose `signal` is a variable or a property read. `pipeline` from `node:stream/promises` accepts a trailing `{ signal, end }` object, with or without function stages, and `finished` accepts `readable`, `writable`, `error`, `cleanup`, and `signal` options, with boolean literals for the flags. Stream constructors accept a `signal` option in an options object literal, and `stream.addAbortSignal` accepts any stream. - The `construct` option and `_construct` method, `decodeStrings`, and `defaultEncoding` work on every stream class, and the `writev` option and `_writev` method on Writable and Duplex; `writev` on a Transform or PassThrough, assigning `_construct` after construction, and a non-literal `decodeStrings` or `defaultEncoding` in an options object literal are unsupported. + The `construct` option and `_construct` method, `decodeStrings`, and `defaultEncoding` work on every stream class, and the `writev` option and `_writev` method on every class with a writable side. + A `decodeStrings` or `defaultEncoding` value known only at run time is unsupported after an option member whose value has effects (a call or an assignment). + In typed code, a `writev` entry's `encoding` declared as a string receives `""` where Node passes `undefined` or `null` (an object-mode write). `setDefaultEncoding()` takes any encoding Node knows. - A `transform` or `flush` option callback whose completion callback types `data` as `any`, as @types/node's `TransformCallback` does, may pass a Buffer, a string, or nothing; any other value throws a TypeError where the callback is called, instead of Node's `ERR_INVALID_ARG_TYPE` stream error. `node:readline` and `node:readline/promises` (also reached as `readline.promises`) interfaces, from `createInterface(options)` or `createInterface(input, output, completer, terminal)`, read `process.stdin` or a readable stream: `question`, `prompt`, `write` (with a key object literal of `name`, `ctrl`, `meta`, and `shift`), `pause`, `resume`, `'line'`/`'close'`/`'pause'`/`'resume'`/`'error'`/`'history'`/`'SIGINT'`/`'SIGTSTP'`/`'SIGCONT'` listeners, and `for await` compile. `on`, `once`, and `addListener` return the interface, so registrations chain. The `history`, `historySize`, `removeHistoryDuplicates`, `completer`, `tabSize`, and `escapeCodeTimeout` options are validated as in Node. @@ -42,7 +43,7 @@ The output's `'resize'` event does not redraw the line. The cursor never stops between the two UTF-16 units of a supplementary character: where Node's Up or Down movement through a multiline line would, the cursor moves to the character's start or end. In JavaScript, a `'history'` listener receives a copy of the history; its changes are written back when the listener returns, but the copy is not `rl.history` itself. - `rl.history`, `historySize`, `historyIndex`, `crlfDelay`, `tabSize`, `escapeCodeTimeout`, `line`, `cursor`, and `terminal` read as in Node, except that `rl.cursor` of an interface that is not a terminal throws instead of answering `undefined`. + `rl.history`, `historySize`, `historyIndex`, `crlfDelay`, `tabSize`, `escapeCodeTimeout`, `line`, `cursor`, and `terminal` read as in Node; `rl.cursor` is `undefined` on an interface that is not a terminal. `rl[Symbol.dispose]()` closes the interface, and `instanceof` with `readline.Interface` or `readlinePromises.Interface` answers whether the interface came from that module, as in Node. In JavaScript, the input and output can be any expression that evaluates to a stream, such as a function's result or an object's member; a value that is not a stream is unsupported. In JavaScript, `question()` and `setPrompt()` accept a value that is not a string: when such a prompt is written, an interface that is not a terminal throws `ERR_INVALID_ARG_TYPE` as Node does (the `readline/promises` question rejects with it) and the question stays pending, and a terminal interface writes `String(value)`. diff --git a/packages/cli/src/bootstrap.ts b/packages/cli/src/bootstrap.ts index 9eb49f823..c8c927739 100644 --- a/packages/cli/src/bootstrap.ts +++ b/packages/cli/src/bootstrap.ts @@ -70,9 +70,6 @@ async function tryFastPath(): Promise { values["profile-generate"] !== undefined || values["profile-use"] !== undefined || values.sparse || - // The full compiler keys these builds by the recorded libnode pack. - values.libnode !== undefined || - process.env["SCRIPTC_LIBNODE"] !== undefined || (values["external-types"] ?? []).length > 0 ) return null; @@ -159,6 +156,8 @@ async function tryFastPath(): Promise { ) : startup.executableNativeEnvironmentFingerprint(), nodeVersion: process.version, + // An entry of a program that calls into Node.js serves only the same pack. + libnodeIdentity: () => startup.libnodeCacheKey(values.libnode, process.env, dirname(input)), }); if (hit === null) return null; if (!values["keep-llvm"]) await rm(hit.llvmPath, { force: true }); diff --git a/packages/cli/test/native-link-info.test.ts b/packages/cli/test/native-link-info.test.ts index a5edb11bb..8ac579449 100644 --- a/packages/cli/test/native-link-info.test.ts +++ b/packages/cli/test/native-link-info.test.ts @@ -72,7 +72,7 @@ describe.runIf(supported)("macOS arm64 native link info", () => { minimum_os: "14.0", }, program: { object, entry_symbol: "main" }, - runtime_abi: { version: 36, marker: "scr_runtime_abi_v36" }, + runtime_abi: { version: 37, marker: "scr_runtime_abi_v37" }, runtime_pack: { kind: "precompiled", package: "@scriptc/runtime-darwin-arm64", @@ -165,6 +165,6 @@ describe.runIf(supported)("macOS arm64 native link info", () => { () => null, (failure: { stderr?: string }) => failure, ); - expect(error?.stderr).toContain("scr_runtime_abi_v36"); + expect(error?.stderr).toContain("scr_runtime_abi_v37"); }); }); diff --git a/packages/compiler/src/backend/llvm/dyn.ts b/packages/compiler/src/backend/llvm/dyn.ts index 94581cfa0..86626d304 100644 --- a/packages/compiler/src/backend/llvm/dyn.ts +++ b/packages/compiler/src/backend/llvm/dyn.ts @@ -370,6 +370,36 @@ export class LlDyn { B.line(`call void @scr_dyn_build_add(ptr %d, ptr @${name}, ptr ${value})`); } + /** The order a dyn value tries a union's arms in: the canonical order, + * except that among the record arms, those with more required fields go + * first, then those with more fields. Records are width-tolerant, so an + * arm with fewer fields (`{}`, or one whose fields are all optional, + * matches every object) would otherwise take an object that a wider arm + * also matches and drop the fields only the wider arm has. */ + private unionArmOrder(def: IrUnionDef): number[] { + const width = (i: number): [number, number] | null => { + const arm = def.arms[i]!; + if (arm.kind !== "record") return null; + const shape = this.host.recordsById.get(arm.shapeId); + if (!shape || shape.tuple) return null; + const required = shape.fields.filter( + (field) => + field.type.kind !== "union" || + !this.host.unionsById.get(field.type.unionId)?.arms.some((x) => x.kind === "undefinedT"), + ).length; + return [required, shape.fields.length]; + }; + const order = def.arms.map((_, i) => i); + const records = order.filter((i) => width(i) !== null); + const widest = [...records].sort((x, y) => { + const [rx, fx] = width(x)!; + const [ry, fy] = width(y)!; + return ry - rx || fy - fx || x - y; + }); + records.forEach((slot, k) => (order[slot] = widest[k]!)); + return order; + } + /** Branch to a variant only after its literal discriminator and complete * structure match. Use the same path for predicates and builders. */ private unionArmMatch( @@ -2368,9 +2398,10 @@ export class LlDyn { throw new InternalCompilerError( `llvm emitter bug: dynCheck of unknown union ${t.unionId}`, ); - // Arms in CANONICAL order, first FULL match wins. The matched - // arm's builder can no longer fail. - def.arms.forEach((arm, i) => { + // Arms in unionArmOrder, first FULL match wins. The matched arm's + // builder can no longer fail. + for (const i of this.unionArmOrder(def)) { + const arm = def.arms[i]!; const lHit = B.newLabel("dcu.h"); const lNext = B.newLabel("dcu.n"); this.unionArmMatch(B, def, i, lHit, lNext, preserveRefs); @@ -2414,7 +2445,7 @@ export class LlDyn { B.terminate(`ret ptr ${u}`); } B.startBlock(lNext); - }); + } if (preserveRefs) { B.line( `call void @scr_dyn_check_fail(ptr %path, ptr ${host.cstr("reference with its original layout")}, ptr %d)`, diff --git a/packages/compiler/src/backend/llvm/emit-module.ts b/packages/compiler/src/backend/llvm/emit-module.ts index c16834910..f4f3bd0f1 100644 --- a/packages/compiler/src/backend/llvm/emit-module.ts +++ b/packages/compiler/src/backend/llvm/emit-module.ts @@ -17,6 +17,7 @@ import { } from "../mangle.js"; import { type IrGlobal, + type IrNodeIslandModules, type IrType, RUNTIME_ERROR_CLASSES, RUNTIME_EVENT_CLASS, @@ -57,7 +58,8 @@ import { ffiNativeTypeLl, } from "./common.js"; import { literalCapWord, literalCapWord32 } from "./string-key-hash.js"; -import { hasLoneSurrogate, wtf8Bytes } from "../literals.js"; +import { hasLoneSurrogate, llvmBytes, wtf8Bytes } from "../literals.js"; +import { nodeIslandManifest } from "../../node-islands/modules.js"; import { undefinedArmTag } from "../../ir/analysis.js"; import { NULLABLE_ABSENT, NULLABLE_NULL } from "./nullable-unions.js"; import { literalEqualityDefinition } from "./string-literal-equality.js"; @@ -381,6 +383,11 @@ export function emitModuleParts(emitter: LlEmitter): string[] { emitter.cstr(emitter.mod.nodeIsland?.nodeVersion ?? ""), ] : null; + // The island module tree: its manifest, and its files unless they ship + // beside the executable (node-islands/modules.ts). + const nodeIslandModules = usesNodeIsland ? emitter.mod.nodeIsland?.modules : undefined; + const nodeIslandModulesCall = + nodeIslandModules === undefined ? null : nodeIslandModulesInstall(emitter, nodeIslandModules); if (emitter.ffiHasForeignCallback) emitter.declare(`declare void @scr_ffi_install()`); if (usesStream) emitter.declare(`declare void @scr_stream_install()`); if (usesNetLookup) emitter.declare(`declare void @scr_net_lookup_install()`); @@ -732,6 +739,7 @@ export function emitModuleParts(emitter: LlEmitter): string[] { for (let site = 0; site < emitter.jsIcSiteCount; site++) out.push(`@sc_js_ic_${site} = internal ${tl}global [${JS_IC_WORDS} x i64] zeroinitializer`); } + if (nodeIslandModulesCall !== null) out.push(...nodeIslandModulesCall.globals); for (const table of emitter.constantNumericTables.values()) { const n = table.values.length; let tableTmp = 0; @@ -1027,6 +1035,7 @@ export function emitModuleParts(emitter: LlEmitter): string[] { ` call void @scr_node_island_install(ptr ${nodeIslandConfig[0]}, ptr ${nodeIslandConfig[1]})`, ] : []), + ...(nodeIslandModulesCall === null ? [] : [nodeIslandModulesCall.call]), ...(emitter.ffiHasForeignCallback ? [` call void @scr_ffi_install()`] : []), ...(snapshotsTlsCa ? [` call void @scr_tls_ca_install()`] : []), ...(usesFetch ? [` call void @scr_fetch_install()`] : []), @@ -1171,6 +1180,33 @@ export function emitModuleParts(emitter: LlEmitter): string[] { /** The shared abort helpers (emitted only when referenced): the OOM * abort of untraced shape allocation and the invalid-union-tag abort — * both print the runtime ABI’s exact message on fd 2 and abort. */ +/** The island module tree's install call before %main, and the constant + * that carries its files when the executable ships them itself + * (scr_node_island.c: scr_node_island_install_modules). */ +function nodeIslandModulesInstall( + emitter: LlEmitter, + modules: IrNodeIslandModules, +): { call: string; globals: string[] } { + emitter.declare( + `declare void @scr_node_island_install_modules(ptr, ${emitter.sizeType}, ptr, ${emitter.sizeType}, ptr)`, + ); + const manifest = nodeIslandManifest(modules); + const manifestPtr = emitter.cstr(manifest); + const location = emitter.cstr(modules.location ?? ""); + const data = modules.location === undefined ? emitter.nodeIslandModuleData : null; + const globals = + data === null || data.length === 0 + ? [] + : [ + `@sc_node_island_modules = private unnamed_addr constant [${data.length} x i8] c"${llvmBytes(data, false)}", align 16`, + ]; + const st = emitter.sizeType; + return { + call: ` call void @scr_node_island_install_modules(ptr ${manifestPtr}, ${st} ${Buffer.byteLength(manifest, "utf8")}, ptr ${globals.length > 0 ? "@sc_node_island_modules" : "null"}, ${st} ${globals.length > 0 ? data!.length : 0}, ptr ${location})`, + globals, + }; +} + export function helperDefs(emitter: LlEmitter): string[] { const defs: string[] = []; const msgHelper = (fnName: string, msgSym: string, msg: string): void => { diff --git a/packages/compiler/src/backend/llvm/emitter.ts b/packages/compiler/src/backend/llvm/emitter.ts index 050c3fb90..545d146a4 100644 --- a/packages/compiler/src/backend/llvm/emitter.ts +++ b/packages/compiler/src/backend/llvm/emitter.ts @@ -473,6 +473,9 @@ export interface LlvmTargetOptions { * teardown after `process.exit` to a helper process (`--detached-exit`). * Executables only; the runtime's weak default keeps it off. */ detachedExit?: boolean; + /** The files of the program's Node island module tree, in manifest order + * (node-islands/modules.ts), when the executable carries the tree. */ + nodeIslandModuleData?: Uint8Array; /** Translated WebAssembly memories check bounds explicitly instead of * relying on guard pages (wasm-modules.ts wasmBoundsChecks). */ wasmBoundsChecks?: boolean; @@ -1036,6 +1039,7 @@ export class LlEmitter { readonly emitLibraryIdentity: boolean; readonly runtimeAbiMarker: boolean; readonly detachedExit: boolean; + readonly nodeIslandModuleData: Uint8Array | null; readonly wasmBoundsChecks: boolean; /** Interned string literals: UTF-8 text → { symbol, byte length } — * first-use order, the runtime ABI’s determinism discipline. */ @@ -1618,6 +1622,7 @@ export class LlEmitter { this.emitLibraryIdentity = options.emitLibraryIdentity !== false; this.runtimeAbiMarker = options.runtimeAbiMarker === true; this.detachedExit = options.detachedExit === true; + this.nodeIslandModuleData = options.nodeIslandModuleData ?? null; this.wasmBoundsChecks = options.wasmBoundsChecks === true || wasmBoundsChecks({ diff --git a/packages/compiler/src/backend/llvm/expr-stream-callbacks.ts b/packages/compiler/src/backend/llvm/expr-stream-callbacks.ts index 14a2e3432..b368c91f4 100644 --- a/packages/compiler/src/backend/llvm/expr-stream-callbacks.ts +++ b/packages/compiler/src/backend/llvm/expr-stream-callbacks.ts @@ -727,6 +727,17 @@ export function streamCbThunkFor( const ty = host.llType(p); host.declare(`declare zeroext i1 @scr_exc_pending()`); if (hasThis) host.declare(`declare void @scr_stream_release_v(ptr)`); + // An entry `encoding` declared as a string (the declarations' + // `encoding: string`) receives "" where Node passes undefined or + // null (an object-mode write), as a write's encoding parameter does. + const entry = + p.kind === "array" && p.elem.kind === "record" + ? host.recordsById.get(p.elem.shapeId) + : undefined; + if (entry?.fields.some((f) => f.name === "encoding" && f.type.kind === "string")) { + host.declare(`declare void @scr_stream_writev_fill_encodings(ptr)`); + d.push(` call void @scr_stream_writev_fill_encodings(ptr %chunks)`); + } d.push( ` %vt${i} = call ${ty === "i1" ? "zeroext i1" : ty} @${helper}(ptr %chunks, ptr null)`, ` %vtbad${i} = call zeroext i1 @scr_exc_pending()`, diff --git a/packages/compiler/src/backend/llvm/lib-shared.ts b/packages/compiler/src/backend/llvm/lib-shared.ts index 52af89584..2ad98ab7c 100644 --- a/packages/compiler/src/backend/llvm/lib-shared.ts +++ b/packages/compiler/src/backend/llvm/lib-shared.ts @@ -1142,6 +1142,8 @@ export const LIB_FN_SYMS: Record = { "readable.pushEncoding": "scr_stream_set_push_encoding", "stream.objectMode": "scr_stream_object_mode", "stream.decodeStrings": "scr_stream_decode_strings", + "stream.optionValues": "scr_stream_option_values", + "stream.checkEncodingOption": "scr_stream_check_encoding_option", "stream.duplexFromGen": "scr_stream_duplex_from_gen", "stream.duplexFromDyn": "scr_stream_duplex_from_dyn", "stream.duplexifyReadable": "scr_stream_duplexify_readable", @@ -1776,6 +1778,7 @@ export const USES_TIMERS_LIB_FNS = new Set([ "stream.setTransform", "stream.setFlush", "stream.setConstruct", + "stream.replaceConstruct", "stream.duplexFromGen", "stream.duplexFromDyn", "stream.duplexifyReadable", diff --git a/packages/compiler/src/backend/llvm/lib-stream.ts b/packages/compiler/src/backend/llvm/lib-stream.ts index 5fd4af1a9..fdbf13d4f 100644 --- a/packages/compiler/src/backend/llvm/lib-stream.ts +++ b/packages/compiler/src/backend/llvm/lib-stream.ts @@ -189,6 +189,7 @@ export function emitStreamLibCall(host: LlvmEmitterContext, e: LibCallExpr): LlV e.fn === "stream.setTransform" || e.fn === "stream.setFlush" || e.fn === "stream.setConstruct" || + e.fn === "stream.replaceConstruct" || e.fn === "stream.setWritev" || e.fn === "stream.setWritevDefault" ) { @@ -203,6 +204,7 @@ export function emitStreamLibCall(host: LlvmEmitterContext, e: LibCallExpr): LlV "stream.setTransform": "t", "stream.setFlush": "l", "stream.setConstruct": "c", + "stream.replaceConstruct": "c", "stream.setWritev": "v", "stream.setWritevDefault": "v", }; @@ -214,6 +216,7 @@ export function emitStreamLibCall(host: LlvmEmitterContext, e: LibCallExpr): LlV "stream.setTransform": "scr_stream_set_transform", "stream.setFlush": "scr_stream_set_flush", "stream.setConstruct": "scr_stream_set_construct", + "stream.replaceConstruct": "scr_stream_replace_construct", "stream.setWritev": "scr_stream_set_writev", "stream.setWritevDefault": "scr_stream_set_writev_default", }; diff --git a/packages/compiler/src/backend/runtime-abi.ts b/packages/compiler/src/backend/runtime-abi.ts index 582e516ae..11bde6c2b 100644 --- a/packages/compiler/src/backend/runtime-abi.ts +++ b/packages/compiler/src/backend/runtime-abi.ts @@ -2,8 +2,8 @@ * of the link contract: a program object keeps this symbol undefined and the * matching runtime defines it, so incompatible manual links fail before the * program can start. */ -export const RUNTIME_ABI_VERSION = 36 as const; -export const RUNTIME_ABI_MARKER = "scr_runtime_abi_v36" as const; +export const RUNTIME_ABI_VERSION = 37 as const; +export const RUNTIME_ABI_MARKER = "scr_runtime_abi_v37" as const; /** Object consumption is useful today, but the complete scr_* surface may * still change before 1.0. The versioned marker prevents accidental mixing; diff --git a/packages/compiler/src/backend/runtime-pack-targets.test.ts b/packages/compiler/src/backend/runtime-pack-targets.test.ts index 739f6a385..44c3d6290 100644 --- a/packages/compiler/src/backend/runtime-pack-targets.test.ts +++ b/packages/compiler/src/backend/runtime-pack-targets.test.ts @@ -52,7 +52,7 @@ async function fixture(target: NativeTargetSpec, workers = false) { object_format: target.objectFormat, minimum_os: target.minimumOs, }, - runtime_abi: { version: 36, marker: "scr_runtime_abi_v36" }, + runtime_abi: { version: 37, marker: "scr_runtime_abi_v37" }, compiler: { command: "fixture", identity: "fixture", target: target.llvmTriple }, macros: { executable: [], excluded: ["SCR_LIB"], sanitizer: "external-toolchain-required" }, flavors: { diff --git a/packages/compiler/src/backend/runtime-pack.test.ts b/packages/compiler/src/backend/runtime-pack.test.ts index 74bc32e73..e3bc999a7 100644 --- a/packages/compiler/src/backend/runtime-pack.test.ts +++ b/packages/compiler/src/backend/runtime-pack.test.ts @@ -127,7 +127,7 @@ async function fixture(target: NativeTargetSpec = MACOS_ARM64_TARGET) { object_format: target.objectFormat, minimum_os: target.minimumOs, }, - runtime_abi: { version: 36, marker: "scr_runtime_abi_v36" }, + runtime_abi: { version: 37, marker: "scr_runtime_abi_v37" }, compiler: { command: "clang", identity: "fixture clang", diff --git a/packages/compiler/src/cli/usage.ts b/packages/compiler/src/cli/usage.ts index ca3ade8f6..570d81a8f 100644 --- a/packages/compiler/src/cli/usage.ts +++ b/packages/compiler/src/cli/usage.ts @@ -82,9 +82,9 @@ Options: --npm-static compile the named npm packages' shipped JS statically as program modules (repeatable; "auto" opts in every eligible - direct import: own .d.ts, unminified JS, no build-transform - markers; "pkg..." also names every package pkg depends - on). Packages that fail preflight are refused with a + direct import: own .d.ts, no build-transform markers; + "pkg..." also names every package pkg depends on). + Packages that fail preflight are refused with a diagnostic. Eligible dependencies compile automatically. --provenance-sources EXPERIMENTAL: compile npm dependencies from their diff --git a/packages/compiler/src/diagnostics/diagnostic.ts b/packages/compiler/src/diagnostics/diagnostic.ts index 6d6e81516..96b639b96 100644 --- a/packages/compiler/src/diagnostics/diagnostic.ts +++ b/packages/compiler/src/diagnostics/diagnostic.ts @@ -66,7 +66,8 @@ * likely-hot functions that use untyped values (SC6101, * diagnostics/dynamic-tier.ts) * SC8xxx Node islands: the program calls into Node.js, and the build - * finds no usable libnode pack (SC8005) + * finds no usable libnode pack (SC8005), or cannot collect the + * modules a call loads (SC8006) * SC9xxx internal compiler errors (still source-anchored) */ import type { SrcLoc } from "../ir/ir.js"; @@ -133,6 +134,18 @@ export function nodeIslandNodeDiag(message: string, file: string): ScrDiagnostic }; } +/** SC8006 — a Node island call whose modules the build cannot collect into + * the executable: a computed specifier, or a module of the program that + * loads others through computed specifiers. */ +export function nodeIslandModuleDiag(message: string, loc: SrcLoc): ScrDiagnostic { + return { + code: "SC8006", + message, + loc, + hint: "name the module with a string literal (a relative path or a package name); the build ships the modules it can see with the executable", + }; +} + /* ── SC6xxx: divergences from Node (warnings) ─────────────────────────── */ /** SC6001 — a value passed to (or declared as) a narrower record type is diff --git a/packages/compiler/src/executable/executable-cache.ts b/packages/compiler/src/executable/executable-cache.ts index b9be0724d..664b23f65 100644 --- a/packages/compiler/src/executable/executable-cache.ts +++ b/packages/compiler/src/executable/executable-cache.ts @@ -121,6 +121,9 @@ interface EarlyExecutableCacheStamp { native: EarlyExecutableNativeFeatures; /** The build's performance warnings (SC61xx), which a cache hit repeats. */ performance?: ScrDiagnostic[]; + /** Programs that call into Node.js: the libnode pack identity the build + * recorded (EarlyExecutableCacheOptions.libnodeIdentity). */ + libnode?: string; integrity: string; } @@ -139,9 +142,12 @@ export interface EarlyExecutableCacheOptions { windowsSubsystem?: "gui"; /** Omitted unless --detached-exit is set. */ detachedExit?: true; - /** Omitted unless --libnode or SCRIPTC_LIBNODE is set (calls into Node.js - * record the pack). */ - libnode?: string; + /** The identity of the libnode pack a program that calls into Node.js + * records (node-islands/libnode.ts libnodeCacheKey), or undefined when the + * build names none. Not part of the key: an entry of a program that calls + * into Node records it, and a hit requires the same identity, so the pack + * settings never change the key of a program that does not. */ + libnodeIdentity?: () => string | undefined; /** Omitted without --profile-generate/--profile-use; otherwise the * pattern, or the digest of the profile's bytes. */ profile?: string; @@ -283,7 +289,6 @@ function cacheKey(options: EarlyExecutableCacheOptions): string { ...(options.strip ? ["strip"] : []), ...(options.windowsSubsystem === "gui" ? ["windows-subsystem-gui"] : []), ...(options.detachedExit ? ["detached-exit"] : []), - ...(options.libnode !== undefined ? [`libnode:${options.libnode}`] : []), ...(options.profile !== undefined ? [options.profile] : []), ...(options.checker !== undefined ? [`checker:${options.checker}`] : []), ...(options.youngNursery ? ["young-nursery"] : []), @@ -326,7 +331,6 @@ function routeKey(options: Omit = { line: 0, @@ -1341,7 +1358,7 @@ export function lowerReadlineCreateCall( /** Property reads on an Interface: `history` (the history option itself, * or the interface's own empty array), `historySize`, `historyIndex`, * `crlfDelay`, `tabSize`, `escapeCodeTimeout`, `line`, `terminal` and - * `cursor` (a non-terminal interface's undefined cursor throws SC2020). + * `cursor` (undefined on a non-terminal interface). * Null for other receivers. */ export function lowerReadlineProperty( lowerer: Lowerer, @@ -1373,10 +1390,38 @@ export function lowerReadlineProperty( loc, }; } - const reads: Record = { + if (name === "cursor") { + // `number | undefined`: a non-terminal interface has no cursor + const recv = lowerer.lowerExprExpecting(access.expression, F64); + const type: IrType = { kind: "union", unionId: lowerer.unions.intern([F64, UNDEFINED_T]) }; + const local = recv.kind === "varRef" ? null : lowerer.declareHiddenLocal("%rlCursor", F64); + const id: IrExpr = local ? { kind: "varRef", localId: local.id, type: F64, loc } : recv; + const result: IrExpr = { + kind: "ternary", + cond: { kind: "libCall", fn: "rl.isTerminal", args: [id], type: BOOL, loc }, + then: lowerer.coerceToExpected( + { kind: "libCall", fn: "rl.cursor", args: [id], type: F64, loc }, + type, + ), + else_: lowerer.coerceToExpected( + { kind: "unitLit", unit: "undefined", type: UNDEFINED_T, loc }, + type, + ), + type, + loc, + }; + if (!local) return result; + return { + kind: "seqExpr", + stmts: [{ kind: "varDecl", localId: local.id, init: recv, loc }], + result, + type, + loc, + }; + } + const reads: Record = { line: ["rl.line", STRING], terminal: ["rl.isTerminal", BOOL], - cursor: ["rl.cursor", F64], }; const read = reads[name]; if (read !== undefined) { diff --git a/packages/compiler/src/frontend/lowering/containers/string-replacement.ts b/packages/compiler/src/frontend/lowering/containers/string-replacement.ts index 780c77d0f..156f74869 100644 --- a/packages/compiler/src/frontend/lowering/containers/string-replacement.ts +++ b/packages/compiler/src/frontend/lowering/containers/string-replacement.ts @@ -2,6 +2,7 @@ import { keyedHelperName } from "../stable-names.js"; import * as ts from "../../ts7/adapter.js"; import { BOOL, + DYN, F64, REGEX, STRING, @@ -14,7 +15,7 @@ import { type IrType, type SrcLoc, } from "../../../ir/ir.js"; -import { numLit, strLit, varRef } from "../../../ir/build.js"; +import { dynUndefinedExpr, numLit, strLit, varRef } from "../../../ir/build.js"; import { InternalCompilerError } from "../../../errors.js"; import { locOf } from "../../program.js"; import type { Lowerer } from "../lowerer.js"; @@ -65,14 +66,14 @@ export function lowerStringReplacement( const search = argument(args[0], false); const replacement = argument(args[1], true); if (search.type.kind === "dyn" || replacement.type.kind === "dyn") { - if (replacement.type.kind === "func") { - lowerer.noLowering(`string .${method} with checked search values and callbacks`, call); - } // Evaluate all inputs once before choosing the search protocol. Regexes // retain their native pattern; plain values use the string-search helper. + // A typed callback receives the regex match's arguments or the plain + // search's (match, offset, subject). const values = [receiver, search, replacement]; const locals = values.map((value) => lowerer.declareHiddenLocal("%replaceArg", value.type)); const refs = locals.map((local) => varRef(local.id, local.type, loc)); + const callback = replacement.type.kind === "func"; const checkedReplacement = replacement.type.kind === "dyn" ? lowerer.coerceInto(args[1] ?? call, refs[2]!, STRING) @@ -91,14 +92,27 @@ export function lowerStringReplacement( ? { kind: "ternary", cond: { kind: "libCall", fn: "dyn.nativeRegexIs", args: [refs[1]!], type: BOOL, loc }, - then: { - kind: "regexIntrinsic", - method, - receiver: refs[0]!, - args: [lowerer.coerceInto(args[0] ?? call, refs[1]!, REGEX), checkedReplacement], - type: STRING, - loc, - }, + then: callback + ? { + kind: "libCall", + fn: "regex.replaceCallback", + args: [ + refs[0]!, + lowerer.coerceInto(args[0] ?? call, refs[1]!, REGEX), + lowerer.coerceToExpected(refs[2]!, DYN), + { kind: "boolLit", value: method === "replaceAll", type: BOOL, loc }, + ], + type: STRING, + loc, + } + : { + kind: "regexIntrinsic", + method, + receiver: refs[0]!, + args: [lowerer.coerceInto(args[0] ?? call, refs[1]!, REGEX), checkedReplacement], + type: STRING, + loc, + }, else_: plain, type: STRING, loc, @@ -148,11 +162,17 @@ function lowerStringReplacementValues( if (callback.kind !== "func") { throw new InternalCompilerError(`string .${method} replacement of type ${callback.kind}`); } + // A checked parameter receives the boxed argument (undefined after the + // subject), and a checked, number or boolean result converts with + // ToString. const full = [STRING, F64, STRING]; + const ret = callback.ret.kind; if ( - callback.ret.kind !== "string" || - callback.params.length > full.length || - !callback.params.every((type, index) => typeEquals(type, full[index]!)) + (ret !== "string" && ret !== "dyn" && ret !== "f64" && ret !== "bool") || + !callback.params.every( + (type, index) => + type.kind === "dyn" || (index < full.length && typeEquals(type, full[index]!)), + ) ) { lowerer.noLowering( `string .${method} callback signature`, @@ -175,13 +195,21 @@ function lowerStringReplacementValues( const chunksType = arrayOf(STRING); const chunks = varRef("chunks.0", chunksType, loc); const builder = stringBuilder(loc, chunks); - const value: IrExpr = { + const result: IrExpr = { kind: "callValue", callee: replace, - args: [needle, match, subject].slice(0, callback.params.length), - type: STRING, + args: callback.params.map((type, index) => { + const arg = [needle, match, subject][index]; + if (arg === undefined) return dynUndefinedExpr(loc); + return type.kind === "dyn" ? lowerer.coerceToExpected(arg, DYN) : arg; + }), + type: callback.ret, loc, }; + const value: IrExpr = + ret === "dyn" + ? { kind: "libCall", fn: "dyn.toStringCoerce", args: [result], type: STRING, loc } + : lowerer.ensureString(result, call); const stop: IrStmt = { kind: "break", loc }; const body: IrStmt[] = [ builder.declare("chunks.0", { kind: "arrayLit", elems: [], type: chunksType, loc }), diff --git a/packages/compiler/src/frontend/lowering/expressions/binary.ts b/packages/compiler/src/frontend/lowering/expressions/binary.ts index 58f6e16b5..17aef7b85 100644 --- a/packages/compiler/src/frontend/lowering/expressions/binary.ts +++ b/packages/compiler/src/frontend/lowering/expressions/binary.ts @@ -2841,6 +2841,24 @@ function lowerLogicalPair(lowerer: Lowerer, expr: ts.BinaryExpression, left: IrE loc, }; } + // A JavaScript default that tsc types `any` (`e.title || null` in an + // instance of a function with untyped parameters, where `e.title` is a + // string): the checked domain, as for union operands above. + if ( + target === null && + isJsSourceFile(ts.getSourceFileOfNode(expr)) && + lowerer.dynConvertible(left.type) && + lowerer.dynConvertible(right.type) + ) { + return { + kind: "logical", + op: op === ts.SyntaxKind.AmpersandAmpersandToken ? "&&" : "||", + left: lowerer.coerceInto(expr.left, left, DYN), + right: lowerer.coerceInto(expr.right, right, DYN), + type: DYN, + loc, + }; + } lowerer.unsupported("SC1042", expr); } return { diff --git a/packages/compiler/src/frontend/lowering/expressions/narrowing.ts b/packages/compiler/src/frontend/lowering/expressions/narrowing.ts index fa8ad686f..01cab9901 100644 --- a/packages/compiler/src/frontend/lowering/expressions/narrowing.ts +++ b/packages/compiler/src/frontend/lowering/expressions/narrowing.ts @@ -33,6 +33,7 @@ import { isOptionalProcessVersionsRead, } from "../builtins/process.js"; import { requestGeneralStreamSurface } from "../general-streams.js"; +import { isOptionalReadlineCursorRead } from "../builtins/readline.js"; import { literalValues } from "../../literal-values.js"; import { literalUnionArm } from "../../union-discriminants.js"; import { checkedArmExtraction } from "../coercions/unions.js"; @@ -617,6 +618,8 @@ export function maybeNarrow(lowerer: Lowerer, expr: IrExpr, node: ts.Node): IrEx if (ts.isPropertyAccessExpression(node) && isOptionalProcessStreamProperty(lowerer, node)) return expr; if (isOptionalProcessVersionsRead(lowerer, node)) return expr; + if (ts.isPropertyAccessExpression(node) && isOptionalReadlineCursorRead(lowerer, node)) + return expr; // Represented fields and index values can include undefined even when // checker inference or lib.d.ts declares strings (RegExp captures). diff --git a/packages/compiler/src/frontend/lowering/lower-calls.ts b/packages/compiler/src/frontend/lowering/lower-calls.ts index 6b00ddb44..6363a1110 100644 --- a/packages/compiler/src/frontend/lowering/lower-calls.ts +++ b/packages/compiler/src/frontend/lowering/lower-calls.ts @@ -308,6 +308,22 @@ function mayHoldError(lowerer: Lowerer, type: ts.Type, errorType: ts.Type | null ); } +/** The `f` of a `(0, f)` callee (a numeric left operand, so dropping it + * changes only the call's receiver), or null. */ +function receiverlessCalleeOf(callee: ts.Expression): ts.Expression | null { + let e = callee; + while (ts.isParenthesizedExpression(e)) e = e.expression; + if ( + !ts.isBinaryExpression(e) || + e.operatorToken.kind !== ts.SyntaxKind.CommaToken || + !ts.isNumericLiteral(e.left) + ) + return null; + let right = e.right; + while (ts.isParenthesizedExpression(right)) right = right.expression; + return right; +} + export function lowerCall(lowerer: Lowerer, expr: ts.CallExpression): IrExpr { const loc = locOf(expr); if (lowerer.errorAccessorReflection === null) noteErrorAccessorReflection(lowerer, expr); @@ -331,6 +347,17 @@ export function lowerCall(lowerer: Lowerer, expr: ts.CallExpression): IrExpr { const json = lowerJsonRequireCall(lowerer, expr); if (json) return json; + // `(0, ns.member)(...)` over a builtin module (tsc's and esbuild's call + // without a receiver): Node's module functions do not read `this`, so it + // is the member call. + { + const member = receiverlessCalleeOf(expr.expression); + if (member !== null && ts.isPropertyAccessExpression(member)) { + const builtin = lowerer.lowerNamespaceBuiltinCall(expr, member); + if (builtin) return builtin; + } + } + // eval(...) and Function(...): Node's EvalError under // --disallow-code-generation-from-strings (code-generation.ts). { diff --git a/packages/compiler/src/frontend/lowering/lower-classes.ts b/packages/compiler/src/frontend/lowering/lower-classes.ts index 29afbb4fe..42c72c653 100644 --- a/packages/compiler/src/frontend/lowering/lower-classes.ts +++ b/packages/compiler/src/frontend/lowering/lower-classes.ts @@ -2738,8 +2738,8 @@ export function collectClassShapeInner( // state, so an override or shadowing field would split-brain. // Underscore methods are the SUPPORTED override form — but only // the ones the class's own base consumes (a `_read` on a - // Transform, or `_writev`/`_construct` anywhere, would be consumed - // by Node machinery that has no lowering here). + // Transform, or `_writev` on a Readable, would be consumed by Node + // machinery that has no lowering here). if (memberName && ts.isIdentifier(memberName)) { const streamBase = (() => { for (let c = base; c; c = c.base) if (c.builtinStream) return c; @@ -2758,7 +2758,7 @@ export function collectClassShapeInner( lowerer.unsupported( "SC1090", memberName, - `declaring '_writev' on a ${streamBase.def.name.slice(1)} subclass (batched writes are supported on Writable and Duplex subclasses)`, + `declaring '_writev' on a ${streamBase.def.name.slice(1)} subclass (a Readable has no writable side)`, ); } const accepted = streamCtorShape(streamBase.def.name).accepted; diff --git a/packages/compiler/src/frontend/lowering/lower-node-island.ts b/packages/compiler/src/frontend/lowering/lower-node-island.ts index 080a6862c..bab5f15a1 100644 --- a/packages/compiler/src/frontend/lowering/lower-node-island.ts +++ b/packages/compiler/src/frontend/lowering/lower-node-island.ts @@ -8,7 +8,11 @@ * Semantics, shared with packages/node/index.js (the Node implementation): * the arguments are copied with JSON.stringify of the argument array; the * module loads with require() from the calling module, so relative - * specifiers resolve from it; the result crosses as its JSON copy and is + * specifiers resolve from it. The specifier is a string literal: the build + * collects the module and what it loads into the executable + * (node-islands/modules.ts), and the call names its caller by its path in + * that tree, never by a path on the build machine. The result crosses as + * its JSON copy and is * checked against the declared result type, like `JSON.parse(s) as T`; a * thrown Error crosses as its nearest intrinsic class with its message and * string `code`. */ @@ -16,7 +20,8 @@ import { realpathSync } from "node:fs"; import * as ts from "../ts7/adapter.js"; import { locOf } from "../program.js"; import type { NodeApiMember } from "../node-api-import.js"; -import { type Lowerer } from "./lowerer.js"; +import { PoisonError, type Lowerer } from "./lowerer.js"; +import { nodeIslandModuleDiag } from "../../diagnostics/diagnostic.js"; import { DYN, STRING, type IrExpr, type IrType } from "../../ir/ir.js"; const JSON_HINT = @@ -89,7 +94,20 @@ export function lowerNodeApiCall( "pass the module specifier, the export name and each argument explicitly", ); } - const specifier = lowerer.lowerExprExpecting(argNodes[0]!, STRING); + const specifierNode = argNodes[0]!; + if (!ts.isStringLiteral(specifierNode) && !ts.isNoSubstitutionTemplateLiteral(specifierNode)) { + lowerer.pushDiag( + nodeIslandModuleDiag(`${surface} with a computed module specifier`, locOf(specifierNode)), + ); + throw new PoisonError(); + } + const caller = callerPath(call); + const problem = lowerer.nodeIslandModules.add(caller, specifierNode.text); + if (problem !== null) { + lowerer.pushDiag(nodeIslandModuleDiag(`${surface}: ${problem}`, locOf(specifierNode))); + throw new PoisonError(); + } + const specifier = lowerer.lowerExprExpecting(specifierNode, STRING); const name = lowerer.lowerExprExpecting(argNodes[1]!, STRING); const values = argNodes.slice(2).map((arg) => { const value = lowerer.lowerExpr(arg); @@ -109,15 +127,18 @@ export function lowerNodeApiCall( lowerer.noLowering(`'${lowerer.fmt(bad)}' results of ${surface}`, call, JSON_HINT); } lowerer.usesNodeIsland = true; + // The caller's tree path is known once every call is collected (Lowerer). + const callerLiteral: IrExpr & { kind: "strLit" } = { + kind: "strLit", + value: caller, + type: STRING, + loc, + }; + lowerer.nodeIslandCallers.push({ literal: callerLiteral, caller }); const raw: IrExpr = { kind: "libCall", fn: member === "call" ? "nodeIsland.call" : "nodeIsland.callSync", - args: [ - { kind: "strLit", value: callerPath(call), type: STRING, loc }, - specifier, - name, - { kind: "dynArrLit", elems: values, type: DYN, loc }, - ], + args: [callerLiteral, specifier, name, { kind: "dynArrLit", elems: values, type: DYN, loc }], type: member === "call" ? { kind: "promise", inner: DYN } : DYN, loc, }; diff --git a/packages/compiler/src/frontend/lowering/lower-stream.ts b/packages/compiler/src/frontend/lowering/lower-stream.ts index c37ae9074..04c2fa043 100644 --- a/packages/compiler/src/frontend/lowering/lower-stream.ts +++ b/packages/compiler/src/frontend/lowering/lower-stream.ts @@ -106,6 +106,61 @@ function pureRead(node: ts.Expression): boolean { return false; } +/** A literal, `undefined`, or a function expression: a value that reads + * no binding or property. */ +function constantOptionValue(node: ts.Expression): boolean { + let e = node; + while (ts.isParenthesizedExpression(e) || ts.isAsExpression(e) || ts.isTypeAssertion(e)) + e = e.expression; + if (ts.isIdentifier(e)) return e.text === "undefined"; + if ( + ts.isPrefixUnaryExpression(e) && + (e.operator === ts.SyntaxKind.MinusToken || + e.operator === ts.SyntaxKind.PlusToken || + e.operator === ts.SyntaxKind.ExclamationToken) + ) + return constantOptionValue(e.operand); + return ( + ts.isStringLiteralLike(e) || + ts.isNumericLiteral(e) || + ts.isBigIntLiteral(e) || + e.kind === ts.SyntaxKind.TrueKeyword || + e.kind === ts.SyntaxKind.FalseKeyword || + e.kind === ts.SyntaxKind.NullKeyword || + ts.isArrowFunction(e) || + ts.isFunctionExpression(e) + ); +} + +/** Whether a runtime decodeStrings or defaultEncoding value (`prop`'s + * `value`) may be read before the other option members. Those values are + * read into locals before the constructor evaluates the other members, + * while Node evaluates the members in order, so every earlier member must + * evaluate without effects, and when the value itself has effects, every + * earlier member must also read nothing those effects could change. */ +function earlierOptionsPure( + options: ts.ObjectLiteralExpression, + prop: ts.Node, + value: ts.Expression, +): boolean { + const valueHasEffects = !sideEffectFreeOptionValue(value) && !pureRead(value); + for (const member of options.properties.nodes) { + if (member === prop) return true; + if (ts.isPropertyAssignment(member)) { + const init = member.initializer; + if ( + valueHasEffects + ? !constantOptionValue(init) + : !sideEffectFreeOptionValue(init) && !pureRead(init) + ) + return false; + } else if (ts.isShorthandPropertyAssignment(member)) { + if (valueHasEffects) return false; + } else if (ts.isSpreadAssignment(member)) return false; + } + return true; +} + /** The stream sides of a receiver class: the nearest stream-class * ancestor's, or null off the stream hierarchy. */ export function streamSidesOf( @@ -179,15 +234,15 @@ export const UNDERSCORE_METHODS: ReadonlyMap = new Map([ /** The option callbacks applied after construction rather than through * the constructor's callback slots (option name → method name): construct - * on every stream class, and writev on the writable classes whose - * machinery hands it batches (Writable and Duplex). */ + * on every stream class, and writev on every class with a writable side + * (a Transform's batches reach writev, not the transform, as in Node). */ export const STREAM_EXTRA_CALLBACKS: ReadonlyMap = new Map([ ["construct", "_construct"], ["writev", "_writev"], ]); export function streamExtraCallbacks(cls: string): readonly string[] { - return cls === "%Writable" || cls === "%Duplex" ? ["construct", "writev"] : ["construct"]; + return cls === "%Readable" ? ["construct"] : ["construct", "writev"]; } /** The completion callback's parameter position in each underscore @@ -442,6 +497,13 @@ interface StreamOptions { * right after construction (Node's constructors call addAbortSignal * once the state exists), or null when absent. */ signal: IrExpr | null; + /** Non-literal decodeStrings and defaultEncoding values: valueDecls + * evaluates them into locals before construction (the defaultEncoding + * validated there, as the state constructors throw before the stream + * exists), and stream.optionValues applies them to the fresh stream. */ + valueDecls: IrStmt[]; + decodeStringsValue: IrExpr | null; + defaultEncodingValue: IrExpr | null; /** The objectMode option's sides (a Duplex's readableObjectMode and * writableObjectMode set one side): applied with stream.objectMode * right after construction. */ @@ -732,6 +794,12 @@ function pinAnyDataCompletion( if (!symbol || lowerer.stageSourceTypes.has(symbol)) return null; const declared = lowerer.paramShape(param).type; if (declared.kind !== "func" || declared.rest || declared.params[1]?.kind !== "dyn") return null; + // Only when every use calls the callback with a Buffer, a string or + // nothing as `data`: another value (or a callback that escapes) keeps + // `data` checked-dynamic, so the runtime pushes it as push(data) does + // and reports ERR_INVALID_ARG_TYPE through 'error', as Node does. + if (fnNode.body === undefined || !completionDataIsChunk(lowerer, fnNode.body, symbol, param.name)) + return null; const data = unionOf(lowerer, [BYTES, STRING, { kind: "undefinedT" }]); lowerer.stageSourceTypes.set( symbol, @@ -740,6 +808,59 @@ function pinAnyDataCompletion( return symbol; } +/** Whether every reference to the completion callback `symbol` in `body` + * calls it with a `data` argument that is a Buffer, a string, null or + * undefined (or with no data). */ +function completionDataIsChunk( + lowerer: Lowerer, + body: ts.Node, + symbol: ts.Symbol, + declaration: ts.Node, +): boolean { + let ok = true; + const visit = (node: ts.Node): void => { + if (!ok) return; + if ( + ts.isIdentifier(node) && + node !== declaration && + lowerer.checker.getSymbolAtLocation(node) === symbol + ) { + const call = node.parent; + if (!call || !ts.isCallExpression(call) || call.expression !== node) { + ok = false; + return; + } + const data = call.arguments.nodes[1]; + if (data !== undefined) { + let t: IrType | null; + try { + t = lowerer.mapTypeOf(lowerer.typeOf(data)); + } catch { + t = null; + } + const arms = + t?.kind === "union" ? (lowerer.unions.get(t.unionId)?.arms ?? null) : t ? [t] : null; + if ( + arms === null || + !arms.every( + (a) => + (a.kind === "bytes" && a.elem === "u8") || + a.kind === "string" || + a.kind === "undefinedT" || + a.kind === "nullT", + ) + ) { + ok = false; + return; + } + } + } + ts.forEachChild(node, visit); + }; + visit(body); + return ok; +} + /** Parses the options object literal of a stream constructor. `which` * names the callbacks this class accepts, in canonical flag order. */ function parseStreamOptions( @@ -767,6 +888,9 @@ function parseStreamOptions( badEncoding: null, pushEncoding: null, signal: null, + valueDecls: [], + decodeStringsValue: null, + defaultEncodingValue: null, readableObject: false, writableObject: false, decodeStringsOff: false, @@ -802,6 +926,21 @@ function parseStreamOptions( optsLit !== undefined && ts.isObjectLiteralExpression(optsLit) && literalWritesValues(optsLit, ctorName !== "Readable" && ctorName !== "Writable"); + // In JavaScript a decodeStrings value known only at run time may let + // string writes through, so the write and transform callbacks' chunk + // parameters read as any (the declarations type it Buffer). + const runtimeDecodeStrings = + optsLit !== undefined && + isJsSourceFile(ts.getSourceFileOfNode(optsLit)) && + ts.isObjectLiteralExpression(optsLit) && + (optsLit.properties?.nodes ?? []).some( + (p) => + ts.isPropertyAssignment(p) && + (ts.isIdentifier(p.name) || ts.isStringLiteral(p.name)) && + p.name.text === "decodeStrings" && + p.initializer.kind !== ts.SyntaxKind.TrueKeyword && + p.initializer.kind !== ts.SyntaxKind.FalseKeyword, + ); if (out.readableObject) out.hwmR = numLit(16, loc); if (out.writableObject) out.hwmW = numLit(16, loc); const tuples = cbTuples(lowerer); @@ -851,6 +990,7 @@ function parseStreamOptions( } const full = tuples[name]!.map((t) => (t === "fn" ? null : t)); const pinned = out.readableObject ? null : pinAnyDataCompletion(lowerer, name, fnNode); + let anyChunk: ts.Symbol | undefined; try { // The completion-callback tail: typed by the checker contextually; // accept the user's declared func type as the tuple's entry so the @@ -867,12 +1007,22 @@ function parseStreamOptions( } return declared ?? funcOf([], VOID); }); + const chunkParam = + runtimeDecodeStrings && (name === "write" || name === "transform") + ? fnNode.parameters.nodes[0] + : undefined; + anyChunk = + chunkParam !== undefined && ts.isIdentifier(chunkParam.name) && !chunkParam.type + ? lowerer.checker.getSymbolAtLocation(chunkParam.name) + : undefined; + if (anyChunk) lowerer.anySymbols.add(anyChunk); present.set( name, lowerStreamCallback(lowerer, name, fnNode, thisType, tuple, writesValues), ); } finally { if (pinned) lowerer.stageSourceTypes.delete(pinned); + if (anyChunk) lowerer.anySymbols.delete(anyChunk); } continue; } @@ -925,11 +1075,25 @@ function parseStreamOptions( if (writableClass) out.decodeStringsOff = true; break; } - lowerer.noLowering( - `${ctorName} with a runtime 'decodeStrings' value`, - prop, - "write 'decodeStrings: true' or 'decodeStrings: false'", - ); + // a runtime value: WritableState's `options.decodeStrings !== false` + if ( + optsLit !== undefined && + ts.isObjectLiteralExpression(optsLit) && + !earlierOptionsPure(optsLit, prop, value) + ) + lowerer.noLowering( + `the ${ctorName} option 'decodeStrings' with a runtime value that an earlier option value could observe or change`, + prop, + "compute the option values into consts before the options object (the runtime value is read before the other options run)", + ); + const local = lowerer.declareHiddenLocal("%decodeStrings", DYN); + out.valueDecls.push({ + kind: "varDecl", + localId: local.id, + init: lowerer.lowerExprExpecting(value, DYN), + loc, + }); + if (writableClass) out.decodeStringsValue = varRef(local.id, DYN, loc); break; } case "encoding": { @@ -973,11 +1137,43 @@ function parseStreamOptions( break; } } - lowerer.noLowering( - `the ${ctorName} option 'defaultEncoding' with a runtime value`, - prop, - "write the encoding as a string literal", - ); + { + // a runtime value: null, undefined and utf8 are the default, + // another name Buffer.isEncoding knows is kept as given, and any + // other value throws ERR_UNKNOWN_ENCODING before construction + if ( + optsLit !== undefined && + ts.isObjectLiteralExpression(optsLit) && + !earlierOptionsPure(optsLit, prop, value) + ) + lowerer.noLowering( + `the ${ctorName} option 'defaultEncoding' with a runtime value that an earlier option value could observe or change`, + prop, + "compute the option values into consts before the options object (the runtime value is read before the other options run)", + ); + const local = lowerer.declareHiddenLocal("%defaultEncoding", DYN); + const ref = varRef(local.id, DYN, loc); + out.valueDecls.push( + { + kind: "varDecl", + localId: local.id, + init: lowerer.lowerExprExpecting(value, DYN), + loc, + }, + { + kind: "exprStmt", + expr: { + kind: "libCall", + fn: "stream.checkEncodingOption", + args: [ref], + type: VOID, + loc, + }, + loc, + }, + ); + out.defaultEncodingValue = ref; + } break; case "signal": { // `if (options.signal) addAbortSignal(options.signal, this)`: @@ -1041,9 +1237,7 @@ function parseStreamOptions( lowerer.noLowering( `the ${ctorName} option 'writev'`, prop, - ctorName === "Readable" - ? "a Readable has no writable side" - : "batched writes on a Transform would bypass its transform — writev is supported on Writable and Duplex", + "a Readable has no writable side", ); break; default: @@ -1083,6 +1277,9 @@ function streamCtorArgs( badEncoding: string | null; pushEncoding: string | null; signal: IrExpr | null; + valueDecls: IrStmt[]; + decodeStringsValue: IrExpr | null; + defaultEncodingValue: IrExpr | null; readableObject: boolean; writableObject: boolean; decodeStringsOff: boolean; @@ -1117,6 +1314,9 @@ function streamCtorArgs( badEncoding: o.badEncoding, pushEncoding: o.pushEncoding, signal: o.signal, + valueDecls: o.valueDecls, + decodeStringsValue: o.decodeStringsValue, + defaultEncodingValue: o.defaultEncodingValue, readableObject: o.readableObject, writableObject: o.writableObject, decodeStringsOff: o.decodeStringsOff, @@ -1237,6 +1437,18 @@ export function lowerStreamNew(lowerer: Lowerer, expr: ts.NewExpression, info: C loc, }; } + const values = optionValuesArgs(parsed, loc); + if (values) { + // runtime decodeStrings and defaultEncoding values (answers the + // receiver +1) + built = { + kind: "libCall", + fn: "stream.optionValues", + args: [built, ...values], + type: thisType, + loc, + }; + } if (parsed.pushEncoding !== null) { // { defaultEncoding }: chain the push-side decode default (the // setEncoding chaining shape — answers the receiver +1). @@ -1269,8 +1481,11 @@ export function lowerStreamNew(lowerer: Lowerer, expr: ts.NewExpression, info: C loc, }; } - if (!parsed.decodeStringsOff && parsed.writeEncoding === null && parsed.extras.length === 0) - return built; + if (!parsed.decodeStringsOff && parsed.writeEncoding === null && parsed.extras.length === 0) { + return parsed.valueDecls.length > 0 + ? { kind: "seqExpr", stmts: parsed.valueDecls, result: built, type: thisType, loc } + : built; + } const fresh = lowerer.declareHiddenLocal("%stream", thisType); const followUps = streamOptionFollowUps( parsed, @@ -1280,13 +1495,30 @@ export function lowerStreamNew(lowerer: Lowerer, expr: ts.NewExpression, info: C ); return { kind: "seqExpr", - stmts: [{ kind: "varDecl", localId: fresh.id, init: built, loc }, ...followUps], + stmts: [ + ...parsed.valueDecls, + { kind: "varDecl", localId: fresh.id, init: built, loc }, + ...followUps, + ], result: varRef(fresh.id, thisType, loc), type: thisType, loc, }; } +/** stream.optionValues' (decodeStrings, defaultEncoding) arguments, or + * null when both options are literals or absent. */ +function optionValuesArgs( + parsed: { decodeStringsValue: IrExpr | null; defaultEncodingValue: IrExpr | null }, + loc: SrcLoc, +): IrExpr[] | null { + if (parsed.decodeStringsValue === null && parsed.defaultEncodingValue === null) return null; + return [ + parsed.decodeStringsValue ?? dynUndefinedExpr(loc), + parsed.defaultEncodingValue ?? dynUndefinedExpr(loc), + ]; +} + /** True when a value of Node's argument type `passed` widens implicitly * into the declared parameter type `declared` (one of its union arms, or a * sub-union of it). */ @@ -1589,6 +1821,7 @@ export function lowerStreamSuperCall( ]; } const out: IrStmt[] = [ + ...parsed.valueDecls, { kind: "exprStmt", expr: { @@ -1614,6 +1847,20 @@ export function lowerStreamSuperCall( loc, }); } + const values = optionValuesArgs(parsed, loc); + if (values) { + out.push({ + kind: "exprStmt", + expr: { + kind: "libCall", + fn: "stream.optionValues", + args: [thisRef(), ...values], + type: thisType, + loc, + }, + loc, + }); + } if (parsed.pushEncoding !== null) { out.push({ kind: "exprStmt", @@ -1697,16 +1944,15 @@ export function lowerStreamUnderscoreAssign( const loc = locOf(expr); const cls = base.def.name; const clsName = cls.slice(1); - if (methodName === "_construct") { - // Node's constructor reads _construct once; a later assignment has no - // effect there - lowerer.noLowering( - `assigning '${methodName}' on a ${clsName}`, - expr.left, - "declare _construct on the class or pass the construct option", - ); - } - const which = methodName === "_writev" ? "writev" : optionByMethod.get(methodName)!; + // `_construct`: Node's constructNT reads it on the construction tick, so + // an assignment before that tick replaces a scheduled hook + // (stream.replaceConstruct) and has no effect after it or without one. + const which = + methodName === "_writev" + ? "writev" + : methodName === "_construct" + ? "construct" + : optionByMethod.get(methodName)!; const { accepted } = streamCtorShape(cls); if (!accepted.includes(which) && !streamExtraCallbacks(cls).includes(which)) { lowerer.noLowering( @@ -1745,6 +1991,7 @@ export function lowerStreamUnderscoreAssign( transform: "stream.setTransform", flush: "stream.setFlush", writev: "stream.setWritev", + construct: "stream.replaceConstruct", }; return { kind: "exprStmt", @@ -2699,6 +2946,22 @@ class StreamArgTypeThrow extends Error { } } +/** Whether a checked-dynamic pipeline callback can take undefined as its + * error: a callback that is already checked-dynamic, one without + * parameters, and one whose first parameter (or rest element) type holds + * undefined. */ +function callbackErrorAdmitsUndefined(lowerer: Lowerer, cb: IrExpr): boolean { + if (cb.kind !== "dynFrom" || cb.value.type.kind !== "func") return true; + const fnT = cb.value.type; + const first = fnT.params[0]; + if (first === undefined) return true; + const p0 = + fnT.rest === true && fnT.params.length === 1 && first.kind === "array" ? first.elem : first; + if (p0.kind === "dyn" || p0.kind === "jsval" || p0.kind === "undefinedT") return true; + if (p0.kind !== "union") return false; + return lowerer.unions.get(p0.unionId)?.arms.some((a) => a.kind === "undefinedT") ?? false; +} + /** The finished/pipeline completion callback: an inline function lowers * through the option-callback machinery (leading `this`, an `Error | * null` prefix — Node binds the stream and passes the error); any other @@ -3160,12 +3423,16 @@ export function lowerStreamModuleCall( // an async generator) reaches the callback through the // checked-dynamic path. const cb = lowerer.lowerExprExpecting(args[args.length - 1]!, DYN); - // SCR_PIPE_TYPED_CB: typed code's callback takes null for no error + // SCR_PIPE_TYPED_CB: a typed callback whose error parameter cannot + // hold undefined takes null for no error; others take undefined. const stagedArgs = [...staged.args]; const flagsArg = stagedArgs[3]!; if (flagsArg.kind !== "numLit") throw new InternalCompilerError("pipeline flags not a literal"); - if (!isJsSourceFile(ts.getSourceFileOfNode(call))) + if ( + !isJsSourceFile(ts.getSourceFileOfNode(call)) && + !callbackErrorAdmitsUndefined(lowerer, cb) + ) stagedArgs[3] = numLit(flagsArg.value | 128, loc); return staged.wrap({ kind: "libCall", diff --git a/packages/compiler/src/frontend/lowering/lowerer.ts b/packages/compiler/src/frontend/lowering/lowerer.ts index 83adae9f8..4b6b0acdf 100644 --- a/packages/compiler/src/frontend/lowering/lowerer.ts +++ b/packages/compiler/src/frontend/lowering/lowerer.ts @@ -496,7 +496,7 @@ import { lowerBuiltinExtraProperty } from "./builtins/properties.js"; import { registerPromisifiedBuiltinDecl } from "./builtins/callable-aliases.js"; import { lowerExecFileAsyncCall } from "./builtins/child-process-async.js"; import { lowerStringDecoderMethodCall, lowerTextCodecCall } from "./builtins/text-codecs.js"; -import { lowerReadlineMethodCall } from "./builtins/readline.js"; +import { isOptionalReadlineCursorRead, lowerReadlineMethodCall } from "./builtins/readline.js"; import { lowerDiagnosticsChannelMethodCall, lowerDiagnosticsChannelProperty, @@ -675,6 +675,7 @@ import { maybePromiseUnionOf, promiseArmedUnionOf, } from "./maybe-promise.js"; +import { NodeIslandModuleCollector } from "../../node-islands/modules.js"; /** Entry function name. '%' cannot appear in a TS identifier, so a user * function can never collide with it (mangling is injective per prefix). */ @@ -3140,6 +3141,10 @@ export class Lowerer { /** The program calls into Node.js (@scriptc/node): the module carries * the nodeIsland record, which the executable build fills in. */ usesNodeIsland = false; + /** The modules Node island calls load (lower-node-island.ts). */ + readonly nodeIslandModules = new NodeIslandModuleCollector(); + /** Each call's caller literal, set to its tree path when lowering ends. */ + readonly nodeIslandCallers: { literal: IrExpr & { kind: "strLit" }; caller: string }[] = []; /** Per-directory answers of experimental-import.ts's declaration-file test. */ readonly experimentalFiles = new Map(); /** Interned LifetimeRegion.run helpers, by callback type key. */ @@ -5558,6 +5563,7 @@ export class Lowerer { if (isIndexedRead(e)) return true; if (ts.isPropertyAccessExpression(e) && isRecordIndexPropertyRead(e)) return true; if (ts.isPropertyAccessExpression(e) && isOptionalProcessStreamProperty(this, e)) return true; + if (ts.isPropertyAccessExpression(e) && isOptionalReadlineCursorRead(this, e)) return true; if (isOptionalProcessVersionsRead(this, e)) return true; if (ts.isPropertyAccessExpression(e) && isSettledIteratorValueRead(this, e)) return true; if (ts.isPropertyAccessExpression(e) && isOsConstantsSymbol(symbolOf(e.expression), this)) @@ -7804,7 +7810,7 @@ export class Lowerer { entry: ENTRY_NAME, ...(this.usesWorkers ? { workers: true } : {}), ...(this.usesLifetimeRegions ? { lifetimeRegions: true as const } : {}), - ...(this.usesNodeIsland ? { nodeIsland: { libnode: "", nodeVersion: "" } } : {}), + ...(this.usesNodeIsland ? { nodeIsland: this.nodeIslandRecord() } : {}), ...(this.declBoundary ? { declBoundary: true as const } : {}), ...(this.threadSlotKeys.size > 0 ? { threadSlotKeys: [...this.threadSlotKeys].sort() } @@ -8643,6 +8649,16 @@ export class Lowerer { this.diags.push(d); } + /** The module's Node island record: the collected module tree, with each + * call's caller literal set to its path in that tree. The executable + * build fills in the libnode pack (index.ts). */ + private nodeIslandRecord(): NonNullable { + const modules = this.nodeIslandModules.tree(); + for (const { literal, caller } of this.nodeIslandCallers) + literal.value = NodeIslandModuleCollector.callerPath(modules, caller); + return { libnode: "", nodeVersion: "", ...(modules === null ? {} : { modules }) }; + } + unsupported( code: keyof typeof UNSUPPORTED & `SC${number}`, node: ts.Node, diff --git a/packages/compiler/src/frontend/lowering/var-hoisting.ts b/packages/compiler/src/frontend/lowering/var-hoisting.ts index 3f950f3ca..0b415bb3d 100644 --- a/packages/compiler/src/frontend/lowering/var-hoisting.ts +++ b/packages/compiler/src/frontend/lowering/var-hoisting.ts @@ -6,9 +6,11 @@ * * A binding settles at the first statement of its hoisting scope (the * function body or the source file) that stores it unconditionally: a - * direct `var x = e`, `x = e;`, or a loop declaring it (`for (var i = 0; - * ...)`, `for (var k in o)`), whose own condition, update and body run - * after the store. A read settles when it runs after that statement: + * direct `var x = e`, `x = e;`, or a `for` loop declaring it (`for (var + * i = 0; ...)`), whose own condition, update and body run after the store. + * A `for (var k in o)` or `for (var v of xs)` loop stores the binding only + * when it iterates, so it settles the reads in its body and no others. A + * read settles when it runs after the settling statement: * - a read in the scope's own code, in a TypeScript file: tsc's * definite-assignment analysis rejects any earlier one; * - a read in the scope's own code, in a JavaScript file: its statement @@ -194,6 +196,18 @@ function referencesBySymbol( return out; } +/** Where a binding is first stored unconditionally: the scope statement's + * index, and for a declarator of a `var` statement or a `for` initializer, + * the list and the declarator's end. */ +interface Settle { + index: number; + list: ts.VariableDeclarationList | null; + end: number; + /** The first for-in/of loop declaring the binding before `index`, whose + * body runs after its store, or -1. */ + loop: number; +} + class ScopeAnalysis { private readonly js: boolean; /** Whether a function declaration's references all settle, by scope @@ -214,10 +228,13 @@ class ScopeAnalysis { } /** The index of the scope statement that first stores `symbol` - * unconditionally, or Infinity. */ - settleIndex(scope: Scope, symbol: ts.Symbol): number { + * unconditionally (Infinity when none does), and, when that statement + * declares it in a `var` statement or a `for` initializer, the end of the + * storing declarator: later declarators of the list run after it. */ + settleIndex(scope: Scope, symbol: ts.Symbol): Settle { const statements = statementsOf(scope); const checker = this.lowerer.checker; + let loopIndex = -1; for (let i = 0; i < statements.length; i++) { const s = statements[i]!; const list = ts.isVariableStatement(s) @@ -233,8 +250,10 @@ class ScopeAnalysis { if ( (d.initializer !== undefined || loop) && boundIdentifiersOf(d.name).some((n) => checker.getSymbolAtLocation(n) === symbol) - ) - return i; + ) { + if (!loop) return { index: i, list, end: d.end, loop: loopIndex }; + if (loopIndex < 0) loopIndex = i; + } } } else if ( ts.isExpressionStatement(s) && @@ -243,20 +262,21 @@ class ScopeAnalysis { ts.isIdentifier(s.expression.left) && checker.getSymbolAtLocation(s.expression.left) === symbol ) { - return i; + return { index: i, list: null, end: -1, loop: loopIndex }; } } - return Infinity; + return { index: Infinity, list: null, end: -1, loop: loopIndex }; } /** Whether `ref` (inside `scope`, or in another module) runs only after - * the statement at index `settle`. `read`: `ref` reads the binding - * itself, so tsc vouches for one in the scope's own code. */ - settles(scope: Scope, settle: number, ref: ts.Identifier, read = true): boolean { + * the store `settle` describes. `read`: `ref` reads the binding itself, + * so tsc vouches for one in the scope's own code. */ + settles(scope: Scope, settle: Settle, ref: ts.Identifier, read = true): boolean { if (ts.getSourceFileOfNode(ref) !== this.sf) return true; const statements = statementsOf(scope); let deferred: ts.Node | null = null; let top: ts.Node | null = null; + const container: ts.Node | undefined = ts.isSourceFile(scope) ? scope : scope.body; for (let n: ts.Node = ref; n.parent !== undefined && n !== scope; n = n.parent) { if ( ts.isFunctionLike(n) || @@ -265,18 +285,26 @@ class ScopeAnalysis { ts.isClassStaticBlockDeclaration(n) ) deferred = n; - if (n.parent === scope || (!ts.isSourceFile(scope) && n.parent === scope.body)) top = n; + // The scope's own statement, not the function body that holds it. + if (n.parent === container) top = n; } if (top === null) return false; if (deferred === null && read && !this.js) return true; if (deferred !== null && ts.isFunctionDeclaration(deferred) && deferred.name !== undefined) return this.functionSettles(scope, settle, deferred); const index = statements.indexOf(top as ts.Statement); - return index > settle || (index === settle && settlesWithin(top, ref)); + return ( + index > settle.index || + (index === settle.index && + (settlesWithin(top, ref) || + // A later declarator of the storing list (`var a = {}, b = { ...a }`). + (settle.list !== null && isInside(ref, settle.list) && ref.pos >= settle.end))) || + (index === settle.loop && settlesWithin(top, ref)) + ); } - private functionSettles(scope: Scope, settle: number, fn: ts.FunctionDeclaration): boolean { - const key = `${fn.pos}:${settle}`; + private functionSettles(scope: Scope, settle: Settle, fn: ts.FunctionDeclaration): boolean { + const key = `${fn.pos}:${settle.index}:${settle.end}:${settle.loop}`; const known = this.functionMemo.get(key); if (known !== undefined) return known; this.functionMemo.set(key, true); diff --git a/packages/compiler/src/frontend/npm-static-bundled-chunks.test.ts b/packages/compiler/src/frontend/npm-static-bundled-chunks.test.ts index 0779db3ce..dc238c622 100644 --- a/packages/compiler/src/frontend/npm-static-bundled-chunks.test.ts +++ b/packages/compiler/src/frontend/npm-static-bundled-chunks.test.ts @@ -276,13 +276,14 @@ export default require_f(); ).toBeNull(); }); -test("an identifier-minified bundle names its lost helper names", () => { - // esbuild --minify output: the helper shapes survive, their names do not. - const result = split( +test("an identifier-minified bundle splits through its helper's shape", () => { + // esbuild --minify output: the helper shape survives, its name does not. + const answer = expectSameBehavior( `var c=(r,e)=>()=>(e||r((e={exports:{}}).exports,e),e.exports);` + - `var x=c((t,m)=>{m.exports={v:1}});console.log(x().v);`, + `var x=c((t,m)=>{m.exports={v:1}});var y=c(t=>{t.w=x().v+1});` + + `console.log(x().v,y().w,x()===x());`, ); - expect(result).toEqual({ degrade: expect.stringContaining("identifier-minified") as string }); + expect(answer.chunks.map((c) => basename(c.path))).toEqual(["index.js.x.cjs", "index.js.y.cjs"]); }); test("an ES-module bundle of CommonJS chunks names the chunk interop", () => { diff --git a/packages/compiler/src/frontend/npm-static-bundled-chunks.ts b/packages/compiler/src/frontend/npm-static-bundled-chunks.ts index 37a74f2be..66c25403b 100644 --- a/packages/compiler/src/frontend/npm-static-bundled-chunks.ts +++ b/packages/compiler/src/frontend/npm-static-bundled-chunks.ts @@ -525,7 +525,7 @@ export function splitBundledChunks( } } } - const helperNames = [...top.keys()].filter((n) => helperBase(n) === "__commonJS"); + let helperNames = [...top.keys()].filter((n) => helperBase(n) === "__commonJS"); // Every name a recognizer relies on is written nowhere and declared // once at the top level; `Object` itself is never rebound. @@ -552,18 +552,14 @@ export function splitBundledChunks( ); }; if (helperNames.length === 0) { - // The helper's shape under another name: an identifier-minified bundle, - // whose other interop helpers lost their names as well. - for (const [name, t] of top) { + // An identifier-minified bundle: the helper's shape under another name. + helperNames = [...top].flatMap(([name, t]) => { const init = ts.isVariableDeclaration(t.node) ? t.node.initializer : undefined; - if (init !== undefined && stable(name) && helperForm(init, sf, ownNames) !== null) - return { - degrade: - "its shipped JS is an identifier-minified bundle whose bundler-interop helpers " + - "(__commonJS and the rest) have lost the names the recognizers verify", - }; - } - return null; + return init !== undefined && stable(name) && helperForm(init, sf, ownNames) !== null + ? [name] + : []; + }); + if (helperNames.length === 0) return null; } const forms = new Map(); for (const name of helperNames) { diff --git a/packages/compiler/src/frontend/npm-static-rewrite-syntax.ts b/packages/compiler/src/frontend/npm-static-rewrite-syntax.ts index c48420625..917041603 100644 --- a/packages/compiler/src/frontend/npm-static-rewrite-syntax.ts +++ b/packages/compiler/src/frontend/npm-static-rewrite-syntax.ts @@ -260,11 +260,21 @@ function definePropertyExportOf( value = getterReturnOf(p); } } - const esModuleStamp = - name === "__esModule" && value !== null && value.kind === ts.SyntaxKind.TrueKeyword; + const esModuleStamp = name === "__esModule" && value !== null && isTrueLiteral(value); return { name, value, esModuleStamp }; } +/** `true`, or `!0`, its minified spelling. */ +function isTrueLiteral(expr: ts.Expression): boolean { + return ( + expr.kind === ts.SyntaxKind.TrueKeyword || + (ts.isPrefixUnaryExpression(expr) && + expr.operator === ts.SyntaxKind.ExclamationToken && + ts.isNumericLiteral(expr.operand) && + expr.operand.text === "0") + ); +} + /** Space-pads [start, end) of `text` preserving newlines — the recognized * statement disappears while every original offset survives. */ function padSpan(text: string, start: number, end: number): string { diff --git a/packages/compiler/src/frontend/npm-static.ts b/packages/compiler/src/frontend/npm-static.ts index 800d02eff..eed02c95c 100644 --- a/packages/compiler/src/frontend/npm-static.ts +++ b/packages/compiler/src/frontend/npm-static.ts @@ -614,27 +614,12 @@ export function clearProgramBuiltinAliasCache(): void { * but the bundler-emitted CJS export rewrite (npm-static-rewrite.ts) * types the getter-table shapes and the offender attribution degrades a * package whose surface still breaks its consumers — so those bundles are - * worth ATTEMPTING: eligible when the .d.ts and unminified criteria hold, - * gracefully per-package-degraded when the attempt fails. */ + * worth ATTEMPTING: eligible when the .d.ts criteria hold, gracefully + * per-package-degraded when the attempt fails. Minified JS is attempted + * too: a minified dist is ordinary JavaScript, and the bundle recognizers + * know esbuild's __commonJS helper by its shape under any name. */ const TRANSFORM_MARKERS = ["__webpack_require__"]; -/** Unminified-JS heuristic over an entry source: at least two lines and - * an average line length under 200 characters (minified dists are one - * enormous line; readable code averages well under 100). */ -function looksUnminified(source: string): boolean { - const lines = source.split("\n"); - if (lines.length < 2) return false; - return source.length / lines.length <= 200; -} - -/** A whitespace/syntax-minified bundler dist whose interop helpers keep - * their names (`var __toESM=…,__toCommonJS=mod=>…`): the recognizers - * verify those shapes and refuse the package precisely when they cannot, - * so the minification heuristic need not refuse it up front. */ -function namedBundlerHelpers(source: string): boolean { - return /\b__(?:toCommonJS|commonJS|toESM)\d*\s*=\s*(?:\(|[A-Za-z_$][\w$]*\s*=>)/.test(source); -} - function hasTransformMarkers(source: string): boolean { return TRANSFORM_MARKERS.some((m) => source.includes(m)); } @@ -676,8 +661,6 @@ export function npmStaticIneligibleReason( if (jsEntry === null) return "no runtime JS entry resolves"; const source = trackedReadFile(jsEntry); if (source === null) return `its runtime entry ${jsEntry} cannot be read`; - if (!looksUnminified(source) && !namedBundlerHelpers(source)) - return "its shipped JS looks minified"; if (hasTransformMarkers(source)) { return "its shipped JS carries build-transform markers (bundled/transpiled dist)"; } diff --git a/packages/compiler/src/frontend/program.ts b/packages/compiler/src/frontend/program.ts index 68b649b07..932ac7054 100644 --- a/packages/compiler/src/frontend/program.ts +++ b/packages/compiler/src/frontend/program.ts @@ -1436,8 +1436,20 @@ function requiresOf7( return out; } +/** `true`, or `!0`, its minified spelling. */ +function isTrueLiteral(expr: ts.Expression): boolean { + return ( + expr.kind === ts.SyntaxKind.TrueKeyword || + (ts.isPrefixUnaryExpression(expr) && + expr.operator === ts.SyntaxKind.ExclamationToken && + ts.isNumericLiteral(expr.operand) && + expr.operand.text === "0") + ); +} + /** The exact `Object.defineProperty(exports|module.exports, "__esModule", - * { value: true })` interop stamp (shared by preflight and the no-op lowering). */ + * { value: true })` interop stamp (shared by preflight and the no-op lowering); + * `{ value: !0 }` is the same stamp. */ export function isEsModuleStamp(expr: ts.Expression): boolean { if (!ts.isCallExpression(expr) || expr.questionDotToken !== undefined) return false; const callee = expr.expression; @@ -1471,7 +1483,7 @@ export function isEsModuleStamp(expr: ts.Expression): boolean { ts.isPropertyAssignment(p) && ts.isIdentifier(p.name) && p.name.text === "value" && - p.initializer.kind === ts.SyntaxKind.TrueKeyword + isTrueLiteral(p.initializer) ); } diff --git a/packages/compiler/src/index.ts b/packages/compiler/src/index.ts index ca84bc18f..bbdab43bc 100644 --- a/packages/compiler/src/index.ts +++ b/packages/compiler/src/index.ts @@ -111,9 +111,11 @@ import { iceDiag, nativeCodegenDiag, nodeIslandNodeDiag, + nodeIslandModuleDiag, type ScrDiagnostic, } from "./diagnostics/diagnostic.js"; import { libnodeCacheKey, libnodeRecord } from "./node-islands/libnode.js"; +import { placeNodeIslandModules, readNodeIslandModules } from "./node-islands/modules.js"; import { loadLibraryProfile, type LibraryProfile } from "./library/library-profile.js"; import { clearFenceEvalCaches, decorateLibraryRefusals } from "./library/fence-eval.js"; import { @@ -785,7 +787,32 @@ async function prepareExecutableInput( const recorded = libnodeRecord(opts.libnode, opts.outPath, process.env, dirname(entryPath)); if (typeof recorded === "string") return { ok: false, diagnostics: [nodeIslandNodeDiag(recorded, entryPath)], sourceTexts }; - mod.nodeIsland = recorded; + mod.nodeIsland = { ...mod.nodeIsland, ...recorded }; + } + } + // The island module tree ships with the program: beside the executable + // with --libnode=beside, else inside it (node-islands/modules.ts). + let nodeIslandModuleData: Uint8Array | undefined; + const islandModules = mod.nodeIsland?.modules; + if (islandModules !== undefined && outputKind !== "ir") { + if (outputKind === "exe" && opts.libnode === "beside") { + const problem = placeNodeIslandModules(islandModules, `${opts.outPath}.libnode/modules`); + if (problem !== null) + return { + ok: false, + diagnostics: [nodeIslandModuleDiag(problem, { file: entryPath, start: 0, end: 0 })], + sourceTexts, + }; + islandModules.location = `$ORIGIN/${basename(opts.outPath)}.libnode/modules`; + } else { + const data = readNodeIslandModules(islandModules); + if (typeof data === "string") + return { + ok: false, + diagnostics: [nodeIslandModuleDiag(data, { file: entryPath, start: 0, end: 0 })], + sourceTexts, + }; + nodeIslandModuleData = data; } } if (warningSink !== undefined && prepared.warnings.length > 0) { @@ -820,6 +847,7 @@ async function prepareExecutableInput( lowRegion: lowRegionTarget(buildPlatform, opts.sanitize === true), ...(opts.sanitize === true ? { wasmBoundsChecks: true } : {}), ...inlineRcOption(opts.optimization, opts.sanitize === true), + ...(nodeIslandModuleData === undefined ? {} : { nodeIslandModuleData }), }); } catch (err) { if (!(err instanceof LlvmUnsupportedError)) throw err; @@ -890,6 +918,7 @@ async function prepareExecutableInput( ...(opts.sanitize === true ? { wasmBoundsChecks: true } : {}), ...inlineRcOption(opts.optimization, opts.sanitize === true), ...(opts.detachedExit === true ? { detachedExit: true } : {}), + ...(nodeIslandModuleData === undefined ? {} : { nodeIslandModuleData }), }); } catch (err) { if (!(err instanceof LlvmUnsupportedError)) throw err; @@ -1469,8 +1498,6 @@ async function compileTracked( if (outputKind === "exe") { const helperObjectRoute = opts.nativeProgramObject === true || usesPrecompiledRuntimePack(opts, "llvm"); - // Calls into Node record the libnode pack, keyed by its manifest (node-islands/libnode.ts). - const libnodeKey = libnodeCacheKey(opts.libnode); const routeOptions: Omit< EarlyExecutableCacheOptions, "implementation" | "implementationDependencies" | "nativeEnvironment" @@ -1485,7 +1512,9 @@ async function compileTracked( ...(opts.strip ? { strip: true as const } : {}), ...(opts.windowsSubsystem === "gui" ? { windowsSubsystem: "gui" as const } : {}), ...(opts.detachedExit === true ? { detachedExit: true as const } : {}), - ...(libnodeKey === undefined ? {} : { libnode: libnodeKey }), + // Calls into Node record the libnode pack; their entries record its + // identity (node-islands/libnode.ts) and a hit requires it unchanged. + libnodeIdentity: () => libnodeCacheKey(opts.libnode, process.env, dirname(entryPath)), ...(profile === undefined ? {} : { profile: nativeProfileKey(profile) }), ...checkerField(), ...(youngNurseryRequested() ? { youngNursery: true as const } : {}), @@ -1531,26 +1560,22 @@ async function compileTracked( implementationDependencies: implementation.dependencies, }; } - const earlyHit = + const cachedHit = earlyCacheOptions === null ? null : await readEarlyExecutableCache(cacheRoot, earlyCacheOptions); + // A --libnode=beside build that calls into Node places its island modules + // next to the executable from the lowered module, which a cache hit does + // not have: such builds take the full path. + const earlyHit = + cachedHit !== null && cachedHit.native.nodeIsland === true && opts.libnode === "beside" + ? null + : cachedHit; if (earlyHit !== null) { timing("executable-cache-hit"); if (earlyCacheOptions === null) { throw new InternalCompilerError("executable cache hit without executable cache options"); } - // The pack a --libnode=beside build places next to the executable is an output of the - // build, like the executable: a restored executable that calls into Node gets it again. - if (earlyHit.native.nodeIsland === true && opts.libnode === "beside") { - const recorded = libnodeRecord(opts.libnode, opts.outPath); - if (typeof recorded === "string") - return { - ok: false, - diagnostics: [nodeIslandNodeDiag(recorded, entryPath)], - sourceTexts: new Map(), - }; - } // A cached build repeats its performance warning (SC6101): the code it // names is just as untyped as when it was built. if (warningSink !== undefined && earlyHit.performance !== undefined) { diff --git a/packages/compiler/src/ir/builtin-effects.ts b/packages/compiler/src/ir/builtin-effects.ts index 76431450d..6ac3c9068 100644 --- a/packages/compiler/src/ir/builtin-effects.ts +++ b/packages/compiler/src/ir/builtin-effects.ts @@ -366,6 +366,7 @@ export const MAY_THROW_LIB_FNS: ReadonlySet = new Set([ "transform.newDyn", "passthrough.newDyn", "readable.initDyn", + "stream.checkEncodingOption", "writable.initDyn", "duplex.initDyn", "transform.initDyn", diff --git a/packages/compiler/src/ir/builtin-signatures.ts b/packages/compiler/src/ir/builtin-signatures.ts index 06a9c73ea..afb06b737 100644 --- a/packages/compiler/src/ir/builtin-signatures.ts +++ b/packages/compiler/src/ir/builtin-signatures.ts @@ -4004,6 +4004,7 @@ export const LIB_FN_SIGS = defineLibFnSignatures({ "stream.setTransform": { argTypes: [null, null], result: VOID }, "stream.setFlush": { argTypes: [null, null], result: VOID }, "stream.setConstruct": { argTypes: [null, null], result: VOID }, + "stream.replaceConstruct": { argTypes: [null, null], result: VOID }, "stream.setWritev": { argTypes: [null, null], result: VOID }, "stream.setWritevDefault": { argTypes: [null, null], result: VOID }, /** The dyn-options twins (a checked-dynamic options record — the JS @@ -4140,6 +4141,11 @@ export const LIB_FN_SIGS = defineLibFnSignatures({ * after construction; answers the receiver (+1, the chaining shape). */ "stream.objectMode": { argTypes: [null, BOOL, BOOL], result: VOID }, "stream.decodeStrings": { argTypes: [null, BOOL], result: VOID }, + /** Runtime decodeStrings and defaultEncoding option values, applied + * after construction (answers the receiver +1); the encoding was + * validated by stream.checkEncodingOption before construction. */ + "stream.optionValues": { argTypes: [null, DYN, DYN], result: VOID }, + "stream.checkEncodingOption": { argTypes: [DYN], result: VOID }, "stream.duplexFromGen": { argTypes: [null, null, null, null, null, null], result: VOID }, "stream.duplexFromDyn": { argTypes: [DYN], result: VOID }, /** Duplex.from over a lone Readable or Writable, or a { readable, diff --git a/packages/compiler/src/ir/function-references.test.ts b/packages/compiler/src/ir/function-references.test.ts index 0e09a7567..9541cb723 100644 --- a/packages/compiler/src/ir/function-references.test.ts +++ b/packages/compiler/src/ir/function-references.test.ts @@ -221,6 +221,12 @@ const MODULE_FIELDS: Record>, Role> = { "threadSlotKeys[]": "other", "nodeIsland.libnode": "other", "nodeIsland.nodeVersion": "other", + "nodeIsland.modules.root": "other", + "nodeIsland.modules.location": "other", + "nodeIsland.modules.files[].path": "other", + "nodeIsland.modules.files[].sha256": "other", + "nodeIsland.modules.links[].path": "other", + "nodeIsland.modules.links[].target": "other", }; const loc = { file: "references.ts", start: 0, end: 0 }; diff --git a/packages/compiler/src/ir/ir.ts b/packages/compiler/src/ir/ir.ts index 667e6de44..e458c9045 100644 --- a/packages/compiler/src/ir/ir.ts +++ b/packages/compiler/src/ir/ir.ts @@ -1009,6 +1009,18 @@ export function isRefCounted(t: IrType): boolean { /* ── module ────────────────────────────────────────────────────────────── */ +/** The island module tree of a program (node-islands/modules.ts): paths + * relative to `root`, the directory on the build machine that holds them. + * Only the relative paths enter an executable. */ +export interface IrNodeIslandModules { + root: string; + files: { path: string; size: number; sha256: string }[]; + links: { path: string; target: string }[]; + /** Where the executable finds the tree: absent when it carries the tree + * itself, else a `$ORIGIN/` path (`--libnode=beside`). */ + location?: string; +} + export interface IrModule { /** Bumped on any breaking IR change; serialize.ts refuses mismatches. */ irVersion: 17; @@ -1067,7 +1079,13 @@ export interface IrModule { * executable) and the exact Node version its bridge links. Set by the * executable build (node-islands/libnode.ts); the emitted main records * both before %main. */ - nodeIsland?: { libnode: string; nodeVersion: string }; + nodeIsland?: { + libnode: string; + nodeVersion: string; + /** The modules the calls load (node-islands/modules.ts), when any call + * loads a file. */ + modules?: IrNodeIslandModules; + }; /** Outbound native FFI declarations used by `ffiCall` expressions. * These are link-time C ABI imports, not runtime dynamic-library handles: * executable builds resolve their symbols from the manifest's archive @@ -3904,8 +3922,10 @@ export type IrExpr = * not shape equality; missing or wrong-typed fields throw, EXCEPT that a * missing key for an undefined-armed union field — an optional field — * builds the interned undefined arm instead); arrays check every element; - * unions try arms in canonical order and the first FULL match wins (no - * match → throw; an undefined arm matches no dyn value); JSON null matches + * unions try arms in canonical order, except that record arms with more + * required fields and then more fields go first, and the first FULL match + * wins (no match → throw; an undefined arm matches no dyn value); JSON + * null matches * exactly the nullT arm of a union target (bare null targets cannot * exist). MAY THROW: * backends' may-throw analyses must treat it like a `throw` statement. diff --git a/packages/compiler/src/ir/runtime-features.ts b/packages/compiler/src/ir/runtime-features.ts index 244d8ef22..9c4c493be 100644 --- a/packages/compiler/src/ir/runtime-features.ts +++ b/packages/compiler/src/ir/runtime-features.ts @@ -220,6 +220,8 @@ const STREAM_LIB_FNS: ReadonlySet = new Set([ "stream.chunkDone", "stream.objectMode", "stream.decodeStrings", + "stream.optionValues", + "stream.checkEncodingOption", "stream.duplexFromGen", "stream.duplexFromDyn", "stream.duplexFromPromise", diff --git a/packages/compiler/src/ir/validate.ts b/packages/compiler/src/ir/validate.ts index 02825f0de..a404c918a 100644 --- a/packages/compiler/src/ir/validate.ts +++ b/packages/compiler/src/ir/validate.ts @@ -5500,6 +5500,7 @@ function validateFunction( e.fn === "stream.destroyErr" || e.fn === "stream.objectMode" || e.fn === "stream.decodeStrings" || + e.fn === "stream.optionValues" || e.fn === "stream.duplexFromGen" || e.fn === "stream.duplexFromDyn" || e.fn === "stream.duplexifyReadable" || @@ -5510,6 +5511,7 @@ function validateFunction( e.fn === "stream.duplexFromGenSignal" || e.fn === "stream.duplexFromAsyncFnSignal" || e.fn === "stream.setConstruct" || + e.fn === "stream.replaceConstruct" || e.fn === "stream.setWritev" || e.fn === "stream.setWritevDefault" || e.fn === "stream.iteratorClose" || @@ -5872,6 +5874,7 @@ function validateFunction( e.fn === "readable.pushEncoding" || e.fn === "stream.objectMode" || e.fn === "stream.decodeStrings" || + e.fn === "stream.optionValues" || e.fn === "writable.writeEncoding" || e.fn === "writable.setDefaultEncoding" || e.fn === "stream.destroy" || diff --git a/packages/compiler/src/node-islands/libnode-pack.test.ts b/packages/compiler/src/node-islands/libnode-pack.test.ts index 8ba25b30a..3508733f7 100644 --- a/packages/compiler/src/node-islands/libnode-pack.test.ts +++ b/packages/compiler/src/node-islands/libnode-pack.test.ts @@ -4,6 +4,7 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { fileURLToPath } from "node:url"; import { afterEach, describe, expect, test } from "vitest"; +import { compilerReleaseVersion } from "../library/sidecar.js"; import { NODE_BRIDGE_ABI, libnodeRecord } from "./libnode.js"; import { LIBNODE_PACK_SCHEMA, @@ -48,7 +49,10 @@ function fakePack( writeFileSync(join(dir, "libnode.so.137"), libnode); writeFileSync(join(dir, "libscr_node_bridge.so"), bridge); writeFileSync(join(dir, "LICENSE"), "Node.js is licensed for use as follows:\n\nMIT\n"); - writeFileSync(join(dir, "package.json"), JSON.stringify({ name, version: "0.4.1" })); + writeFileSync( + join(dir, "package.json"), + JSON.stringify({ name, version: compilerReleaseVersion() }), + ); const manifest: Record = { schema: LIBNODE_PACK_SCHEMA, nodeVersion: "24.22.0", @@ -59,7 +63,7 @@ function fakePack( libnode: "libnode.so.137", bridge: "libscr_node_bridge.so", package: name, - version: "0.4.1", + version: compilerReleaseVersion(), target: "linux-x64-gnu", license: "LICENSE", files: { "libnode.so.137": entry(libnode), "libscr_node_bridge.so": entry(bridge) }, @@ -180,6 +184,22 @@ describe("finding the installed pack", () => { }); }); +test("an installed pack of another scriptc version is refused", () => { + const root = temp(); + const name = libnodePackPackage(hostLibnodeTarget()); + if (name === null) return; + const pack = fakePack( + join(root, "node_modules", name), + (m) => { + m["version"] = "0.0.1"; + }, + name, + ); + expect(libnodeRecord(undefined, join(root, "app"), {}, root)).toBe( + `the installed libnode pack ${name} at ${pack} is version 0.0.1; this scriptc is ${compilerReleaseVersion()}: install ${name}@${compilerReleaseVersion()}`, + ); +}); + describe("readLibnodePack", () => { test("reads a pack in build-pack.mjs's format and checks its package", () => { const pack = fakePack(join(temp(), "pack")); @@ -233,7 +253,7 @@ test("the license notices carry Node's LICENSE", () => { expect(path).toBe(`${out}.licenses.txt`); expect(readFileSync(path, "utf8")).toBe( [ - "server runs part of its code in Node.js 24.22.0, loaded from libnode.so.137 (@scriptc/libnode-linux-x64-gnu@0.4.1).", + `server runs part of its code in Node.js 24.22.0, loaded from libnode.so.137 (@scriptc/libnode-linux-x64-gnu@${compilerReleaseVersion()}).`, "Node.js and the components it bundles are licensed as follows. The scriptc bridge,", "libscr_node_bridge.so, is licensed under Apache-2.0.", "", diff --git a/packages/compiler/src/node-islands/libnode.test.ts b/packages/compiler/src/node-islands/libnode.test.ts index d6ca4bf83..fd775be5d 100644 --- a/packages/compiler/src/node-islands/libnode.test.ts +++ b/packages/compiler/src/node-islands/libnode.test.ts @@ -3,7 +3,8 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { describe, expect, test } from "vitest"; import { USES_TIMERS_LIB_FNS } from "../backend/llvm/lib-shared.js"; -import { libnodeCacheKey, libnodeRecord, NODE_BRIDGE_ABI } from "./libnode.js"; +import { libnodeCacheKey, libnodeRecord, libnodeTarget, NODE_BRIDGE_ABI } from "./libnode.js"; +import { libnodePackPackage } from "./libnode-pack.js"; import { findInstalledLibnodePack, hostLibnodeTarget } from "./libnode-pack.js"; function pack(nodeVersion: string, bridgeAbi = NODE_BRIDGE_ABI): string { @@ -84,6 +85,31 @@ describe("the recorded libnode pack", () => { expect(readFileSync(join(out, "app.libnode/libscr_node_bridge.so"), "utf8")).toBe("bridge"); }); + test("beside refuses to replace the pack it copies", () => { + const out = mkdtempSync(join(tmpdir(), "scriptc-libnode-out-")); + const first = pack("24.22.0"); + libnodeRecord("beside", join(out, "app"), { SCRIPTC_LIBNODE: first }); + // The placed copy as the source of the next beside build. + const placed = join(out, "app.libnode"); + expect(libnodeRecord("beside", join(out, "app"), { SCRIPTC_LIBNODE: placed })).toMatch( + /^--libnode=beside would replace the libnode pack it copies/, + ); + expect(readFileSync(join(placed, "libscr_node_bridge.so"), "utf8")).toBe("bridge"); + }); + + test("beside refuses manifest file names outside the pack", () => { + const dir = pack("24.22.0"); + writeFileSync( + join(dir, "manifest.json"), + JSON.stringify({ nodeVersion: "24.22.0", bridgeAbi: NODE_BRIDGE_ABI, files: { "../x": {} } }), + ); + const out = mkdtempSync(join(tmpdir(), "scriptc-libnode-out-")); + expect(libnodeRecord("beside", join(out, "app"), { SCRIPTC_LIBNODE: dir })).toBe( + `the libnode pack at ${dir} lists a file outside the pack ('../x')`, + ); + expect(existsSync(join(out, "app.libnode"))).toBe(false); + }); + test("a pack the deployment places beside the executable needs no manifest at build time", () => { expect(libnodeRecord("$ORIGIN/node", "/out/app", {})).toEqual({ libnode: "$ORIGIN/node", @@ -92,9 +118,24 @@ describe("the recorded libnode pack", () => { }); }); -describe("the libnode pack in an executable's cache key", () => { - test("is absent without --libnode or SCRIPTC_LIBNODE", () => { - expect(libnodeCacheKey(undefined, {})).toBeUndefined(); +describe("the libnode pack identity a cached build keeps", () => { + test("is absent when no pack is named or installed", () => { + expect(libnodeCacheKey(undefined, {}, "/nonexistent")).toBeUndefined(); + }); + + test("names the installed pack, and changes with it", () => { + const project = mkdtempSync(join(tmpdir(), "scriptc-libnode-project-")); + const name = libnodePackPackage(libnodeTarget({})); + if (name === null) return; + const installed = join(project, "node_modules", name); + mkdirSync(installed, { recursive: true }); + writeFileSync(join(installed, "manifest.json"), JSON.stringify({ nodeVersion: "24.22.0" })); + const before = libnodeCacheKey(undefined, {}, project); + expect(before).toContain(installed); + writeFileSync(join(installed, "manifest.json"), JSON.stringify({ nodeVersion: "24.23.0" })); + expect(libnodeCacheKey(undefined, {}, project)).not.toBe(before); + rmSync(installed, { recursive: true, force: true }); + expect(libnodeCacheKey(undefined, {}, project)).toBeUndefined(); }); test("changes when the pack in the same directory is upgraded", () => { diff --git a/packages/compiler/src/node-islands/libnode.ts b/packages/compiler/src/node-islands/libnode.ts index cb4e7e21c..e52339b30 100644 --- a/packages/compiler/src/node-islands/libnode.ts +++ b/packages/compiler/src/node-islands/libnode.ts @@ -13,9 +13,26 @@ * executable and its Node travel together; a recorded `$ORIGIN/` path is * also the hook for bundling Node into the executable itself later. */ import { createHash } from "node:crypto"; -import { copyFileSync, linkSync, mkdirSync, readFileSync, rmSync, statSync } from "node:fs"; -import { basename, isAbsolute, join, resolve } from "node:path"; -import { findInstalledLibnodePack, hostLibnodeTarget, libnodePackPackage } from "./libnode-pack.js"; +import { + copyFileSync, + linkSync, + mkdirSync, + readFileSync, + realpathSync, + rmSync, + statSync, +} from "node:fs"; +import { basename, isAbsolute, join, resolve, sep } from "node:path"; +import { compilerReleaseVersion } from "../library/sidecar.js"; +import { nativeCodegenTarget } from "../backend/targets.js"; +import { + findInstalledLibnodePack, + hostLibnodeTarget, + LibnodePackError, + libnodePackPackage, + readLibnodePack, + type LibnodePack, +} from "./libnode-pack.js"; /** The oldest Node.js release Node islands support: synchronous * module.registerHooks, which shared modules need, is a release candidate @@ -23,7 +40,7 @@ import { findInstalledLibnodePack, hostLibnodeTarget, libnodePackPackage } from export const NODE_ISLAND_MIN_NODE = "24.13.1"; /** The bridge ABI this compiler's runtime speaks (scr_node_bridge.h). */ -export const NODE_BRIDGE_ABI = 2; +export const NODE_BRIDGE_ABI = 3; /** The prefix of a recorded path relative to the executable's directory. */ export const ORIGIN = "$ORIGIN/"; @@ -68,23 +85,66 @@ function readManifest(dir: string): PackManifest | string { }; } -/** Copies the pack next to the executable (hard links where possible). */ -function placeBeside(source: string, manifest: PackManifest, target: string): void { +/** Copies the pack next to the executable (hard links where possible), or + * returns why it cannot: the target would replace the pack itself, or the + * manifest names a file outside the pack. */ +function placeBeside(source: string, manifest: PackManifest, target: string): string | undefined { + const names = ["manifest.json", "LICENSE", ...Object.keys(manifest.files)]; + const unsafe = names.find( + (name) => name === "" || name === "." || name === ".." || /[/\\]/.test(name), + ); + if (unsafe !== undefined) + return `the libnode pack at ${source} lists a file outside the pack ('${unsafe}')`; + const from = realpathOr(source); + const to = realpathOr(target); + if (from === to || from.startsWith(to + sep) || to.startsWith(from + sep)) + return `--libnode=beside would replace the libnode pack it copies (${source}); point SCRIPTC_LIBNODE at a pack outside ${target}`; rmSync(target, { recursive: true, force: true }); mkdirSync(target, { recursive: true }); - for (const name of ["manifest.json", "LICENSE", ...Object.keys(manifest.files)]) { - const from = join(source, name); + for (const name of names) { + const file = join(source, name); try { - statSync(from); + statSync(file); } catch { continue; } try { - linkSync(from, join(target, name)); + linkSync(file, join(target, name)); } catch { - copyFileSync(from, join(target, name)); + copyFileSync(file, join(target, name)); } } + return undefined; +} + +/** The canonical path of `path`, or `path` resolved when it does not exist. */ +function realpathOr(path: string): string { + try { + return realpathSync(path); + } catch { + return resolve(path); + } +} + +/** The native target whose libnode pack a build records: the codegen + * target (SCRIPTC_TARGET, and the host's libc), else the host. */ +export function libnodeTarget(env: NodeJS.ProcessEnv = process.env): string { + return nativeCodegenTarget(env)?.name ?? hostLibnodeTarget(); +} + +/** Why the installed pack in `dir` cannot serve this compiler, or null: it + * must be the target's package at the compiler's own version. */ +function installedPackProblem(dir: string, packageName: string): string | null { + let pack: LibnodePack; + try { + pack = readLibnodePack(dir, { packageName }); + } catch (error) { + return error instanceof LibnodePackError ? error.message : String(error); + } + const version = compilerReleaseVersion(); + if (pack.manifest.version !== version) + return `the installed libnode pack ${packageName} at ${dir} is version ${pack.manifest.version ?? "unknown"}; this scriptc is ${version}: install ${packageName}@${version}`; + return null; } /** What to record for a program that calls into Node, or the reason the @@ -98,13 +158,23 @@ export function libnodeRecord( const fromEnv = env["SCRIPTC_LIBNODE"]; const beside = option === "beside"; const explicit = option !== undefined && !beside ? option : fromEnv; - // Without an explicit pack, the installed @scriptc/libnode-. - const source = + const target = libnodeTarget(env); + // Without an explicit pack, the installed @scriptc/libnode-. + const installed = explicit !== undefined && explicit !== "" - ? explicit - : findInstalledLibnodePack({ targetName: hostLibnodeTarget(), projectDir }); - if (source === undefined || source === null || source === "") - return `this program calls into Node.js (@scriptc/node), but no libnode pack is installed: install ${libnodePackPackage(hostLibnodeTarget()) ?? "a libnode pack"}, set SCRIPTC_LIBNODE or pass --libnode=`; + ? null + : findInstalledLibnodePack({ targetName: target, projectDir }); + const source = installed ?? explicit; + if (source === undefined || source === null || source === "") { + const pack = libnodePackPackage(target); + return pack === null + ? `this program calls into Node.js (@scriptc/node), but there is no libnode pack for ${target}: set SCRIPTC_LIBNODE or pass --libnode=` + : `this program calls into Node.js (@scriptc/node), but no libnode pack is installed: install ${pack}, set SCRIPTC_LIBNODE or pass --libnode=`; + } + if (installed !== null) { + const problem = installedPackProblem(installed, libnodePackPackage(target)!); + if (problem !== null) return problem; + } if (source.startsWith(ORIGIN)) { // A pack the deployment places next to the executable: nothing to read. return { libnode: source, nodeVersion: "" }; @@ -114,25 +184,30 @@ export function libnodeRecord( if (typeof manifest === "string") return manifest; if (!beside) return { libnode: resolve(source), nodeVersion: manifest.nodeVersion }; const name = `${basename(outPath)}.libnode`; - placeBeside(source, manifest, `${outPath}.libnode`); + const refused = placeBeside(source, manifest, `${outPath}.libnode`); + if (refused !== undefined) return refused; return { libnode: `${ORIGIN}${name}`, nodeVersion: manifest.nodeVersion }; } -/** The part of an executable's cache key that names its libnode pack, or - * undefined when neither --libnode nor SCRIPTC_LIBNODE is set. It holds the - * option, the pack directory and a digest of the pack's manifest.json, so - * that a pack upgraded in place (another Node version, bridge ABI or file - * list) gives cached executables a new key instead of their old record. - * An installed @scriptc/libnode- pack needs no part: it versions - * in lockstep with the compiler, whose version every key already holds, so - * the bootstrap's routed cache keeps serving builds where one is installed. */ +/** The identity of the libnode pack a build that calls into Node.js + * records, which the program's executable cache entry keeps + * (EarlyExecutableCacheOptions.libnodeIdentity): the option, the pack + * directory (named by --libnode or SCRIPTC_LIBNODE, or the installed pack + * found from `projectDir`), and a digest of the pack's manifest.json. A pack + * upgraded in place, removed, or replaced by another installed pack changes + * it. Undefined when no pack is named or installed. */ export function libnodeCacheKey( option: string | undefined, env: NodeJS.ProcessEnv = process.env, + projectDir: string = process.cwd(), ): string | undefined { const fromEnv = env["SCRIPTC_LIBNODE"]; - if (option === undefined && fromEnv === undefined) return undefined; - const source = option !== undefined && option !== "beside" ? option : (fromEnv ?? ""); + const explicit = option !== undefined && option !== "beside" ? option : fromEnv; + const source = + explicit !== undefined && explicit !== "" + ? explicit + : findInstalledLibnodePack({ targetName: libnodeTarget(env), projectDir }); + if (source === undefined || source === null) return option === undefined ? undefined : option; let manifest = "none"; if (isAbsolute(source)) { try { diff --git a/packages/compiler/src/node-islands/modules.test.ts b/packages/compiler/src/node-islands/modules.test.ts new file mode 100644 index 000000000..6badf376a --- /dev/null +++ b/packages/compiler/src/node-islands/modules.test.ts @@ -0,0 +1,188 @@ +import { mkdirSync, mkdtempSync, realpathSync, symlinkSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { describe, expect, test } from "vitest"; +import { + NodeIslandModuleCollector, + nodeIslandManifest, + readNodeIslandModules, + scanModuleSpecifiers, +} from "./modules.js"; + +function project(files: Record): string { + const root = realpathSync(mkdtempSync(join(tmpdir(), "scriptc-island-modules-"))); + for (const [path, text] of Object.entries(files)) { + mkdirSync(dirname(join(root, path)), { recursive: true }); + writeFileSync(join(root, path), text); + } + return root; +} + +describe("module specifiers in JavaScript", () => { + test("string-literal loads in every form", () => { + const scan = scanModuleSpecifiers(` + const a = require("./a"); + const b = require.resolve('./b.json'); + import c from "c"; + import * as d from "d/sub"; + import { e } from "./e.mjs"; + import "./side"; + export * from "./f"; + export { g } from "./g"; + const h = await import("./h.js"); + import.meta.url; + obj.require("x"); + const t = \`\${require("./t")}\`; + `); + expect(scan.specifiers).toEqual([ + "./a", + "./b.json", + "c", + "d/sub", + "./e.mjs", + "./side", + "./f", + "./g", + "./h.js", + "./t", + ]); + expect(scan.computed).toEqual([]); + }); + + test("comments, strings and regular expressions hide nothing and invent nothing", () => { + const scan = scanModuleSpecifiers(` + // require("./commented") + /* import x from "./block" */ + const s = "require('./in-string')"; + const r = /require\\("\\.\\/in-regex"\\)/g; + const q = a / b / c; + const real = require("./real"); + `); + expect(scan.specifiers).toEqual(["./real"]); + expect(scan.computed).toEqual([]); + }); + + test("computed loads are reported", () => { + const forms = (text: string) => scanModuleSpecifiers(text).computed.map((c) => c.form); + expect(forms("require(name)")).toEqual(["require() with a computed specifier"]); + expect(forms("require('./a' + b)")).toEqual(["require() with a computed specifier"]); + expect(forms("import(`./x/${y}`)")).toEqual(["import() with a computed specifier"]); + expect(forms("const r = require; r('./a')")).toEqual(["require as a value"]); + expect(forms("createRequire(import.meta.url)('./a')")).toEqual([ + "a require() from createRequire()", + ]); + expect(forms("function require(x) {} typeof require")).toEqual([]); + }); +}); + +describe("the collected module tree", () => { + test("a relative module and what it loads; missing modules are not shipped", () => { + const root = project({ + "src/main.ts": "", + "src/math.cjs": + "module.exports = require('./util');\ntry { require('./optional'); } catch {}", + "src/util.js": "exports.x = require('../data.json');", + "data.json": "{}", + "src/unused.js": "", + }); + const collector = new NodeIslandModuleCollector(); + expect(collector.add(join(root, "src/main.ts"), "./math.cjs")).toBeNull(); + expect(collector.add(join(root, "src/main.ts"), "./does-not-exist.cjs")).toBeNull(); + expect(collector.add(join(root, "src/main.ts"), "node:fs")).toBeNull(); + const tree = collector.tree()!; + expect(tree.root).toBe(root); + expect(tree.files.map((f) => f.path)).toEqual(["data.json", "src/math.cjs", "src/util.js"]); + expect(NodeIslandModuleCollector.callerPath(tree, join(root, "src/main.ts"))).toBe( + "src/main.ts", + ); + }); + + test("a module of the program ships with the package.json that scopes it", () => { + const root = project({ + "package.json": JSON.stringify({ type: "module" }), + "src/main.ts": "", + "src/esm.js": "export const x = 1;", + }); + const collector = new NodeIslandModuleCollector(); + expect(collector.add(join(root, "src/main.ts"), "./esm.js")).toBeNull(); + expect(collector.tree()!.files.map((f) => f.path)).toEqual(["package.json", "src/esm.js"]); + }); + + test("a computed load in a module of the program is refused", () => { + const root = project({ "main.ts": "", "dyn.cjs": "module.exports = require(process.env.X);" }); + const problem = new NodeIslandModuleCollector().add(join(root, "main.ts"), "./dyn.cjs"); + expect(problem).toBe( + `${join(root, "dyn.cjs")}:1 loads a module through require() with a computed specifier; a module that runs in a Node island loads modules by string-literal specifiers only`, + ); + }); + + test("a package ships whole, with its dependencies, but not its nested node_modules", () => { + const root = project({ + "main.ts": "", + "node_modules/pkg/package.json": JSON.stringify({ dependencies: { dep: "1", absent: "1" } }), + "node_modules/pkg/index.js": "module.exports = require(process.env.ANYTHING_GOES);", + "node_modules/pkg/lib/extra.json": "{}", + "node_modules/pkg/node_modules/.bin/x": "", + "node_modules/dep/package.json": "{}", + "node_modules/dep/index.js": "", + "node_modules/other/index.js": "", + }); + const collector = new NodeIslandModuleCollector(); + expect(collector.add(join(root, "main.ts"), "pkg/lib/extra.json")).toBeNull(); + expect(collector.tree()!.files.map((f) => f.path)).toEqual([ + "node_modules/dep/index.js", + "node_modules/dep/package.json", + "node_modules/pkg/index.js", + "node_modules/pkg/lib/extra.json", + "node_modules/pkg/package.json", + ]); + }); + + test("pnpm's links between package directories stay links inside the tree", () => { + const root = project({ + "app/main.ts": "", + "app/node_modules/.pnpm/a@1/node_modules/a/package.json": JSON.stringify({ + dependencies: { b: "1" }, + }), + "app/node_modules/.pnpm/a@1/node_modules/a/index.js": "", + "app/node_modules/.pnpm/b@1/node_modules/b/index.js": "", + }); + symlinkSync(".pnpm/a@1/node_modules/a", join(root, "app/node_modules/a")); + symlinkSync( + "../../b@1/node_modules/b", + join(root, "app/node_modules/.pnpm/a@1/node_modules/b"), + ); + const collector = new NodeIslandModuleCollector(); + expect(collector.add(join(root, "app/main.ts"), "a")).toBeNull(); + const tree = collector.tree()!; + expect(tree.root).toBe(join(root, "app")); + expect(tree.links).toEqual([ + { path: "node_modules/.pnpm/a@1/node_modules/b", target: "../../b@1/node_modules/b" }, + { path: "node_modules/a", target: ".pnpm/a@1/node_modules/a" }, + ]); + expect(tree.files.map((f) => f.path)).toEqual([ + "node_modules/.pnpm/a@1/node_modules/a/index.js", + "node_modules/.pnpm/a@1/node_modules/a/package.json", + "node_modules/.pnpm/b@1/node_modules/b/index.js", + ]); + }); + + test("built-in modules alone ship no tree", () => { + const root = project({ "main.ts": "" }); + const collector = new NodeIslandModuleCollector(); + expect(collector.add(join(root, "main.ts"), "os")).toBeNull(); + expect(collector.tree()).toBeNull(); + expect(NodeIslandModuleCollector.callerPath(null, join(root, "main.ts"))).toBe("main.ts"); + }); + + test("the manifest names relative paths only, and the data is checked against it", () => { + const root = project({ "main.ts": "", "m.cjs": "exports.f = () => 1;" }); + const collector = new NodeIslandModuleCollector(); + collector.add(join(root, "main.ts"), "./m.cjs"); + const tree = collector.tree()!; + expect(nodeIslandManifest(tree)).not.toContain(root); + expect(readNodeIslandModules(tree)).toEqual(new TextEncoder().encode("exports.f = () => 1;")); + writeFileSync(join(root, "m.cjs"), "exports.f = () => 2;"); + expect(readNodeIslandModules(tree)).toBe(`${join(root, "m.cjs")} changed during the build`); + }); +}); diff --git a/packages/compiler/src/node-islands/modules.ts b/packages/compiler/src/node-islands/modules.ts new file mode 100644 index 000000000..0d63dc333 --- /dev/null +++ b/packages/compiler/src/node-islands/modules.ts @@ -0,0 +1,627 @@ +/* The island module tree: the JavaScript a program's Node island calls load, + * collected at build time and shipped with the executable, so a deployed + * binary never reads modules from the machine that built it. + * + * Each `call`/`callSync` names its module with a string literal. The build + * resolves it from the calling module as Node's require() does: + * - a relative or absolute specifier names a module of the program: the + * module and every module its string-literal `require()`, `import` and + * `export ... from` reach are collected; a computed specifier in such a + * module is refused, since the build cannot know what it loads; + * - a package specifier names a package: its whole directory is collected + * (computed requires inside packages keep working), and so are its + * `dependencies`, `optionalDependencies` and `peerDependencies`, found as + * Node finds them; + * - a Node built-in module needs nothing, and neither does a module the + * build cannot find: loading it throws MODULE_NOT_FOUND when the call + * runs, as under Node. + * Paths keep their layout relative to one root directory (the deepest + * directory that holds every collected path), with symbolic links between + * package directories (pnpm's layout) kept as links. Only these relative + * paths enter the executable. Every read goes through the frontend input + * tracker, so an edit to an island module invalidates cached builds. */ +import { createHash } from "node:crypto"; +import { mkdirSync, readFileSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; +import { builtinModules } from "node:module"; +import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "node:path"; +import { + trackedAccessibleEntries, + trackedDirectoryExists, + trackedFileExists, + trackedReadBytes, + trackedReadFile, + trackedRealpath, +} from "../frontend/input-tracker.js"; +import type { IrNodeIslandModules } from "../ir/ir.js"; + +export type NodeIslandModuleTree = IrNodeIslandModules; +type NodeIslandModuleFile = IrNodeIslandModules["files"][number]; +type NodeIslandModuleLink = IrNodeIslandModules["links"][number]; + +const BUILTINS: ReadonlySet = new Set(builtinModules); + +/** Extensions require() tries for an extensionless file, in its order. */ +const FILE_EXTENSIONS = [".js", ".json", ".node"]; + +/** Module files whose imports are traced. */ +const TRACED = /\.(?:[cm]?js|[cm]?ts)$/; + +function isBuiltin(specifier: string): boolean { + return specifier.startsWith("node:") || BUILTINS.has(specifier); +} + +function isPathSpecifier(specifier: string): boolean { + return ( + specifier === "." || + specifier === ".." || + specifier.startsWith("./") || + specifier.startsWith("../") || + isAbsolute(specifier) + ); +} + +function packageName(specifier: string): string { + const parts = specifier.split("/"); + return specifier.startsWith("@") ? parts.slice(0, 2).join("/") : parts[0]!; +} + +/** The file require() loads for `base` (a file, an extension away, or a + * directory with a package.json main or an index file), or null. */ +function resolveFile(base: string): string | null { + if (trackedFileExists(base)) return base; + for (const extension of FILE_EXTENSIONS) + if (trackedFileExists(base + extension)) return base + extension; + if (!trackedDirectoryExists(base)) return null; + const manifest = trackedReadFile(join(base, "package.json")); + if (manifest !== null) { + let main: unknown; + try { + main = (JSON.parse(manifest) as { main?: unknown }).main; + } catch { + main = undefined; + } + if (typeof main === "string" && main !== "") { + const target = join(base, main); + const found = resolveFile(target) ?? resolveFile(join(target, "index")); + if (found !== null) return found; + } + } + for (const extension of FILE_EXTENSIONS) { + const index = join(base, `index${extension}`); + if (trackedFileExists(index)) return index; + } + return null; +} + +/** The directory of package `name` as require() finds it from `fromDir`: + * `node_modules/` in `fromDir` and each ancestor not itself named + * node_modules. */ +function packageDirectory(fromDir: string, name: string): string | null { + for (let dir = fromDir; ;) { + if (basename(dir) !== "node_modules") { + const candidate = join(dir, "node_modules", name); + if (trackedDirectoryExists(candidate)) return candidate; + } + const parent = dirname(dir); + if (parent === dir) return null; + dir = parent; + } +} + +/** The package.json nearest to `dir`, as Node finds a module's package + * scope, or null. */ +function packageScope(dir: string): string | null { + for (let at = dir; ;) { + if (basename(at) === "node_modules") return null; + const candidate = join(at, "package.json"); + if (trackedFileExists(candidate)) return candidate; + const parent = dirname(at); + if (parent === at) return null; + at = parent; + } +} + +function sha256(bytes: Uint8Array): string { + return createHash("sha256").update(bytes).digest("hex"); +} + +/** The deepest directory that contains every path. */ +function commonDirectory(paths: readonly string[]): string { + let common = paths[0]!.split(sep); + for (const path of paths.slice(1)) { + const parts = path.split(sep); + let n = 0; + while (n < common.length && n < parts.length && common[n] === parts[n]) n++; + common = common.slice(0, n); + } + const joined = common.join(sep); + return joined === "" ? sep : joined; +} + +function portable(path: string): string { + return sep === "/" ? path : path.split(sep).join("/"); +} + +export class NodeIslandModuleCollector { + /** Real paths of collected files. */ + private readonly files = new Set(); + /** Link path (its directory real) → real target. */ + private readonly links = new Map(); + private readonly packages = new Set(); + private readonly traced = new Set(); + /** Real paths of the calling modules. */ + private readonly callers = new Set(); + + /** Collects the module `specifier` names from `caller` (a real path). + * Returns null, or the reason the build cannot ship it. */ + add(caller: string, specifier: string): string | null { + this.callers.add(caller); + return this.resolveFrom(caller, specifier); + } + + /** The collected tree, or null when no call loads a file. */ + tree(): NodeIslandModuleTree | null { + if (this.files.size === 0 && this.links.size === 0) return null; + const anchors = [ + ...this.files, + ...this.links.keys(), + ...this.links.values(), + ...[...this.callers].map((caller) => dirname(caller)), + ]; + const root = commonDirectory(anchors); + const files: NodeIslandModuleFile[] = []; + for (const file of [...this.files].sort()) { + const bytes = trackedReadBytes(file); + if (bytes === null) continue; + files.push({ + path: portable(relative(root, file)), + size: bytes.length, + sha256: sha256(bytes), + }); + } + const links: NodeIslandModuleLink[] = [...this.links] + .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)) + .map(([path, target]) => ({ + path: portable(relative(root, path)), + target: portable(relative(dirname(path), target)), + })); + return { root, files, links }; + } + + /** The tree path a caller's module has: relative to the tree root. */ + static callerPath(tree: NodeIslandModuleTree | null, caller: string): string { + return tree === null ? basename(caller) : portable(relative(tree.root, caller)); + } + + private resolveFrom(fromFile: string, specifier: string): string | null { + if (isBuiltin(specifier)) return null; + if (specifier.startsWith("#")) + return `'${specifier}' is a package import ("imports" in package.json), which Node island modules of the program cannot use`; + if (specifier.startsWith("file:") || /^[a-z][a-z0-9+.-]*:/i.test(specifier)) + return `'${specifier}' is a URL; name the module by its relative path or package name`; + // A module the build cannot find is not shipped: loading it throws + // MODULE_NOT_FOUND when the call runs, as it does under Node (an + // optional require in a try block keeps working). + if (isPathSpecifier(specifier)) { + const target = resolveFile(resolve(dirname(fromFile), specifier)); + return target === null ? null : this.addModule(this.placed(target)); + } + const dir = packageDirectory(dirname(fromFile), packageName(specifier)); + if (dir !== null) this.addPackage(dir); + return null; + } + + /** The real path of `path`, recording a link when `path` itself (not an + * ancestor) is a symbolic link. */ + private placed(path: string): string { + const target = trackedRealpath(path) ?? path; + const parent = trackedRealpath(dirname(path)) ?? dirname(path); + const link = join(parent, basename(path)); + if (link !== target) this.links.set(link, target); + return target; + } + + /** A module of the program: collected with the package.json that scopes + * it (Node reads its "type"), and its imports traced. */ + private addModule(file: string): string | null { + if (this.traced.has(file)) return null; + this.traced.add(file); + this.files.add(file); + const scope = packageScope(dirname(file)); + if (scope !== null) this.files.add(scope); + if (!TRACED.test(file)) return null; + const text = trackedReadFile(file); + if (text === null) return `cannot read ${file}`; + const scan = scanModuleSpecifiers(text); + if (scan.computed.length > 0) { + const first = scan.computed[0]!; + const line = text.slice(0, first.at).split("\n").length; + return `${file}:${line} loads a module through ${first.form}; a module that runs in a Node island loads modules by string-literal specifiers only`; + } + for (const specifier of scan.specifiers) { + const problem = this.resolveFrom(file, specifier); + if (problem !== null) return problem; + } + return null; + } + + /** A package: its whole directory, then its dependencies. */ + private addPackage(found: string): void { + const dir = this.placed(found); + if (this.packages.has(dir)) return; + this.packages.add(dir); + this.addDirectory(dir, new Set()); + const manifest = trackedReadFile(join(dir, "package.json")); + if (manifest === null) return; + let parsed: Record; + try { + parsed = JSON.parse(manifest) as Record; + } catch { + return; + } + const names = new Set(); + for (const field of ["dependencies", "optionalDependencies", "peerDependencies"]) { + const deps = parsed[field]; + if (deps !== null && typeof deps === "object") + for (const name of Object.keys(deps)) names.add(name); + } + for (const name of [...names].sort()) { + // A dependency that is not installed (optional, or a peer the program + // does not use) is not loaded either. + const depDir = packageDirectory(dir, name); + if (depDir !== null) this.addPackage(depDir); + } + } + + /** Every file below `dir`, except nested node_modules (dependencies come + * through package.json) and .git. */ + private addDirectory(dir: string, visited: Set): void { + const real = trackedRealpath(dir) ?? dir; + if (visited.has(real)) return; + visited.add(real); + const entries = trackedAccessibleEntries(dir); + if (entries === null) return; + for (const file of entries.files) this.files.add(join(dir, file)); + for (const sub of entries.directories) { + if (sub === "node_modules" || sub === ".git") continue; + this.addDirectory(join(dir, sub), visited); + } + } +} + +/* ── module specifiers in JavaScript source ───────────────────────────── */ + +interface Token { + kind: "name" | "string" | "punct" | "other"; + value: string; + at: number; +} + +/** Keywords after which a `/` starts a regular expression. */ +const REGEX_AFTER = new Set([ + "return", + "typeof", + "instanceof", + "in", + "of", + "new", + "delete", + "void", + "throw", + "case", + "do", + "else", + "yield", + "await", +]); + +function isNameStart(c: string): boolean { + return /[A-Za-z_$\u0080-￿]/.test(c); +} + +function isNamePart(c: string): boolean { + return /[A-Za-z0-9_$\u0080-￿]/.test(c); +} + +/** The significant tokens of a JavaScript or TypeScript source: names, + * string literals (cooked) and punctuators, without comments, template + * text or regular expressions. */ +export function moduleTokens(text: string): Token[] { + const tokens: Token[] = []; + /** Brace depths at which a template literal's `${` was opened. */ + const templates: number[] = []; + let depth = 0; + let i = 0; + const n = text.length; + const regexAllowed = (): boolean => { + const last = tokens[tokens.length - 1]; + if (last === undefined) return true; + if (last.kind === "name") return REGEX_AFTER.has(last.value); + if (last.kind === "punct") + return last.value !== ")" && last.value !== "]" && last.value !== "}"; + return false; + }; + const template = (start: number): void => { + // From just after a backtick or a closing `}` to the end of the + // template or the next `${`. + while (i < n) { + const c = text[i]!; + if (c === "\\") { + i += 2; + continue; + } + if (c === "`") { + i++; + tokens.push({ kind: "other", value: "`", at: start }); + return; + } + if (c === "$" && text[i + 1] === "{") { + i += 2; + templates.push(depth); + depth++; + tokens.push({ kind: "punct", value: "${", at: start }); + return; + } + i++; + } + }; + while (i < n) { + const c = text[i]!; + if (c === " " || c === "\t" || c === "\n" || c === "\r" || c === "\f" || c === "\v") { + i++; + continue; + } + if (c === "/" && text[i + 1] === "/") { + while (i < n && text[i] !== "\n") i++; + continue; + } + if (c === "/" && text[i + 1] === "*") { + const end = text.indexOf("*/", i + 2); + i = end < 0 ? n : end + 2; + continue; + } + if (c === "#" && i === 0 && text[1] === "!") { + while (i < n && text[i] !== "\n") i++; + continue; + } + if (c === "'" || c === '"') { + const start = i; + let value = ""; + i++; + while (i < n && text[i] !== c && text[i] !== "\n") { + if (text[i] === "\\") { + const e = text[i + 1] ?? ""; + value += e === "n" ? "\n" : e === "t" ? "\t" : e === "r" ? "\r" : e === "\n" ? "" : e; + i += 2; + continue; + } + value += text[i]; + i++; + } + i++; + tokens.push({ kind: "string", value, at: start }); + continue; + } + if (c === "`") { + const start = i; + i++; + template(start); + continue; + } + if (c === "/" && regexAllowed()) { + const start = i; + i++; + let inClass = false; + while (i < n && text[i] !== "\n") { + const r = text[i]!; + if (r === "\\") { + i += 2; + continue; + } + if (r === "[") inClass = true; + else if (r === "]") inClass = false; + else if (r === "/" && !inClass) break; + i++; + } + i++; + while (i < n && isNamePart(text[i]!)) i++; + tokens.push({ kind: "other", value: "/re/", at: start }); + continue; + } + if (isNameStart(c)) { + const start = i; + while (i < n && isNamePart(text[i]!)) i++; + tokens.push({ kind: "name", value: text.slice(start, i), at: start }); + continue; + } + if (/[0-9]/.test(c) || (c === "." && /[0-9]/.test(text[i + 1] ?? ""))) { + const start = i; + while (i < n && /[0-9A-Za-z_.]/.test(text[i]!)) i++; + tokens.push({ kind: "other", value: text.slice(start, i), at: start }); + continue; + } + if (c === "{") depth++; + if (c === "}") { + if (templates.length > 0 && templates[templates.length - 1] === depth - 1) { + templates.pop(); + depth--; + const start = i; + i++; + tokens.push({ kind: "punct", value: "}", at: start }); + template(start); + continue; + } + depth--; + } + if (c === "." && text[i + 1] === "." && text[i + 2] === ".") { + tokens.push({ kind: "punct", value: "...", at: i }); + i += 3; + continue; + } + if (c === "?" && text[i + 1] === ".") { + tokens.push({ kind: "punct", value: "?.", at: i }); + i += 2; + continue; + } + tokens.push({ kind: "punct", value: c, at: i }); + i++; + } + return tokens; +} + +export interface ModuleSpecifierScan { + /** String-literal specifiers of require(), require.resolve(), import() + * and import/export declarations, in source order. */ + specifiers: string[]; + /** Loads whose module the source does not name with a string literal. */ + computed: { at: number; form: string }[]; +} + +export function scanModuleSpecifiers(text: string): ModuleSpecifierScan { + const tokens = moduleTokens(text); + const specifiers: string[] = []; + const computed: { at: number; form: string }[] = []; + const at = (k: number): Token | undefined => tokens[k]; + const is = (k: number, kind: Token["kind"], value?: string): boolean => { + const t = tokens[k]; + return t !== undefined && t.kind === kind && (value === undefined || t.value === value); + }; + /** A call `(` at k whose single argument is a string literal. */ + const literalCall = (k: number): string | null => + is(k, "punct", "(") && + is(k + 1, "string") && + (is(k + 2, "punct", ")") || is(k + 2, "punct", ",")) + ? at(k + 1)!.value + : null; + for (let k = 0; k < tokens.length; k++) { + const t = tokens[k]!; + if (t.kind !== "name") continue; + const member = is(k - 1, "punct", ".") || is(k - 1, "punct", "?."); + if (t.value === "require" && !member && !is(k + 1, "punct", ":")) { + if (is(k - 1, "name", "function") || is(k - 1, "name", "typeof")) continue; + let call = k + 1; + let form = "require()"; + if (is(k + 1, "punct", ".") && is(k + 2, "name", "resolve")) { + call = k + 3; + form = "require.resolve()"; + } else if (is(k + 1, "punct", ".")) { + continue; + } + if (!is(call, "punct", "(")) { + computed.push({ at: t.at, form: "require as a value" }); + continue; + } + const literal = literalCall(call); + if (literal === null) computed.push({ at: t.at, form: `${form} with a computed specifier` }); + else specifiers.push(literal); + continue; + } + if (t.value === "createRequire" && !member && is(k + 1, "punct", "(")) { + computed.push({ at: t.at, form: "a require() from createRequire()" }); + continue; + } + if (t.value === "import" && !member) { + if (is(k + 1, "punct", ".")) continue; // import.meta + if (is(k + 1, "punct", "(")) { + const literal = literalCall(k + 1); + if (literal === null) + computed.push({ at: t.at, form: "import() with a computed specifier" }); + else specifiers.push(literal); + continue; + } + if (is(k + 1, "string")) { + specifiers.push(at(k + 1)!.value); + continue; + } + for (let j = k + 1; j < tokens.length && j < k + 400; j++) { + if (is(j, "punct", ";")) break; + if (is(j, "name", "from") && is(j + 1, "string")) { + specifiers.push(at(j + 1)!.value); + break; + } + } + continue; + } + if (t.value === "export" && !member) { + let j = k + 1; + if (is(j, "name", "type")) j++; + if (is(j, "punct", "*")) { + for (; j < tokens.length && j < k + 400; j++) { + if (is(j, "punct", ";")) break; + if (is(j, "name", "from") && is(j + 1, "string")) { + specifiers.push(at(j + 1)!.value); + break; + } + } + } else if (is(j, "punct", "{")) { + while (j < tokens.length && !is(j, "punct", "}")) j++; + if (is(j + 1, "name", "from") && is(j + 2, "string")) specifiers.push(at(j + 2)!.value); + } + } + } + return { specifiers, computed }; +} + +/* ── shipping the tree ────────────────────────────────────────────────── */ + +/** The manifest the executable records (scr_node_island.c reads it): + * `{"files":[[path,size,sha256],...],"links":[[path,target],...]}`. Its + * SHA-256 names the tree. */ +export function nodeIslandManifest(modules: IrNodeIslandModules): string { + return JSON.stringify({ + files: modules.files.map((file) => [file.path, file.size, file.sha256]), + links: modules.links.map((link) => [link.path, link.target]), + }); +} + +/** The tree's files concatenated in manifest order, each checked against + * the digest the build recorded, or the reason one no longer matches. */ +export function readNodeIslandModules(modules: IrNodeIslandModules): Uint8Array | string { + const parts: Uint8Array[] = []; + let total = 0; + for (const file of modules.files) { + const path = join(modules.root, ...file.path.split("/")); + let bytes: Uint8Array; + try { + bytes = new Uint8Array(readFileSync(path)); + } catch { + return `${path} was removed during the build`; + } + if (bytes.length !== file.size || sha256(bytes) !== file.sha256) + return `${path} changed during the build`; + parts.push(bytes); + total += bytes.length; + } + const data = new Uint8Array(total); + let offset = 0; + for (const part of parts) { + data.set(part, offset); + offset += part.length; + } + return data; +} + +/** Writes the tree to `target` (replacing it), for `--libnode=beside`, or + * returns the reason it cannot. */ +export function placeNodeIslandModules( + modules: IrNodeIslandModules, + target: string, +): string | null { + const data = readNodeIslandModules(modules); + if (typeof data === "string") return data; + rmSync(target, { recursive: true, force: true }); + mkdirSync(target, { recursive: true }); + let offset = 0; + for (const file of modules.files) { + const path = join(target, ...file.path.split("/")); + mkdirSync(dirname(path), { recursive: true }); + writeFileSync(path, data.subarray(offset, offset + file.size)); + offset += file.size; + } + for (const link of modules.links) { + const path = join(target, ...link.path.split("/")); + mkdirSync(dirname(path), { recursive: true }); + symlinkSync(link.target, path); + } + return null; +} diff --git a/packages/compiler/src/startup-cache.ts b/packages/compiler/src/startup-cache.ts index 1bc74bef0..950a96d08 100644 --- a/packages/compiler/src/startup-cache.ts +++ b/packages/compiler/src/startup-cache.ts @@ -2,6 +2,7 @@ * full lowering/emitter graph; exact builds need only these cache/native * identity primitives before deciding whether that graph is necessary. */ export { readRoutedExecutableCache } from "./executable/executable-cache.js"; +export { libnodeCacheKey } from "./node-islands/libnode.js"; export type { EarlyExecutableRouteOptions } from "./executable/executable-cache.js"; export { executableNativeEnvironmentFingerprint, diff --git a/packages/compiler/test/native-codegen-integration.test.ts b/packages/compiler/test/native-codegen-integration.test.ts index d398fa1d3..cecebb49c 100644 --- a/packages/compiler/test/native-codegen-integration.test.ts +++ b/packages/compiler/test/native-codegen-integration.test.ts @@ -378,7 +378,7 @@ done: expect(await symbols(helperObject, ["-u"])).toEqual(await symbols(clangObject, ["-u"])); expect(await symbols(helperObject, ["-gU"])).toEqual(await symbols(clangObject, ["-gU"])); expect(await symbols(helperObject, ["-gU"])).toEqual(["0000000000000000 T _main"]); - expect(await symbols(helperObject, ["-u"])).toContain("_scr_runtime_abi_v36"); + expect(await symbols(helperObject, ["-u"])).toContain("_scr_runtime_abi_v37"); const clangExe = join(dir, "clang-program"); const linkDriver = join(dir, "clang-driver.c"); @@ -712,7 +712,7 @@ done: .split("\n"); expect(undefinedSymbols).toContain("_sf_scale"); expect(undefinedSymbols).toContain("_sf_callback_mix"); - expect(undefinedSymbols).toContain("_scr_runtime_abi_v36"); + expect(undefinedSymbols).toContain("_scr_runtime_abi_v37"); }); test("helper and clang object paths retain the same outbound FFI C ABI", async () => { diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7447-union-record-arm-width.mjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7447-union-record-arm-width.mjs.json new file mode 100644 index 000000000..68bd750d9 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7447-union-record-arm-width.mjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7447-union-record-arm-width.mjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7448-union-record-arm-width.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7448-union-record-arm-width.ts.json new file mode 100644 index 000000000..498ee884a --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7448-union-record-arm-width.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7448-union-record-arm-width.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7460-transform-writev.mjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7460-transform-writev.mjs.json new file mode 100644 index 000000000..1015e1354 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7460-transform-writev.mjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7460-transform-writev.mjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7461-transform-writev.cjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7461-transform-writev.cjs.json new file mode 100644 index 000000000..4d3c86215 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7461-transform-writev.cjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7461-transform-writev.cjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7462-transform-writev.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7462-transform-writev.ts.json new file mode 100644 index 000000000..9b42f2d60 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7462-transform-writev.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7462-transform-writev.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7463-stream-option-values.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7463-stream-option-values.ts.json new file mode 100644 index 000000000..5aa543dea --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7463-stream-option-values.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7463-stream-option-values.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7464-stream-option-values.mjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7464-stream-option-values.mjs.json new file mode 100644 index 000000000..2a438d418 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7464-stream-option-values.mjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7464-stream-option-values.mjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7465-transform-callback-values.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7465-transform-callback-values.ts.json new file mode 100644 index 000000000..17633d208 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7465-transform-callback-values.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7465-transform-callback-values.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7466-stream-construct-assign.mjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7466-stream-construct-assign.mjs.json new file mode 100644 index 000000000..f5bb52517 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7466-stream-construct-assign.mjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7466-stream-construct-assign.mjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7500-pipeline-callback-undefined.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7500-pipeline-callback-undefined.ts.json new file mode 100644 index 000000000..fe65b63f0 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7500-pipeline-callback-undefined.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7500-pipeline-callback-undefined.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7501-pipeline-callback-undefined.mjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7501-pipeline-callback-undefined.mjs.json new file mode 100644 index 000000000..8d52980fe --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7501-pipeline-callback-undefined.mjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7501-pipeline-callback-undefined.mjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7502-readline-cursor-undefined.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7502-readline-cursor-undefined.ts.json new file mode 100644 index 000000000..6fd23042e --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7502-readline-cursor-undefined.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7502-readline-cursor-undefined.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7543-var-list-declarator-order.js.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7543-var-list-declarator-order.js.json new file mode 100644 index 000000000..d193e84ed --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7543-var-list-declarator-order.js.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7543-var-list-declarator-order.js" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7545-string-regex-checked-arguments.js.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7545-string-regex-checked-arguments.js.json new file mode 100644 index 000000000..03b3197c4 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7545-string-regex-checked-arguments.js.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7545-string-regex-checked-arguments.js" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7548-cjs-receiverless-builtin-calls/main.cjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7548-cjs-receiverless-builtin-calls/main.cjs.json new file mode 100644 index 000000000..f3b8ba419 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7548-cjs-receiverless-builtin-calls/main.cjs.json @@ -0,0 +1,7 @@ +{ + "order": [ + "/tests/corpus/7548-cjs-receiverless-builtin-calls/lib.cjs", + "/tests/corpus/7548-cjs-receiverless-builtin-calls/main.cjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7549-cjs-minified-esmodule-stamp/main.cjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7549-cjs-minified-esmodule-stamp/main.cjs.json new file mode 100644 index 000000000..08adae885 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7549-cjs-minified-esmodule-stamp/main.cjs.json @@ -0,0 +1,8 @@ +{ + "order": [ + "/tests/corpus/7549-cjs-minified-esmodule-stamp/stamped.cjs", + "/tests/corpus/7549-cjs-minified-esmodule-stamp/module-stamped.cjs", + "/tests/corpus/7549-cjs-minified-esmodule-stamp/main.cjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7553-var-for-in-of-zero-iterations.js.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7553-var-for-in-of-zero-iterations.js.json new file mode 100644 index 000000000..9f470ca37 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7553-var-for-in-of-zero-iterations.js.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7553-var-for-in-of-zero-iterations.js" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7620-js-class-view-descriptors.js.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7620-js-class-view-descriptors.js.json new file mode 100644 index 000000000..2cb1ece22 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7620-js-class-view-descriptors.js.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7620-js-class-view-descriptors.js" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7661-node-call-modules/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7661-node-call-modules/main.ts.json new file mode 100644 index 000000000..85440ddc4 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7661-node-call-modules/main.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7661-node-call-modules/main.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7662-node-call-exit-events/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7662-node-call-exit-events/main.ts.json new file mode 100644 index 000000000..455735463 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7662-node-call-exit-events/main.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7662-node-call-exit-events/main.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/diagnostics/stream-runtime-option-order.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/diagnostics/stream-runtime-option-order.ts.json new file mode 100644 index 000000000..40b64edac --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/diagnostics/stream-runtime-option-order.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/diagnostics/stream-runtime-option-order.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/7546-minified-esm-rules/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/7546-minified-esm-rules/main.ts.json new file mode 100644 index 000000000..d026fb402 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/7546-minified-esm-rules/main.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/fixtures/npm/cases/7546-minified-esm-rules/main.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/7547-minified-cjs-chunks/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/7547-minified-cjs-chunks/main.ts.json new file mode 100644 index 000000000..212551ccd --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/7547-minified-cjs-chunks/main.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/fixtures/npm/cases/7547-minified-cjs-chunks/main.ts" + ], + "diags": [] +} diff --git a/packages/libnode-common/bridge/scr_node_bridge.cc b/packages/libnode-common/bridge/scr_node_bridge.cc index 1b578a0f7..ef60851d6 100644 --- a/packages/libnode-common/bridge/scr_node_bridge.cc +++ b/packages/libnode-common/bridge/scr_node_bridge.cc @@ -49,6 +49,7 @@ node::EmbedderSnapshotData::Pointer g_snapshot; std::unique_ptr g_setup; Locker *g_locker = nullptr; Global g_call; +Global g_finish; char *Copy(const std::string &text, size_t *len) { char *out = static_cast(std::malloc(text.size() + 1)); @@ -59,11 +60,23 @@ char *Copy(const std::string &text, size_t *len) { return out; } +// The start of the runtime's stack when the caller runs on it (see the +// v8::base::Stack definitions at the end of this file), else 0. +uintptr_t HostStackStart() { + return g_host != nullptr && g_host->stack_start != nullptr ? g_host->stack_start() : 0; +} + Local Utf8(Isolate *isolate, const char *text, size_t len) { return String::NewFromUtf8(isolate, text, NewStringType::kNormal, static_cast(len)) .ToLocalChecked(); } +// A string longer than V8 can hold is empty instead of a fatal error. +v8::MaybeLocal MaybeUtf8(Isolate *isolate, const char *text, size_t len) { + if (len > static_cast(String::kMaxLength)) return {}; + return String::NewFromUtf8(isolate, text, NewStringType::kNormal, static_cast(len)); +} + Local Utf8(Isolate *isolate, const char *text) { return Utf8(isolate, text, std::strlen(text)); } @@ -75,15 +88,20 @@ std::string ToStdString(Isolate *isolate, Local value) { // The outcome of an exception the dispatcher itself did not catch (a stack // overflow, for example): an Error copy with its message. +// Reading the message runs user code (a getter, a toString), which may throw +// again; the copy then has an empty message. std::string ThrownOutcome(Isolate *isolate, Local context, Local exception) { + TryCatch inner(isolate); Local message = exception; if (exception->IsObject()) { Local own; if (exception.As()->Get(context, Utf8(isolate, "message")).ToLocal(&own)) message = own; } + Local text; Local quoted; - if (!v8::JSON::Stringify(context, message->ToString(context).ToLocalChecked()).ToLocal("ed)) + if (!message->ToString(context).ToLocal(&text) || + !v8::JSON::Stringify(context, text).ToLocal("ed)) return "T{\"error\":{\"kind\":0,\"message\":\"\"}}"; return "T{\"error\":{\"kind\":0,\"message\":" + ToStdString(isolate, quoted) + "}}"; } @@ -106,12 +124,18 @@ char *Start(const ScrNodeBridgeHost *host, const char *bootstrap, const char *ar if (dladdr(reinterpret_cast(&napi_create_function), &info) && info.dli_fname) dlopen(info.dli_fname, RTLD_NOW | RTLD_NOLOAD | RTLD_GLOBAL); std::vector args = {argv0}; + // The program is not a node process: NODE_OPTIONS from its environment + // does not apply, and Node leaves the process's resource limits and code + // pages as they are. auto result = node::InitializeOncePerProcess( args, {node::ProcessInitializationFlags::kNoInitializeV8, node::ProcessInitializationFlags::kNoInitializeNodeV8Platform, node::ProcessInitializationFlags::kNoStdioInitialization, node::ProcessInitializationFlags::kNoDefaultSignalHandling, - node::ProcessInitializationFlags::kNoPrintHelpOrVersionOutput}); + node::ProcessInitializationFlags::kNoPrintHelpOrVersionOutput, + node::ProcessInitializationFlags::kDisableNodeOptionsEnv, + node::ProcessInitializationFlags::kNoAdjustResourceLimits, + node::ProcessInitializationFlags::kNoUseLargePages}); if (result->early_return() || result->exit_code() != 0) { std::string message = "Node initialization failed"; for (const std::string &error : result->errors()) message += ": " + error; @@ -121,9 +145,12 @@ char *Start(const ScrNodeBridgeHost *host, const char *bootstrap, const char *ar v8::V8::InitializePlatform(g_platform.get()); v8::V8::Initialize(); g_snapshot = node::EmbedderSnapshotData::BuiltinSnapshotData(); + // Modules resolve inside the program's own module tree, never from + // NODE_PATH or the user's global module directories. auto flags = static_cast( node::EnvironmentFlags::kDefaultFlags | node::EnvironmentFlags::kNoCreateInspector | - node::EnvironmentFlags::kNoStartDebugSignalHandler); + node::EnvironmentFlags::kNoStartDebugSignalHandler | + node::EnvironmentFlags::kNoGlobalSearchPaths); std::vector errors; g_setup = g_snapshot ? node::CommonEnvironmentSetup::CreateFromSnapshot( g_platform.get(), &errors, g_snapshot.get(), result->args(), @@ -171,12 +198,16 @@ char *Start(const ScrNodeBridgeHost *host, const char *bootstrap, const char *ar } Local dispatcher; Local call; + Local finish; if (!global->Get(context, Utf8(isolate, "__scriptcNodeIsland")).ToLocal(&dispatcher) || !dispatcher->IsObject() || !dispatcher.As()->Get(context, Utf8(isolate, "call")).ToLocal(&call) || - !call->IsFunction()) + !call->IsFunction() || + !dispatcher.As()->Get(context, Utf8(isolate, "finish")).ToLocal(&finish) || + !finish->IsFunction()) return Copy("the Node island bootstrap left no dispatcher", nullptr); g_call.Reset(isolate, call.As()); + g_finish.Reset(isolate, finish.As()); global->Delete(context, Utf8(isolate, "__scriptcNodeIsland")).FromMaybe(false); global->Delete(context, Utf8(isolate, "__scriptcNodeIslandHost")).FromMaybe(false); return nullptr; @@ -192,12 +223,23 @@ char *Call(uint32_t id, int sync, const char *file, const char *specifier, const { node::CallbackScope scope(isolate, context->Global(), node::async_context{0, 0}); TryCatch try_catch(isolate); + Local args_text; + if (!MaybeUtf8(isolate, args, args_len).ToLocal(&args_text)) { + // The argument JSON is longer than V8's string limit: the RangeError + // V8 throws for such a string. + outcome = "T{\"error\":{\"kind\":2,\"message\":\"Invalid string length\"}}"; + if (!sync) { + g_host->settle(id, outcome.data(), outcome.size()); + return nullptr; + } + return Copy(outcome, len); + } Local argv[] = {Integer::NewFromUnsigned(isolate, id), v8::Boolean::New(isolate, sync != 0), Utf8(isolate, file), Utf8(isolate, specifier), Utf8(isolate, name), - Utf8(isolate, args, args_len)}; + args_text}; Local result; if (g_call.Get(isolate) ->Call(context, v8::Undefined(isolate), sizeof argv / sizeof argv[0], argv) @@ -232,11 +274,29 @@ void Pump() { g_platform->DrainTasks(isolate); } +// The program ends with `code`: Node's process ends with it too. Node's +// 'beforeExit' runs (and the work it schedules), then 'exit'; the result +// is the exit code Node's process ends with (process.exitCode). +int Finish(int code) { + if (!g_setup) return code; + Isolate *isolate = g_setup->isolate(); + { + HandleScope handles(isolate); + Local context = g_setup->context(); + Context::Scope context_scope(context); + TryCatch try_catch(isolate); + Local argv[] = {Integer::New(isolate, code)}; + (void)g_finish.Get(isolate)->Call(context, v8::Undefined(isolate), 1, argv); + } + return node::SpinEventLoop(g_setup->env()).FromMaybe(code); +} + void Stop() { if (!g_setup) return; Isolate *isolate = g_setup->isolate(); node::Stop(g_setup->env()); g_call.Reset(); + g_finish.Reset(); isolate->Exit(); delete g_locker; g_locker = nullptr; @@ -249,10 +309,37 @@ void Stop() { const ScrNodeBridge kBridge = { SCR_NODE_BRIDGE_ABI, NODE_VERSION_STRING, NODE_MODULE_VERSION, Start, Call, Release, BackendFd, BackendTimeout, Alive, Pump, Stop, + Finish, }; } // namespace +#if defined(__linux__) +// V8 asks v8::base::Stack for the start of the current thread's stack when +// it records the bound of its conservative stack scan (isolate creation, +// Locker). libnode calls these exported functions through its PLT, and this +// library precedes libnode in the lookup scope, so the definitions below +// answer for it: the runtime's stack while the caller runs on it, else +// libnode's own answer (RTLD_NEXT). Without them a garbage collection that +// scans the stack while JavaScript runs on the runtime's stack walks from +// that stack up to the main thread's and faults in the unmapped gap. +// (macOS binds libnode's calls inside libnode; its packs need another way.) +#define SCR_V8_STACK_START(fn, symbol) \ + extern "C" __attribute__((visibility("default"))) void *fn() __asm__(symbol); \ + extern "C" void *fn() { \ + static void *(*next)() = nullptr; \ + uintptr_t start = HostStackStart(); \ + if (start != 0) return reinterpret_cast(start); \ + if (next == nullptr) next = reinterpret_cast(dlsym(RTLD_NEXT, symbol)); \ + return next != nullptr ? next() : nullptr; \ + } +SCR_V8_STACK_START(ScrV8GetStackStart, "_ZN2v84base5Stack13GetStackStartEv") +SCR_V8_STACK_START(ScrV8GetStackStartUnchecked, "_ZN2v84base5Stack22GetStackStartUncheckedEv") +SCR_V8_STACK_START(ScrV8ObtainCurrentThreadStackStart, + "_ZN2v84base5Stack29ObtainCurrentThreadStackStartEv") +#undef SCR_V8_STACK_START +#endif + extern "C" __attribute__((visibility("default"))) const ScrNodeBridge *scr_node_bridge_v1() { return &kBridge; } diff --git a/packages/libnode-common/scripts/smoke.mjs b/packages/libnode-common/scripts/smoke.mjs index 578a926a0..caf2d7725 100644 --- a/packages/libnode-common/scripts/smoke.mjs +++ b/packages/libnode-common/scripts/smoke.mjs @@ -84,10 +84,13 @@ if (process.argv[1] === fileURLToPath(import.meta.url)) { const { values } = parseArgs({ options: { pack: { type: "string" }, platform: { type: "string" } }, }); + // An explicit --pack or --platform names the pack; SCRIPTC_LIBNODE only + // replaces the host's default. const pack = resolve( values.pack ?? - process.env.SCRIPTC_LIBNODE ?? - join(repoRoot, "packages", `libnode-${values.platform ?? hostPlatform()}`), + (values.platform !== undefined + ? join(repoRoot, "packages", `libnode-${values.platform}`) + : (process.env.SCRIPTC_LIBNODE ?? join(repoRoot, "packages", `libnode-${hostPlatform()}`))), ); const manifest = JSON.parse(readFileSync(join(pack, "manifest.json"), "utf8")); const { status, stdout, stderr } = runSmoke(pack); diff --git a/packages/libnode-common/test/bridge.test.ts b/packages/libnode-common/test/bridge.test.ts index 1ab6ab95b..4c37b31a3 100644 --- a/packages/libnode-common/test/bridge.test.ts +++ b/packages/libnode-common/test/bridge.test.ts @@ -45,7 +45,7 @@ exports.quit = (code) => process.exit(code); expect(result.stderr).toBe(""); expect(result.stdout).toBe( [ - "abi 2 ok 1", + "abi 3 ok 1", "sync add -> R5", 'sync fails -> T{"error":{"kind":2,"message":"out of range","code":"ERR_X"}}', `sync missing -> T{"error":{"kind":1,"message":"'missing' exported by './smoke.cjs' is not a function"}}`, diff --git a/packages/libnode-linux-arm64-gnu/package.json b/packages/libnode-linux-arm64-gnu/package.json index 2b6e9b66f..6ac8aa7c3 100644 --- a/packages/libnode-linux-arm64-gnu/package.json +++ b/packages/libnode-linux-arm64-gnu/package.json @@ -16,6 +16,9 @@ "cpu": [ "arm64" ], + "libc": [ + "glibc" + ], "files": [ "libnode.so.137", "libscr_node_bridge.so", diff --git a/packages/libnode-linux-x64-gnu/package.json b/packages/libnode-linux-x64-gnu/package.json index 05e36bd1e..e47e01f93 100644 --- a/packages/libnode-linux-x64-gnu/package.json +++ b/packages/libnode-linux-x64-gnu/package.json @@ -16,6 +16,9 @@ "cpu": [ "x64" ], + "libc": [ + "glibc" + ], "files": [ "libnode.so.137", "libscr_node_bridge.so", diff --git a/packages/node/README.md b/packages/node/README.md index 77002db49..7b1c3656d 100644 --- a/packages/node/README.md +++ b/packages/node/README.md @@ -10,6 +10,8 @@ const config = await call<{ regions: string[] }>("config-parser", "parse", sourc ``` - The module loads with `require()` from the calling module: relative specifiers resolve from its directory, bare ones from its `node_modules`. +- In scriptc builds the specifier is a string literal. The build collects the module into the executable: a module of the program with the modules its string-literal `require()` and `import` reach, or a whole package with its dependencies. The executable loads them from its own copy, never from the build machine's files. A module of the program that loads others through computed specifiers is refused (SC8006); a module the build cannot find throws `MODULE_NOT_FOUND` when the call runs. +- The executable extracts its modules into `$XDG_CACHE_HOME/scriptc/node-islands/` (or `~/.cache/scriptc/node-islands/`) and checks every file against the build's digests on each start. With `--libnode=beside` they are written next to the executable instead, in `.libnode/modules/`. - Arguments and results cross as JSON copies; scriptc checks the result against the declared type. - A thrown `Error` crosses as its nearest intrinsic error class with its `message` and string `code`. - The executable loads Node from the libnode pack recorded at build time (`SCRIPTC_LIBNODE` or `scriptc build --libnode=`; `--libnode=beside` copies the pack next to the executable). `SCRIPTC_LIBNODE` overrides it at run time. A program that never calls into Node never loads it. diff --git a/packages/node/index.d.ts b/packages/node/index.d.ts index 10642b098..87a8eb541 100644 --- a/packages/node/index.d.ts +++ b/packages/node/index.d.ts @@ -9,6 +9,8 @@ * - the module loads with require() from the calling module, so a relative * specifier resolves from the caller's directory and a bare one from its * node_modules (ES modules load through require(esm)); + * - in scriptc builds `specifier` is a string literal: the build ships the + * module, and what it loads, inside the executable; * - the arguments cross as a copy, `JSON.parse(JSON.stringify(args))`, and * the function runs with the module namespace as `this`; * - the result crosses as its JSON copy (undefined stays undefined), and diff --git a/packages/runtime-darwin-arm64/scripts/build.mjs b/packages/runtime-darwin-arm64/scripts/build.mjs index 069cb66e9..e492c36b4 100644 --- a/packages/runtime-darwin-arm64/scripts/build.mjs +++ b/packages/runtime-darwin-arm64/scripts/build.mjs @@ -340,7 +340,7 @@ async function build() { package: packageManifest.name, version: packageManifest.version, target: RUNTIME_PACK_MATRIX.target, - runtime_abi: { version: 36, marker: "scr_runtime_abi_v36" }, + runtime_abi: { version: 37, marker: "scr_runtime_abi_v37" }, compiler: { command: compiler, identity: compilerVersion, diff --git a/packages/runtime-pack-common/scripts/build.mjs b/packages/runtime-pack-common/scripts/build.mjs index 54563ff64..b31ebe819 100644 --- a/packages/runtime-pack-common/scripts/build.mjs +++ b/packages/runtime-pack-common/scripts/build.mjs @@ -437,7 +437,7 @@ async function build() { package: packageManifest.name, version: packageManifest.version, target: matrix.target, - runtime_abi: { version: 36, marker: "scr_runtime_abi_v36" }, + runtime_abi: { version: 37, marker: "scr_runtime_abi_v37" }, compiler: { command: compiler, identity: compilerVersion, diff --git a/packages/runtime/src/scr_console.c b/packages/runtime/src/scr_console.c index 6def8d866..086340014 100644 --- a/packages/runtime/src/scr_console.c +++ b/packages/runtime/src/scr_console.c @@ -148,7 +148,7 @@ static void scr_enable_utf8_console_output(void) { } #endif -void scr_runtime_abi_v36(void) {} +void scr_runtime_abi_v37(void) {} void scr_init(void) { #ifdef SCR_WORKERS diff --git a/packages/runtime/src/scr_core.h b/packages/runtime/src/scr_core.h index f9cc8cc5f..3707b8d50 100644 --- a/packages/runtime/src/scr_core.h +++ b/packages/runtime/src/scr_core.h @@ -81,7 +81,7 @@ void scr_init(void); /* Program objects emitted by the bundled LLVM helper reference this symbol. * Its versioned spelling makes a mismatched manual runtime link fail before * the program can start. */ -void scr_runtime_abi_v36(void); +void scr_runtime_abi_v37(void); void scr_runtime_workers_v8(void); /* ── the trap funnel (scr_console.c; scr_library.c under -DSCR_LIB) ────── diff --git a/packages/runtime/src/scr_dyn_builtin_stream.c b/packages/runtime/src/scr_dyn_builtin_stream.c index 926f85eea..2875cfa06 100644 --- a/packages/runtime/src/scr_dyn_builtin_stream.c +++ b/packages/runtime/src/scr_dyn_builtin_stream.c @@ -34,7 +34,7 @@ * receiver without a reference; the hook closures outlive the record. * * Not supported here (each throws the "not supported yet" Error where it - * runs): the options scr_stream.c's dyn option walk refuses, `_writev` and `_construct`, symbol event names, listeners() / + * runs): the options scr_stream.c's dyn option walk refuses, symbol event names, listeners() / * rawListeners(), `wrap()`, and pipes into values that are neither runtime * streams nor constructor-function streams. */ #include "scr_runtime.h" @@ -898,20 +898,15 @@ static ScrDyn *bcs_construct(int kind, ScrDyn *self, ScrDyn *const *args, size_t } bool failed = false; bool writes = kind != BCS_READABLE; - bool transformish = kind == BCS_TRANSFORM || kind == BCS_PASSTHROUGH; ScrDyn *opts = bcs_arg(args, argc, 0); /* Node's constructors assign options.writev and options.construct to * the instance (this._writev = options.writev), then read _writev and * _construct, so an option shadows the prototype's method. */ if (opts->kind == SCR_DYN_OBJ) { - if (writes && !transformish && !bcs_copy_option(self, opts, "writev", "_writev")) return NULL; + if (writes && !bcs_copy_option(self, opts, "writev", "_writev")) return NULL; if (!bcs_copy_option(self, opts, "construct", "_construct")) return NULL; } bool has_writev = writes && bcs_is_function(self, "_writev", &failed); - if (has_writev && transformish) { - bcs_unsupported("_writev on a constructor-function Transform is not supported yet"); - return NULL; - } bool has_construct = !failed && bcs_is_function(self, "_construct", &failed); if (failed) return NULL; ScrStream *s = kind == BCS_READABLE ? scr_stream_new_readable_dyn(opts) diff --git a/packages/runtime/src/scr_fetch.c b/packages/runtime/src/scr_fetch.c index 6049a4936..aebe62659 100644 --- a/packages/runtime/src/scr_fetch.c +++ b/packages/runtime/src/scr_fetch.c @@ -3358,6 +3358,10 @@ static bool sf_header_value_ok(const ScrStr *value) { } static ScrStr *sf_header_value_bytestring(ScrStr *value) { + /* An ASCII value is its own ByteString: each byte is one UTF-16 unit. */ + size_t ascii = 0; + while (ascii < value->len && (unsigned char)value->data[ascii] < 0x80) ascii++; + if (ascii == value->len) return scr_str_retain(value); size_t len = (size_t)scr_str_utf16_len(value); char *bytes = malloc(len ? len : 1); if (!bytes) sf_oom(); diff --git a/packages/runtime/src/scr_json.c b/packages/runtime/src/scr_json.c index 14d0a901f..da18a1c0b 100644 --- a/packages/runtime/src/scr_json.c +++ b/packages/runtime/src/scr_json.c @@ -13898,19 +13898,74 @@ ScrDyn *scr_dyn_object_rest(const ScrDyn *source, const ScrDyn *excluded) { /* Compiler-owned native class views retain every property and its flags, * including nonenumerable data, without invoking accessors. */ -ScrDyn *scr_dyn_copy_property_descriptors(ScrDyn *target, const ScrDyn *src) { - if (!src || src->kind == SCR_DYN_UNDEF) return scr_dyn_retain(target); - ScrDyn *keys = scr_dyn_obj_own_keys(src); - if (!keys) return NULL; - for (size_t i = 0; i < keys->v.arr.len; i++) { - ScrDyn *key = keys->v.arr.items[i]; - ScrDyn *descriptor = scr_dyn_own_descriptor(src, key->v.str); +static void scr_dyn_copy_property_descriptor(ScrDyn *target, const ScrDyn *src, + const ScrDynEntry *from) { + /* Exactly scr_dyn_define_property(target, key, own descriptor of key) + * when `target` has no such property and accepts new ones: the entry + * takes the source's attributes, value or accessor pair. */ + ScrDynEntry *prior = scr_dyn_find_entry(target, from->key, from->key_len); + if (prior || target->non_extensible) { + ScrStr *name = scr_str_new(from->key, from->key_len); + ScrDyn *key = scr_dyn_new_str(name); + ScrDyn *descriptor = scr_dyn_own_descriptor(src, name); ScrDyn *result = descriptor ? scr_dyn_define_property(target, key, descriptor) : NULL; scr_dyn_release(descriptor); scr_dyn_release(result); - if (scr_exc_pending()) break; + scr_dyn_release(key); + scr_str_release(name); + return; + } + /* The names whose definition may move into a native Error's slots + * (scr_error_sync_slots). */ + static const char *const slots[] = {"cause", "stack", "name", "message", "code"}; + bool slot = false; + for (size_t i = 0; i < sizeof slots / sizeof *slots && !slot; i++) + slot = from->key_len == strlen(slots[i]) && memcmp(from->key, slots[i], from->key_len) == 0; + bool accessor = from->accessor, writable = !accessor && from->writable; + bool enumerable = from->enumerable, configurable = from->configurable; + ScrDyn *value = scr_dyn_retain(accessor ? scr_dyn_undefined() : from->value); + ScrDyn *getter = accessor && from->getter ? scr_dyn_retain(from->getter) : NULL; + ScrDyn *setter = accessor && from->setter ? scr_dyn_retain(from->setter) : NULL; + ScrStr *name = slot ? scr_str_new(from->key, from->key_len) : NULL; + scr_dyn_obj_set(target, from->key, from->key_len, value); + ScrDynEntry *entry = &target->v.obj.entries[target->v.obj.len - 1]; + entry->accessor = accessor; + entry->getter = getter; + entry->setter = setter; + entry->writable = writable; + entry->enumerable = enumerable; + entry->configurable = configurable; + if (name) { + scr_error_sync_cause(target, name); + scr_str_release(name); + } +} + +ScrDyn *scr_dyn_copy_property_descriptors(ScrDyn *target, const ScrDyn *src) { + if (!src || src->kind == SCR_DYN_UNDEF) return scr_dyn_retain(target); + if (src->kind == SCR_DYN_OBJ && target->kind == SCR_DYN_OBJ && src != target && + src != scr_builtin_object_prototype && target != scr_builtin_error_prototype) { + /* Plain objects on both sides: no fence, proxy trap or accessor runs, + * so each property is copied from its entry without building the key + * and descriptor objects the general path goes through. */ + if (src->v.obj.len) scr_dyn_region_store(target, NULL); + size_t *order = scr_dyn_obj_key_order(src); + for (size_t i = 0; i < src->v.obj.len && !scr_exc_pending(); i++) + scr_dyn_copy_property_descriptor(target, src, &src->v.obj.entries[order ? order[i] : i]); + free(order); + } else { + ScrDyn *keys = scr_dyn_obj_own_keys(src); + if (!keys) return NULL; + for (size_t i = 0; i < keys->v.arr.len; i++) { + ScrDyn *key = keys->v.arr.items[i]; + ScrDyn *descriptor = scr_dyn_own_descriptor(src, key->v.str); + ScrDyn *result = descriptor ? scr_dyn_define_property(target, key, descriptor) : NULL; + scr_dyn_release(descriptor); + scr_dyn_release(result); + if (scr_exc_pending()) break; + } + scr_dyn_release(keys); } - scr_dyn_release(keys); if (!scr_exc_pending()) { ScrDyn *symbols = scr_dyn_get_own_property_symbols(src); if (!symbols) return NULL; diff --git a/packages/runtime/src/scr_node_bridge.h b/packages/runtime/src/scr_node_bridge.h index 8ccdeaf8d..f88764691 100644 --- a/packages/runtime/src/scr_node_bridge.h +++ b/packages/runtime/src/scr_node_bridge.h @@ -16,7 +16,7 @@ #include #include -#define SCR_NODE_BRIDGE_ABI 2u +#define SCR_NODE_BRIDGE_ABI 3u /* The symbol a bridge library exports: `const ScrNodeBridge *(void)`. */ #define SCR_NODE_BRIDGE_ENTRY "scr_node_bridge_v1" @@ -27,6 +27,12 @@ typedef struct ScrNodeBridgeHost { /* Node ends the process (process.exit, an uncaught exception): the * runtime runs its exit sequence with this code and never returns. */ void (*exit)(int code); + /* The start (highest address) of the stack the caller runs on when the + * runtime provides it (Node's own stack), 0 for a thread's own stack. + * V8 scans the stack conservatively from the current position up to the + * start it records for the thread, so it must not record the thread's + * when JavaScript runs on the runtime's stack. */ + uintptr_t (*stack_start)(void); } ScrNodeBridgeHost; typedef struct ScrNodeBridge { @@ -58,6 +64,10 @@ typedef struct ScrNodeBridge { void (*pump)(void); /* Tears Node down at process exit. */ void (*stop)(void); + /* The program ends with `code`, before stop: Node's 'beforeExit' and + * 'exit' run, and the result is the code Node's process ends with + * (process.exitCode in Node, else `code`). */ + int (*finish)(int code); } ScrNodeBridge; #endif /* SCR_NODE_BRIDGE_H */ diff --git a/packages/runtime/src/scr_node_island.c b/packages/runtime/src/scr_node_island.c index 69aa6b225..e32f8da41 100644 --- a/packages/runtime/src/scr_node_island.c +++ b/packages/runtime/src/scr_node_island.c @@ -6,13 +6,15 @@ * dlopens the bridge library of a libnode pack (scr_node_bridge.h), which * links libnode, so a run that never calls into Node never maps it. * - * Locating Node. The pack directory is SCRIPTC_LIBNODE when set, else the - * path recorded at build time; a recorded path that starts with `$ORIGIN/` - * resolves against the directory of the running executable (the `beside` - * packaging, and the hook for shipping Node with the program). No search - * path is used. The bridge must speak SCR_NODE_BRIDGE_ABI and link the - * Node version recorded at build time; anything else is a fatal error at - * first use, never a silent use of another Node. + * Locating Node. The pack directory is SCRIPTC_LIBNODE when set (an + * absolute path), else the path recorded at build time; a recorded path + * that starts with `$ORIGIN/` resolves against the directory of the + * running executable (the `beside` packaging, and the hook for shipping + * Node with the program). A process with elevated privileges ignores + * SCRIPTC_LIBNODE and refuses `$ORIGIN/`. No search path is used. The + * bridge must speak SCR_NODE_BRIDGE_ABI and link the Node version recorded + * at build time; anything else is a fatal error at first use, never a + * silent use of another Node. * * Calls. A call serializes its arguments as JSON, enters JavaScript through * the bridge (a callback scope, so ticks and microtasks drain when it @@ -43,9 +45,17 @@ #if !defined(_WIN32) && !defined(__wasi__) #include "scr_node_island_host.h" +#include #include +#include +#include #include +#include #include +#include +#if defined(__linux__) +#include +#endif #ifdef __APPLE__ #define NI_BRIDGE_FILE "libscr_node_bridge.dylib" @@ -86,6 +96,15 @@ void scr_node_island_install(const char *libnode, const char *node_version) { SCR_COLD static void ni_oom(void) { scr_trap("scriptc: out of memory\n"); } +/* `len` bytes as a NUL-terminated malloc'd copy. */ +static char *ni_cstr_of(const char *bytes, size_t len) { + char *copy = malloc(len + 1); + if (copy == NULL) ni_oom(); + memcpy(copy, bytes, len); + copy[len] = '\0'; + return copy; +} + SCR_COLD static void ni_fatal(const char *message) { fflush(stdout); fputs(message, stderr); @@ -93,27 +112,600 @@ SCR_COLD static void ni_fatal(const char *message) { _Exit(1); } +/* True when the process runs with elevated privileges (setuid, setgid or + * file capabilities). The pack directory then comes only from an absolute + * path recorded at build time, as the dynamic linker ignores + * LD_LIBRARY_PATH and $ORIGIN for such processes. */ +static bool ni_secure(void) { +#if defined(__APPLE__) + return issetugid() != 0; +#elif defined(__linux__) + return getauxval(AT_SECURE) != 0; +#else + return false; +#endif +} + /* The pack directory: SCRIPTC_LIBNODE, else the recorded path with a - * leading `$ORIGIN/` resolved against the executable's own directory. */ + * leading `$ORIGIN/` resolved against the executable's own directory. + * Both must resolve to absolute paths. */ static const char *ni_pack_dir(void) { - const char *override = getenv("SCRIPTC_LIBNODE"); + bool secure = ni_secure(); + const char *override = secure ? NULL : getenv("SCRIPTC_LIBNODE"); + if (override != NULL && override[0] != '\0' && override[0] != '/') + ni_fatal("scriptc: SCRIPTC_LIBNODE must be an absolute path to a libnode pack directory\n"); const char *dir = override != NULL && override[0] != '\0' ? override : ni_libnode; if (dir == NULL || dir[0] == '\0') ni_fatal("scriptc: this program calls into Node.js (@scriptc/node), but no libnode pack was " "recorded at build time; set SCRIPTC_LIBNODE to a pack directory\n"); static const char origin[] = "$ORIGIN/"; + int written; if (dir != override && strncmp(dir, origin, sizeof origin - 1) == 0) { + if (secure) + ni_fatal("scriptc: a program running with elevated privileges cannot load a libnode pack " + "beside its executable; build it with an absolute SCRIPTC_LIBNODE\n"); ScrStr *self = scr_process_exec_path(); size_t len = self->len; while (len > 0 && self->data[len - 1] != '/') len--; - snprintf(ni_dir, sizeof ni_dir, "%.*s%s", (int)len, self->data, dir + sizeof origin - 1); + if (len == 0 || self->data[0] != '/') { + scr_str_release(self); + ni_fatal("scriptc: cannot locate this program's executable to find its libnode pack\n"); + } + written = snprintf(ni_dir, sizeof ni_dir, "%.*s%s", (int)len, self->data, + dir + sizeof origin - 1); scr_str_release(self); } else { - snprintf(ni_dir, sizeof ni_dir, "%s", dir); + written = snprintf(ni_dir, sizeof ni_dir, "%s", dir); } + if (written < 0 || (size_t)written >= sizeof ni_dir) + ni_fatal("scriptc: the libnode pack path is too long\n"); return ni_dir; } +/* ── the island module tree ───────────────────────────────────────────── */ + +/* The JavaScript the program's calls load (node-islands/modules.ts), named + * by a manifest of relative paths with their sizes and SHA-256 digests, plus + * the links between package directories. Before Node loads anything, the + * tree is made available at one directory, and every file is checked against + * the manifest: + * - beside the executable (`--libnode=beside`), at the recorded `$ORIGIN/` + * path, checked on every start; + * - otherwise from the executable itself, extracted once into + * $XDG_CACHE_HOME/scriptc/node-islands// (or + * ~/.cache/...), checked on every start; when that directory cannot be + * used, into a private directory under $TMPDIR removed at exit. + * Directories are created 0700 and owned by the process's user; no link is + * followed below the cache directory. A process with elevated privileges + * refuses the tree, as it refuses SCRIPTC_LIBNODE. A call names its module + * by its path relative to the tree. */ +static const char *ni_manifest = NULL; +static size_t ni_manifest_len = 0; +static const unsigned char *ni_modules = NULL; +static size_t ni_modules_len = 0; +static const char *ni_modules_location = NULL; +/* The tree's directory at run time; "/" when the calls load only Node's + * built-in modules. */ +static char ni_root[4096] = "/"; + +void scr_node_island_install_modules(const char *manifest, size_t manifest_len, + const unsigned char *data, size_t data_len, + const char *location) { + ni_manifest = manifest; + ni_manifest_len = manifest_len; + ni_modules = data; + ni_modules_len = data_len; + ni_modules_location = location; +} + +/* SHA-256 (FIPS 180-4), for the manifest's digests. */ +typedef struct { + uint32_t h[8]; + uint64_t bytes; + unsigned char tail[64]; + size_t tail_len; +} NiSha256; + +static const uint32_t ni_sha256_k[64] = { + 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5, + 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, + 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, + 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, + 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, + 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, + 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3, + 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2}; + +static uint32_t ni_rotr(uint32_t x, unsigned n) { return (x >> n) | (x << (32 - n)); } + +static void ni_sha256_block(uint32_t h[8], const unsigned char *p) { + uint32_t w[64]; + for (int i = 0; i < 16; i++) + w[i] = ((uint32_t)p[i * 4] << 24) | ((uint32_t)p[i * 4 + 1] << 16) | + ((uint32_t)p[i * 4 + 2] << 8) | (uint32_t)p[i * 4 + 3]; + for (int i = 16; i < 64; i++) { + uint32_t s0 = ni_rotr(w[i - 15], 7) ^ ni_rotr(w[i - 15], 18) ^ (w[i - 15] >> 3); + uint32_t s1 = ni_rotr(w[i - 2], 17) ^ ni_rotr(w[i - 2], 19) ^ (w[i - 2] >> 10); + w[i] = w[i - 16] + s0 + w[i - 7] + s1; + } + uint32_t a = h[0], b = h[1], c = h[2], d = h[3], e = h[4], f = h[5], g = h[6], k = h[7]; + for (int i = 0; i < 64; i++) { + uint32_t t1 = k + (ni_rotr(e, 6) ^ ni_rotr(e, 11) ^ ni_rotr(e, 25)) + ((e & f) ^ (~e & g)) + + ni_sha256_k[i] + w[i]; + uint32_t t2 = (ni_rotr(a, 2) ^ ni_rotr(a, 13) ^ ni_rotr(a, 22)) + ((a & b) ^ (a & c) ^ (b & c)); + k = g; + g = f; + f = e; + e = d + t1; + d = c; + c = b; + b = a; + a = t1 + t2; + } + h[0] += a; + h[1] += b; + h[2] += c; + h[3] += d; + h[4] += e; + h[5] += f; + h[6] += g; + h[7] += k; +} + +static void ni_sha256_init(NiSha256 *s) { + static const uint32_t initial[8] = {0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, + 0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19}; + memcpy(s->h, initial, sizeof initial); + s->bytes = 0; + s->tail_len = 0; +} + +static void ni_sha256_update(NiSha256 *s, const unsigned char *data, size_t len) { + s->bytes += len; + while (len > 0) { + if (s->tail_len == 0 && len >= 64) { + ni_sha256_block(s->h, data); + data += 64; + len -= 64; + continue; + } + size_t take = 64 - s->tail_len < len ? 64 - s->tail_len : len; + memcpy(s->tail + s->tail_len, data, take); + s->tail_len += take; + data += take; + len -= take; + if (s->tail_len == 64) { + ni_sha256_block(s->h, s->tail); + s->tail_len = 0; + } + } +} + +static void ni_sha256_final(NiSha256 *s, unsigned char out[32]) { + uint64_t bits = s->bytes * 8; + unsigned char pad[72] = {0x80}; + size_t padding = (s->tail_len < 56 ? 56 : 120) - s->tail_len; + for (int i = 0; i < 8; i++) pad[padding + i] = (unsigned char)(bits >> (56 - 8 * i)); + ni_sha256_update(s, pad, padding + 8); + for (int i = 0; i < 8; i++) + for (int b = 0; b < 4; b++) out[i * 4 + b] = (unsigned char)(s->h[i] >> (24 - 8 * b)); +} + +typedef struct { + char *path; + size_t size; + unsigned char sha256[32]; + size_t offset; /* into ni_modules */ +} NiFile; + +typedef struct { + char *path, *target; +} NiLink; + +static NiFile *ni_files = NULL; +static size_t ni_nfiles = 0; +static NiLink *ni_links = NULL; +static size_t ni_nlinks = 0; + +SCR_COLD static void ni_bad_manifest(void) { + ni_fatal("scriptc: this program's Node island module manifest is malformed\n"); +} + +static char *ni_dup_str(const ScrDyn *d) { + if (d == NULL || d->kind != SCR_DYN_STR) ni_bad_manifest(); + if (memchr(d->v.str->data, '\0', d->v.str->len) != NULL) ni_bad_manifest(); + return ni_cstr_of(d->v.str->data, d->v.str->len); +} + +/* A relative path inside the tree: no leading `/`, no empty, `.` or `..` + * component. */ +static bool ni_relative_inside(const char *path) { + if (path[0] == '\0' || path[0] == '/') return false; + for (const char *p = path;;) { + const char *slash = strchr(p, '/'); + size_t n = slash != NULL ? (size_t)(slash - p) : strlen(p); + if (n == 0 || (n == 1 && p[0] == '.') || (n == 2 && p[0] == '.' && p[1] == '.')) return false; + if (slash == NULL) return true; + p = slash + 1; + } +} + +/* A link target that, read from the link's directory, stays inside the + * tree. */ +static bool ni_link_inside(const char *link, const char *target) { + if (target[0] == '\0' || target[0] == '/') return false; + long depth = 0; + for (const char *p = link; (p = strchr(p, '/')) != NULL; p++) depth++; + for (const char *p = target;;) { + const char *slash = strchr(p, '/'); + size_t n = slash != NULL ? (size_t)(slash - p) : strlen(p); + if (n == 2 && p[0] == '.' && p[1] == '.') { + if (--depth < 0) return false; + } else if (n == 0 || !(n == 1 && p[0] == '.')) { + if (n == 0) return false; + depth++; + } + if (slash == NULL) return true; + p = slash + 1; + } +} + +static int ni_hex(char c) { + return c >= '0' && c <= '9' ? c - '0' : c >= 'a' && c <= 'f' ? c - 'a' + 10 : -1; +} + +static void ni_parse_manifest(void) { + ScrStr *text = scr_str_new(ni_manifest, ni_manifest_len); + ScrDyn *body = scr_json_parse(text); + scr_str_release(text); + if (body == NULL || body->kind != SCR_DYN_OBJ) ni_bad_manifest(); + ScrDyn *files = scr_dyn_obj_get(body, "files", 5); + ScrDyn *links = scr_dyn_obj_get(body, "links", 5); + if (files == NULL || files->kind != SCR_DYN_ARR || links == NULL || links->kind != SCR_DYN_ARR) + ni_bad_manifest(); + ni_nfiles = (size_t)scr_dyn_arr_len(files); + ni_nlinks = (size_t)scr_dyn_arr_len(links); + ni_files = calloc(ni_nfiles + 1, sizeof *ni_files); + ni_links = calloc(ni_nlinks + 1, sizeof *ni_links); + if (ni_files == NULL || ni_links == NULL) ni_oom(); + size_t offset = 0; + for (size_t i = 0; i < ni_nfiles; i++) { + ScrDyn *entry = scr_dyn_arr_at(files, (double)i); + if (entry->kind != SCR_DYN_ARR || scr_dyn_arr_len(entry) != 3) ni_bad_manifest(); + ScrDyn *path = scr_dyn_arr_at(entry, 0), *size = scr_dyn_arr_at(entry, 1), + *digest = scr_dyn_arr_at(entry, 2); + NiFile *f = &ni_files[i]; + f->path = ni_dup_str(path); + if (!ni_relative_inside(f->path) || size->kind != SCR_DYN_NUM || size->v.num < 0 || + digest->kind != SCR_DYN_STR || digest->v.str->len != 64) + ni_bad_manifest(); + f->size = (size_t)size->v.num; + for (int b = 0; b < 32; b++) { + int hi = ni_hex(digest->v.str->data[b * 2]), lo = ni_hex(digest->v.str->data[b * 2 + 1]); + if (hi < 0 || lo < 0) ni_bad_manifest(); + f->sha256[b] = (unsigned char)(hi * 16 + lo); + } + f->offset = offset; + offset += f->size; + scr_dyn_release(path); + scr_dyn_release(size); + scr_dyn_release(digest); + scr_dyn_release(entry); + } + if (ni_modules != NULL && offset != ni_modules_len) ni_bad_manifest(); + for (size_t i = 0; i < ni_nlinks; i++) { + ScrDyn *entry = scr_dyn_arr_at(links, (double)i); + if (entry->kind != SCR_DYN_ARR || scr_dyn_arr_len(entry) != 2) ni_bad_manifest(); + ScrDyn *path = scr_dyn_arr_at(entry, 0), *target = scr_dyn_arr_at(entry, 1); + ni_links[i].path = ni_dup_str(path); + ni_links[i].target = ni_dup_str(target); + if (!ni_relative_inside(ni_links[i].path) || + !ni_link_inside(ni_links[i].path, ni_links[i].target)) + ni_bad_manifest(); + scr_dyn_release(path); + scr_dyn_release(target); + scr_dyn_release(entry); + } + scr_dyn_release(body); +} + +/* The directory `name` in `dirfd`, never through a link; created 0700 when + * `create` and absent. -1 when it cannot be opened. */ +static int ni_dir_at(int dirfd, const char *name, bool create) { + if (create && mkdirat(dirfd, name, 0700) != 0 && errno != EEXIST) return -1; + return openat(dirfd, name, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC); +} + +/* The parent directory of the relative `path` below `rootfd` (its last + * component is left in *base), opened component by component without + * following links; created when `create`. -1 when it cannot be opened. */ +static int ni_parent_at(int rootfd, const char *path, bool create, const char **base) { + int fd = dup(rootfd); + const char *p = path; + for (const char *slash; fd >= 0 && (slash = strchr(p, '/')) != NULL; p = slash + 1) { + char component[256]; + size_t n = (size_t)(slash - p); + if (n >= sizeof component) { + close(fd); + return -1; + } + memcpy(component, p, n); + component[n] = '\0'; + int next = ni_dir_at(fd, component, create); + close(fd); + fd = next; + } + *base = p; + return fd; +} + +/* A directory this process may trust: owned by its user, writable by no + * one else. */ +static bool ni_private_dir(int fd) { + struct stat st; + return fd >= 0 && fstat(fd, &st) == 0 && S_ISDIR(st.st_mode) && st.st_uid == geteuid() && + (st.st_mode & 022) == 0; +} + +static bool ni_digest_matches(const unsigned char *data, size_t len, + const unsigned char expected[32]) { + NiSha256 s; + unsigned char got[32]; + ni_sha256_init(&s); + ni_sha256_update(&s, data, len); + ni_sha256_final(&s, got); + return memcmp(got, expected, 32) == 0; +} + +/* Whether the tree below `rootfd` holds exactly the manifest's files (by + * size and digest) and links. */ +static bool ni_tree_matches(int rootfd) { + for (size_t i = 0; i < ni_nfiles; i++) { + const char *base; + int dirfd = ni_parent_at(rootfd, ni_files[i].path, false, &base); + if (dirfd < 0) return false; + int fd = openat(dirfd, base, O_RDONLY | O_NOFOLLOW | O_CLOEXEC); + close(dirfd); + struct stat st; + if (fd < 0 || fstat(fd, &st) != 0 || !S_ISREG(st.st_mode) || + (size_t)st.st_size != ni_files[i].size) { + if (fd >= 0) close(fd); + return false; + } + NiSha256 s; + ni_sha256_init(&s); + unsigned char buffer[65536]; + size_t total = 0; + for (;;) { + ssize_t n = read(fd, buffer, sizeof buffer); + if (n < 0 && errno == EINTR) continue; + if (n <= 0) break; + ni_sha256_update(&s, buffer, (size_t)n); + total += (size_t)n; + } + close(fd); + unsigned char got[32]; + ni_sha256_final(&s, got); + if (total != ni_files[i].size || memcmp(got, ni_files[i].sha256, 32) != 0) return false; + } + for (size_t i = 0; i < ni_nlinks; i++) { + const char *base; + int dirfd = ni_parent_at(rootfd, ni_links[i].path, false, &base); + if (dirfd < 0) return false; + char target[4096]; + ssize_t n = readlinkat(dirfd, base, target, sizeof target - 1); + close(dirfd); + if (n < 0) return false; + target[n] = '\0'; + if (strcmp(target, ni_links[i].target) != 0) return false; + } + return true; +} + +/* Writes the executable's tree below `rootfd` (an empty directory). */ +static bool ni_extract(int rootfd) { + for (size_t i = 0; i < ni_nfiles; i++) { + const unsigned char *data = ni_modules + ni_files[i].offset; + if (!ni_digest_matches(data, ni_files[i].size, ni_files[i].sha256)) return false; + const char *base; + int dirfd = ni_parent_at(rootfd, ni_files[i].path, true, &base); + if (dirfd < 0) return false; + int fd = openat(dirfd, base, O_WRONLY | O_CREAT | O_EXCL | O_NOFOLLOW | O_CLOEXEC, 0600); + close(dirfd); + if (fd < 0) return false; + size_t done = 0; + while (done < ni_files[i].size) { + ssize_t n = write(fd, data + done, ni_files[i].size - done); + if (n < 0 && errno == EINTR) continue; + if (n <= 0) break; + done += (size_t)n; + } + if (close(fd) != 0 || done != ni_files[i].size) return false; + } + for (size_t i = 0; i < ni_nlinks; i++) { + const char *base; + int dirfd = ni_parent_at(rootfd, ni_links[i].path, true, &base); + if (dirfd < 0) return false; + int status = symlinkat(ni_links[i].target, dirfd, base); + close(dirfd); + if (status != 0) return false; + } + return true; +} + +/* Removes everything below `dirfd` and closes it, never following links. */ +static void ni_remove_below(int dirfd) { + DIR *dir = fdopendir(dirfd); + if (dir == NULL) { + close(dirfd); + return; + } + for (struct dirent *e; (e = readdir(dir)) != NULL;) { + if (strcmp(e->d_name, ".") == 0 || strcmp(e->d_name, "..") == 0) continue; + int sub = openat(dirfd, e->d_name, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC); + if (sub >= 0) { + ni_remove_below(sub); + (void)unlinkat(dirfd, e->d_name, AT_REMOVEDIR); + } else { + (void)unlinkat(dirfd, e->d_name, 0); + } + } + closedir(dir); +} + + +static char ni_temp_root[4096] = ""; + +static void ni_remove_temp_root(void) { + int fd = open(ni_temp_root, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC); + if (fd >= 0) ni_remove_below(fd); + (void)rmdir(ni_temp_root); +} + +static bool ni_set_root(const char *a, const char *b, const char *c) { + int n = snprintf(ni_root, sizeof ni_root, "%s%s%s", a, b, c); + return n > 0 && (size_t)n < sizeof ni_root; +} + +/* The tree in the cache directory, extracted there when absent. */ +static bool ni_cached_root(const char *digest) { + const char *xdg = getenv("XDG_CACHE_HOME"); + const char *home = getenv("HOME"); + char base[4096]; + int n; + if (xdg != NULL && xdg[0] == '/') + n = snprintf(base, sizeof base, "%s", xdg); + else if (home != NULL && home[0] == '/') + n = snprintf(base, sizeof base, "%s/.cache", home); + else + return false; + if (n <= 0 || (size_t)n >= sizeof base) return false; + (void)mkdir(base, 0700); + int basefd = open(base, O_RDONLY | O_DIRECTORY | O_CLOEXEC); + if (basefd < 0) return false; + int scriptc = ni_dir_at(basefd, "scriptc", true); + close(basefd); + int islands = ni_private_dir(scriptc) ? ni_dir_at(scriptc, "node-islands", true) : -1; + if (scriptc >= 0) close(scriptc); + if (!ni_private_dir(islands)) { + if (islands >= 0) close(islands); + return false; + } + bool ok = false; + int tree = ni_dir_at(islands, digest, false); + if (tree < 0 && errno == ENOENT) { + /* Extract into a fresh sibling, then move it into place. */ + char staging[128]; + snprintf(staging, sizeof staging, "%s.%ld.tmp", digest, (long)getpid()); + int fd = ni_dir_at(islands, staging, true); + if (ni_private_dir(fd) && ni_extract(fd) && ni_tree_matches(fd) && + renameat(islands, staging, islands, digest) == 0) { + close(fd); + tree = ni_dir_at(islands, digest, false); + } else { + if (fd >= 0) ni_remove_below(fd); + (void)unlinkat(islands, staging, AT_REMOVEDIR); + tree = ni_dir_at(islands, digest, false); /* another process may have placed it */ + } + } + if (ni_private_dir(tree) && ni_tree_matches(tree)) + ok = ni_set_root(base, "/scriptc/node-islands/", digest); + if (tree >= 0) close(tree); + close(islands); + return ok; +} + +/* The tree in a private directory of this run, removed at exit. */ +static bool ni_temporary_root(void) { + const char *tmp = getenv("TMPDIR"); + if (tmp == NULL || tmp[0] != '/') tmp = "/tmp"; + int n = snprintf(ni_temp_root, sizeof ni_temp_root, "%s/scriptc-node-islands-XXXXXX", tmp); + if (n <= 0 || (size_t)n >= sizeof ni_temp_root || mkdtemp(ni_temp_root) == NULL) return false; + scr_atexit(ni_remove_temp_root); + int fd = open(ni_temp_root, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC); + bool ok = ni_private_dir(fd) && ni_extract(fd) && ni_tree_matches(fd); + if (fd >= 0) close(fd); + return ok && ni_set_root(ni_temp_root, "", ""); +} + +/* Makes the tree available and sets ni_root; fatal when it cannot. */ +static void ni_open_modules(void) { + if (ni_manifest == NULL) return; /* only built-in modules */ + if (ni_secure()) + ni_fatal("scriptc: a program running with elevated privileges cannot load Node island " + "modules\n"); + ni_parse_manifest(); + if (ni_modules_location != NULL && ni_modules_location[0] != '\0') { + static const char origin[] = "$ORIGIN/"; + if (strncmp(ni_modules_location, origin, sizeof origin - 1) != 0) ni_bad_manifest(); + ScrStr *self = scr_process_exec_path(); + size_t len = self->len; + while (len > 0 && self->data[len - 1] != '/') len--; + bool absolute = len > 0 && self->data[0] == '/'; + char dir[4096]; + int n = snprintf(dir, sizeof dir, "%.*s", (int)len, self->data); + scr_str_release(self); + if (!absolute || n <= 0 || (size_t)n >= sizeof dir || + !ni_set_root(dir, ni_modules_location + sizeof origin - 1, "")) + ni_fatal("scriptc: cannot locate this program's Node island modules\n"); + int fd = open(ni_root, O_RDONLY | O_DIRECTORY | O_CLOEXEC); + bool ok = fd >= 0 && ni_tree_matches(fd); + if (fd >= 0) close(fd); + if (!ok) { + char message[8192]; + snprintf(message, sizeof message, + "scriptc: the Node island modules at %s do not match this program\n", ni_root); + ni_fatal(message); + } + return; + } + NiSha256 s; + unsigned char digest[32]; + ni_sha256_init(&s); + ni_sha256_update(&s, (const unsigned char *)ni_manifest, ni_manifest_len); + ni_sha256_final(&s, digest); + char hex[65]; + for (int i = 0; i < 32; i++) snprintf(hex + i * 2, 3, "%02x", digest[i]); + if (ni_modules == NULL && ni_nfiles > 0) ni_bad_manifest(); + if (!ni_cached_root(hex) && !ni_temporary_root()) + ni_fatal("scriptc: cannot write this program's Node island modules to a private directory\n"); +} + +/* The calling module's path inside the tree: `/`. */ +static char *ni_caller_path(const ScrStr *relative) { + size_t root = strlen(ni_root); + bool slash = root > 0 && ni_root[root - 1] == '/'; + char *path = malloc(root + 1 + relative->len + 1); + if (path == NULL) ni_oom(); + memcpy(path, ni_root, root); + if (!slash) path[root++] = '/'; + memcpy(path + root, relative->data, relative->len); + path[root + relative->len] = '\0'; + return path; +} + +/* The bootstrap: the tree root for the host's resolution guard, then the + * host script. +1 malloc'd. */ +static char *ni_bootstrap(void) { + size_t host = strlen(scr_node_island_host_js); + size_t cap = 64 + strlen(ni_root) * 6 + host; + char *out = malloc(cap); + if (out == NULL) ni_oom(); + size_t n = (size_t)snprintf(out, cap, "globalThis.__scriptcNodeIslandRoot = \""); + for (const char *p = ni_root; *p != '\0'; p++) { + unsigned char c = (unsigned char)*p; + if (c == '"' || c == '\\') + n += (size_t)snprintf(out + n, cap - n, "\\%c", c); + else if (c < 0x20) + n += (size_t)snprintf(out + n, cap - n, "\\u%04x", c); + else + out[n++] = (char)c; + } + n += (size_t)snprintf(out + n, cap - n, "\";\n"); + memcpy(out + n, scr_node_island_host_js, host + 1); + return out; +} + /* ── outcomes ─────────────────────────────────────────────────────────── */ /* The result of an "R" outcome: its JSON copy, or undefined. +1, or NULL @@ -180,7 +772,10 @@ static void ni_exit(int code) { ni_fatal(""); } -static const ScrNodeBridgeHost ni_host = {&ni_settle, &ni_exit}; +/* The start of Node's stack for callers on it (V8's stack-scan bound). */ +static uintptr_t ni_stack_start(void); + +static const ScrNodeBridgeHost ni_host = {&ni_settle, &ni_exit, &ni_stack_start}; /* ── the loop slot ────────────────────────────────────────────────────── */ @@ -191,7 +786,27 @@ static void ni_on_node_stack(void (*job)(void *), void *arg); static void ni_pump(void) { ni_on_node_stack(ni_pump_job, NULL); } static int ni_backend_fd(void) { return ni_bridge->backend_fd(); } static int ni_backend_timeout(void) { return ni_bridge->backend_timeout(); } -static void ni_stop_atexit(void) { ni_on_node_stack(ni_stop_job, NULL); } +typedef struct { + int code; +} NiFinish; +static void ni_finish_job(void *p); + +/* At exit, after the program's own exit work: Node's process ends with + * the program's code ('beforeExit', then 'exit'), and Node stops. A code + * Node's process changed (process.exitCode in Node) is the program's. */ +static void ni_stop_atexit(void) { + int code = (int)scr_process_exit_code_or_zero(); + NiFinish finish = {code}; + /* Node writes to the same stdout. */ + fflush(stdout); + ni_on_node_stack(ni_finish_job, &finish); + ni_on_node_stack(ni_stop_job, NULL); + if (finish.code != code) { + fflush(NULL); + if (ni_temp_root[0] != '\0') ni_remove_temp_root(); + _Exit(finish.code & 0xff); + } +} /* ── Node's stack ─────────────────────────────────────────────────────── */ @@ -219,6 +834,13 @@ static void ni_trampoline(void) { } } +static uintptr_t ni_stack_start(void) { + uintptr_t sp = (uintptr_t)__builtin_frame_address(0); + return ni_stack != NULL && sp >= (uintptr_t)ni_stack && sp < (uintptr_t)ni_stack + NI_STACK + ? (uintptr_t)ni_stack + NI_STACK + : 0; +} + /* Runs job(arg) on Node's stack (directly when already there). */ static void ni_on_node_stack(void (*job)(void *), void *arg) { if (ni_on_stack) { @@ -294,13 +916,21 @@ static void ni_stop_job(void *p) { ni_bridge->stop(); } +static void ni_finish_job(void *p) { + NiFinish *a = p; + a->code = ni_bridge->finish(a->code); +} + /* ── loading Node ─────────────────────────────────────────────────────── */ static void ni_start(void) { if (ni_bridge != NULL) return; const char *dir = ni_pack_dir(); + ni_open_modules(); char path[4352]; - snprintf(path, sizeof path, "%s/%s", dir, NI_BRIDGE_FILE); + int written = snprintf(path, sizeof path, "%s/%s", dir, NI_BRIDGE_FILE); + if (written < 0 || (size_t)written >= sizeof path) + ni_fatal("scriptc: the libnode pack path is too long\n"); /* RTLD_GLOBAL: native addons resolve napi_* and node::* from the global * namespace. The executable exports no symbols libnode could bind to. */ void *library = dlopen(path, RTLD_NOW | RTLD_GLOBAL); @@ -337,8 +967,10 @@ static void ni_start(void) { /* Flush first: Node writes to the same stdout. */ fflush(stdout); ni_bridge = bridge; - NiStart start = {scr_node_island_host_js, argv0, NULL}; + char *bootstrap = ni_bootstrap(); + NiStart start = {bootstrap, argv0, NULL}; ni_on_node_stack(ni_start_job, &start); + free(bootstrap); char *error = start.error; if (error != NULL) { char message[8192]; @@ -364,17 +996,11 @@ static ScrDyn *ni_args_json(ScrDyn *args) { } /* A borrowed ScrStr as a NUL-terminated copy for the bridge. */ -static char *ni_cstr(const ScrStr *s) { - char *copy = malloc(s->len + 1); - if (copy == NULL) ni_oom(); - memcpy(copy, s->data, s->len); - copy[s->len] = '\0'; - return copy; -} +static char *ni_cstr(const ScrStr *s) { return ni_cstr_of(s->data, s->len); } static char *ni_call(uint32_t id, bool sync, ScrStr *file, ScrStr *specifier, ScrStr *name, ScrDyn *json, size_t *len) { - char *f = ni_cstr(file), *s = ni_cstr(specifier), *n = ni_cstr(name); + char *f = ni_caller_path(file), *s = ni_cstr(specifier), *n = ni_cstr(name); NiCall call = {id, sync ? 1 : 0, f, s, n, json->v.str->data, json->v.str->len, 0, NULL}; ni_on_node_stack(ni_call_job, &call); char *outcome = call.outcome; @@ -385,7 +1011,23 @@ static char *ni_call(uint32_t id, bool sync, ScrStr *file, ScrStr *specifier, Sc return outcome; } +/* Node runs on the main thread only: its isolate is entered under the + * Locker the starting thread holds, and the loader's state and Node's + * stack are process-wide. A call from a worker thread throws an Error. */ +static bool ni_main_thread(void) { + if (scr_context_is_main()) return true; + ScrStr *code = scr_str_new("", 0); + static const char text[] = + "Node islands run on the main thread only; this call came from a worker thread"; + ScrStr *message = scr_str_new(text, sizeof text - 1); + scr_throw_node_coded(0, code, message); + scr_str_release(message); + scr_str_release(code); + return false; +} + ScrDyn *scr_node_island_call_sync(ScrStr *file, ScrStr *specifier, ScrStr *name, ScrDyn *args) { + if (!ni_main_thread()) return NULL; ScrDyn *json = ni_args_json(args); if (json == NULL) return NULL; ni_start(); @@ -400,6 +1042,10 @@ ScrDyn *scr_node_island_call_sync(ScrStr *file, ScrStr *specifier, ScrStr *name, ScrPromise *scr_node_island_call(ScrStr *file, ScrStr *specifier, ScrStr *name, ScrDyn *args) { ScrPromise *p = scr_promise_new(); + if (!ni_main_thread()) { + scr_promise_reject_pending(p); + return p; + } ScrDyn *json = ni_args_json(args); if (json == NULL) { scr_promise_reject_pending(p); @@ -434,6 +1080,16 @@ void scr_node_island_install(const char *libnode, const char *node_version) { (void)node_version; } +void scr_node_island_install_modules(const char *manifest, size_t manifest_len, + const unsigned char *data, size_t data_len, + const char *location) { + (void)manifest; + (void)manifest_len; + (void)data; + (void)data_len; + (void)location; +} + SCR_COLD static void ni_unsupported(void) { fputs("scriptc: Node islands are not supported on this target\n", stderr); fflush(stderr); diff --git a/packages/runtime/src/scr_node_island.h b/packages/runtime/src/scr_node_island.h index 9904047f5..33d1d17b4 100644 --- a/packages/runtime/src/scr_node_island.h +++ b/packages/runtime/src/scr_node_island.h @@ -14,6 +14,15 @@ * before %main. */ void scr_node_island_install(const char *libnode, const char *node_version); +/* Records the island module tree (node-islands/modules.ts): its manifest, + * and its files concatenated in manifest order when the executable carries + * them (NULL when they ship beside it at `location`, a `$ORIGIN/` path; "" + * otherwise). All static. The emitted main calls it after + * scr_node_island_install. */ +void scr_node_island_install_modules(const char *manifest, size_t manifest_len, + const unsigned char *data, size_t data_len, + const char *location); + /* @scriptc/node callSync(specifier, name, ...args): `file` is the calling * module's path (relative specifiers resolve from it), `args` the boxed * argument array, copied as JSON. Returns the JSON copy of the result, +1, diff --git a/packages/runtime/src/scr_node_island_host.cjs b/packages/runtime/src/scr_node_island_host.cjs index b18af6319..90c6187ab 100644 --- a/packages/runtime/src/scr_node_island_host.cjs +++ b/packages/runtime/src/scr_node_island_host.cjs @@ -13,10 +13,36 @@ * them in-process under Node: both must produce the same results, errors * and messages. Outcomes are "R" (returned; "R" alone is undefined) * or "T" (threw), the encoding of scr_node_bridge.h. */ -const { createRequire } = require("node:module"); +const { createRequire, registerHooks } = require("node:module"); const nativeHost = globalThis.__scriptcNodeIslandHost; +// The program's island module tree (scr_node_island.c prefixes this script +// with its root, then the bridge removes the global): modules resolve only +// to files inside it, never to a file elsewhere on the machine. +const islandRoot = globalThis.__scriptcNodeIslandRoot; +delete globalThis.__scriptcNodeIslandRoot; +if (typeof islandRoot === "string" && islandRoot !== "/") { + const { realpathSync } = require("node:fs"); + const { sep } = require("node:path"); + const { fileURLToPath } = require("node:url"); + const root = realpathSync(islandRoot); + const inside = root.endsWith(sep) ? root : root + sep; + registerHooks({ + resolve(specifier, context, nextResolve) { + const result = nextResolve(specifier, context); + if (result.url.startsWith("file:") && !fileURLToPath(result.url).startsWith(inside)) { + const error = new Error( + `Cannot find module '${specifier}': it is not one of the modules this program ships`, + ); + error.code = "MODULE_NOT_FOUND"; + throw error; + } + return result; + }, + }); +} + // Error classes that cross as copies, at their scr_error.h kind index. const ERROR_KINDS = [ [TypeError, 1], @@ -74,8 +100,18 @@ function errorKind(error) { } // A thrown value crosses as a copy: an Error as its nearest intrinsic -// class with its message and string code, anything else as JSON data. +// class with its message and string code, anything else as JSON data. An +// error whose class, message or code cannot be read crosses as an Error +// with an empty message. function thrown(error) { + try { + return thrownCopy(error); + } catch { + return 'T{"error":{"kind":0,"message":""}}'; + } +} + +function thrownCopy(error) { if (error instanceof Error) { const copy = { kind: errorKind(error), message: String(error.message) }; if (typeof error.code === "string") copy.code = error.code; @@ -125,4 +161,28 @@ for (const stream of [process.stdout, process.stderr]) { stream._handle.setBlocking(true); } -globalThis.__scriptcNodeIsland = { call }; +// The program ends with `code` (the bridge then runs Node's loop to its +// end, which emits 'beforeExit' and 'exit'): Node's process shares it, as +// it shares one process under Node. +function finish(code) { + if (code !== 0) process.exitCode = code; +} + +// process.execPath is this program, not a node executable: a fork without +// an execPath of its own would start the program again. +const childProcess = require("node:child_process"); +const nodeFork = childProcess.fork; +childProcess.fork = function fork(modulePath, args, options) { + const forkOptions = Array.isArray(args) ? options : args; + const execPath = + forkOptions === null || typeof forkOptions !== "object" ? undefined : forkOptions.execPath; + if (execPath === undefined || execPath === process.execPath) { + throw new Error( + "child_process.fork() in a Node island needs options.execPath (a node executable): process.execPath is this program", + ); + } + return Reflect.apply(nodeFork, this, arguments); +}; +require("node:module").syncBuiltinESMExports(); + +globalThis.__scriptcNodeIsland = { call, finish }; diff --git a/packages/runtime/src/scr_node_island_host.h b/packages/runtime/src/scr_node_island_host.h index 8a963535c..ed78c8951 100644 --- a/packages/runtime/src/scr_node_island_host.h +++ b/packages/runtime/src/scr_node_island_host.h @@ -19,10 +19,36 @@ static const char scr_node_island_host_js[] = " * them in-process under Node: both must produce the same results, errors\012" " * and messages. Outcomes are \"R\" (returned; \"R\" alone is undefined)\012" " * or \"T\" (threw), the encoding of scr_node_bridge.h. */\012" - "const { createRequire } = require(\"node:module\");\012" + "const { createRequire, registerHooks } = require(\"node:module\");\012" "\012" "const nativeHost = globalThis.__scriptcNodeIslandHost;\012" "\012" + "// The program's island module tree (scr_node_island.c prefixes this script\012" + "// with its root, then the bridge removes the global): modules resolve only\012" + "// to files inside it, never to a file elsewhere on the machine.\012" + "const islandRoot = globalThis.__scriptcNodeIslandRoot;\012" + "delete globalThis.__scriptcNodeIslandRoot;\012" + "if (typeof islandRoot === \"string\" && islandRoot !== \"/\") {\012" + " const { realpathSync } = require(\"node:fs\");\012" + " const { sep } = require(\"node:path\");\012" + " const { fileURLToPath } = require(\"node:url\");\012" + " const root = realpathSync(islandRoot);\012" + " const inside = root.endsWith(sep) \077 root : root + sep;\012" + " registerHooks({\012" + " resolve(specifier, context, nextResolve) {\012" + " const result = nextResolve(specifier, context);\012" + " if (result.url.startsWith(\"file:\") && !fileURLToPath(result.url).startsWith(inside)) {\012" + " const error = new Error(\012" + " `Cannot find module '${specifier}': it is not one of the modules this program ships`,\012" + " );\012" + " error.code = \"MODULE_NOT_FOUND\";\012" + " throw error;\012" + " }\012" + " return result;\012" + " },\012" + " });\012" + "}\012" + "\012" "// Error classes that cross as copies, at their scr_error.h kind index.\012" "const ERROR_KINDS = [\012" " [TypeError, 1],\012" @@ -80,8 +106,18 @@ static const char scr_node_island_host_js[] = "}\012" "\012" "// A thrown value crosses as a copy: an Error as its nearest intrinsic\012" - "// class with its message and string code, anything else as JSON data.\012" + "// class with its message and string code, anything else as JSON data. An\012" + "// error whose class, message or code cannot be read crosses as an Error\012" + "// with an empty message.\012" "function thrown(error) {\012" + " try {\012" + " return thrownCopy(error);\012" + " } catch {\012" + " return 'T{\"error\":{\"kind\":0,\"message\":\"\"}}';\012" + " }\012" + "}\012" + "\012" + "function thrownCopy(error) {\012" " if (error instanceof Error) {\012" " const copy = { kind: errorKind(error), message: String(error.message) };\012" " if (typeof error.code === \"string\") copy.code = error.code;\012" @@ -131,6 +167,30 @@ static const char scr_node_island_host_js[] = " stream._handle.setBlocking(true);\012" "}\012" "\012" - "globalThis.__scriptcNodeIsland = { call };\012"; + "// The program ends with `code` (the bridge then runs Node's loop to its\012" + "// end, which emits 'beforeExit' and 'exit'): Node's process shares it, as\012" + "// it shares one process under Node.\012" + "function finish(code) {\012" + " if (code !== 0) process.exitCode = code;\012" + "}\012" + "\012" + "// process.execPath is this program, not a node executable: a fork without\012" + "// an execPath of its own would start the program again.\012" + "const childProcess = require(\"node:child_process\");\012" + "const nodeFork = childProcess.fork;\012" + "childProcess.fork = function fork(modulePath, args, options) {\012" + " const forkOptions = Array.isArray(args) \077 options : args;\012" + " const execPath =\012" + " forkOptions === null || typeof forkOptions !== \"object\" \077 undefined : forkOptions.execPath;\012" + " if (execPath === undefined || execPath === process.execPath) {\012" + " throw new Error(\012" + " \"child_process.fork() in a Node island needs options.execPath (a node executable): process.execPath is this program\",\012" + " );\012" + " }\012" + " return Reflect.apply(nodeFork, this, arguments);\012" + "};\012" + "require(\"node:module\").syncBuiltinESMExports();\012" + "\012" + "globalThis.__scriptcNodeIsland = { call, finish };\012"; #endif /* SCR_NODE_ISLAND_HOST_H */ diff --git a/packages/runtime/src/scr_stream.c b/packages/runtime/src/scr_stream.c index 275c03039..cd95e12cd 100644 --- a/packages/runtime/src/scr_stream.c +++ b/packages/runtime/src/scr_stream.c @@ -269,6 +269,7 @@ struct ScrStreamState { * destroyed and runs its _destroy once construction completes. */ bool constructing, destroy_deferred; bool construct_finished; /* onConstruct ran: a second callback is ERR_MULTIPLE_CALLBACK */ + bool construct_called; /* the construct tick called the hook */ ScrError *destroy_err; /* owned: the deferred destroy's error */ ScrClosure *construct_cb; ScrStreamPlainInv construct_inv; @@ -2157,6 +2158,24 @@ static ScrDyn *scr_stream_wentry_encoding(const ScrStreamState *st, ScrDyn *enc) return scr_dyn_retain(enc ? enc : scr_dyn_undefined()); } +/* A compiled writev whose entries declare `encoding` a string: an entry + * without one (an object-mode write passes undefined or null) gets "". + * The batch array is the one this call receives. */ +void scr_stream_writev_fill_encodings(ScrDyn *chunks) { + if (!chunks || chunks->kind != SCR_DYN_ARR) return; + double n = scr_dyn_arr_len(chunks); + for (double i = 0; i < n; i++) { + ScrDyn *entry = scr_dyn_arr_at(chunks, i); /* +1 */ + ScrDyn *enc = entry->kind == SCR_DYN_OBJ ? scr_dyn_obj_get(entry, "encoding", 8) : NULL; + if (entry->kind == SCR_DYN_OBJ && (!enc || enc->kind == SCR_DYN_UNDEF || enc->kind == SCR_DYN_NULL)) { + ScrStr *empty = scr_str_new("", 0); + scr_dyn_obj_set(entry, "encoding", 8, scr_dyn_new_str(empty)); + scr_str_release(empty); + } + scr_dyn_release(entry); + } +} + /* A batch entry's callback: the write's callback, or Node's nop. */ static ScrDyn *scr_stream_writev_entry_cb_call(ScrClosure *clo, ScrDyn *const *args, size_t argc) { (void)args; @@ -2675,9 +2694,8 @@ void *scr_stream_native(ScrStream *s, void (*release)(void *)) { * way the compiler-emitted dyn thunks do (chunk as Buffer-flavored bytes, * encoding 'buffer', the error via the boundary encoding, the completion * callback as a callable dyn over the *_done glue). Options that Node's - * base WOULD consume but have no lowering here (objectMode truthy, - * decodeStrings:false, non-utf8 defaultEncoding, signal, construct, - * writev, and read/write/final on the transformish bases) throw the + * base WOULD consume but have no lowering here (signal, and + * read/write/final on the transformish bases) throw the * "not supported yet" Error — the compile-time fence's runtime twin. */ /* The boxed dyn callable (+1 — scr_box_get_ref retains; callers release @@ -2903,11 +2921,8 @@ static bool scr_stream_parse_dyn_opts(const ScrDyn *opts, char base, ScrStreamDy } } out->construct = scr_stream_dynopt_fn(opts, "construct", 9); + /* a Transform's batches reach writev, not the transform, as in Node */ if (base != 'r') out->writev = scr_stream_dynopt_fn(opts, "writev", 6); - if (transformish && out->writev != NULL) { - static const char m[] = "the stream option 'writev' on a Transform is not supported yet (batches would bypass the transform)"; - return scr_stream_dynopt_unsupported(m, sizeof m - 1); - } { /* a known encoding or ERR_UNKNOWN_ENCODING (the states' constructors) */ ScrDyn *v = scr_dyn_obj_get((ScrDyn *)opts, "defaultEncoding", 15); @@ -6296,6 +6311,55 @@ ScrStream *scr_stream_set_default_encoding(ScrStream *s, ScrDyn *enc) { return scr_stream_retain(s); } +/* Whether a defaultEncoding option value is the default: null, undefined, + * 'utf8' or 'utf-8' (the state constructors' kDefaultUTF8Encoding). */ +static bool scr_stream_default_encoding_value(const ScrDyn *v) { + if (!v || v->kind == SCR_DYN_UNDEF || v->kind == SCR_DYN_NULL) return true; + return v->kind == SCR_DYN_STR && ((v->v.str->len == 4 && memcmp(v->v.str->data, "utf8", 4) == 0) || + (v->v.str->len == 5 && memcmp(v->v.str->data, "utf-8", 5) == 0)); +} + +/* A runtime defaultEncoding option value, checked before construction as + * the state constructors check it: the default, a name Buffer.isEncoding + * knows, or ERR_UNKNOWN_ENCODING (thrown). Borrows v. */ +void scr_stream_check_encoding_option(ScrDyn *v) { + if (scr_stream_default_encoding_value(v)) return; + if (v->kind == SCR_DYN_STR) { + ScrStr *canonical = scr_bytes_normalize_encoding(v->v.str); + if (canonical) scr_str_release(canonical); + return; + } + ScrStr *shown = scr_dyn_string_coerce_js(v); + if (!shown) return; + char msg[128]; + int len = snprintf(msg, sizeof msg, "Unknown encoding: %.*s", (int)(shown->len < 64 ? shown->len : 64), + shown->data); + scr_str_release(shown); + scr_throw_error_msg_code(SCR_ERR_TYPE, msg, (size_t)(len < 0 ? 0 : len), "ERR_UNKNOWN_ENCODING"); +} + +/* Runtime decodeStrings and defaultEncoding option values over a fresh + * stream (after its objectMode): decodeStrings === false keeps string + * writes, and a known defaultEncoding is the write side's default as + * given and how a byte-mode readable side decodes push(string). Borrows + * both; answers the receiver (+1). */ +ScrStream *scr_stream_option_values(ScrStream *s, ScrDyn *decode_strings, ScrDyn *default_encoding) { + ScrStreamState *st = s->st; + if (st->has_w && decode_strings && decode_strings->kind == SCR_DYN_BOOL && !decode_strings->v.b) + scr_stream_set_decode_strings(s, false); + if (!scr_stream_default_encoding_value(default_encoding) && default_encoding->kind == SCR_DYN_STR) { + ScrStr *canonical = scr_bytes_normalize_encoding(default_encoding->v.str); + if (canonical) { + scr_stream_set_write_encoding(s, default_encoding->v.str); + if (st->has_r && !st->r.object_entries) scr_stream_release(scr_stream_set_push_encoding(s, canonical)); + scr_str_release(canonical); + } else if (scr_exc_pending()) { + scr_exc_clear(); /* checked before construction */ + } + } + return scr_stream_retain(s); +} + /* stream.objectMode: scr_stream_set_object_mode, answering the receiver * (+1). */ ScrStream *scr_stream_object_mode(ScrStream *s, bool readable, bool writable) { @@ -7460,6 +7524,17 @@ void scr_stream_set_construct(ScrStream *s, ScrClosure *cb /*moves*/, ScrStreamP scr_stream_construct(s, cb, inv); } +void scr_stream_replace_construct(ScrStream *s, ScrClosure *cb /*moves*/, ScrStreamPlainInv inv) { + ScrStreamState *st = s->st; + if (!st->constructing || st->construct_called || !st->construct_cb) { + scr_closure_release(cb); + return; + } + scr_closure_release(st->construct_cb); + st->construct_cb = cb; + st->construct_inv = inv; +} + /* A class's _writev under options that may carry their own writev (the * option, an instance property in Node, wins). */ void scr_stream_set_writev_default(ScrStream *s, ScrClosure *cb /*moves*/, ScrStreamWritevInv inv) { @@ -7728,6 +7803,7 @@ static void scr_stream_run_tick(ScrStreamTick *t) { scr_stream_error_buffer(s); break; case SCR_ST_CONSTRUCT: { + st->construct_called = true; if (st->construct_cb) { st->construct_inv(st->construct_cb, s); /* a throw inside _construct reaches onConstruct on a tick */ diff --git a/packages/runtime/src/scr_stream.h b/packages/runtime/src/scr_stream.h index d26077b5c..a4798c629 100644 --- a/packages/runtime/src/scr_stream.h +++ b/packages/runtime/src/scr_stream.h @@ -561,8 +561,15 @@ void scr_stream_set_writev_concat(ScrStream *s); /* The user construct option / _construct method (scheduled now; cb * moves), and the writev option / _writev method (cb moves). */ void scr_stream_set_construct(ScrStream *s, ScrClosure *cb, ScrStreamPlainInv inv); +/* `s._construct = fn` after construction: replaces a construct hook whose + * tick has not run yet (Node's constructNT reads _construct then), and has + * no effect otherwise (cb moves). */ +void scr_stream_replace_construct(ScrStream *s, ScrClosure *cb, ScrStreamPlainInv inv); void scr_stream_set_writev(ScrStream *s, ScrClosure *cb, ScrStreamWritevInv inv); void scr_stream_set_writev_default(ScrStream *s, ScrClosure *cb, ScrStreamWritevInv inv); +/* A compiled writev whose entries declare `encoding` a string: entries + * without an encoding (object mode) get "" before the conversion. */ +void scr_stream_writev_fill_encodings(struct ScrDyn *chunks); /* Duplex.from (duplexify.js): over an async generator function (the * compiled stage takes the source generator that the compiled source * makes from next() and ack(); all borrowed, helpers too), and over a @@ -605,6 +612,11 @@ ScrStream *scr_stream_duplex_from_promise(struct ScrPromise *p, ScrClosure *box) * forms answer the stream (+1). */ void scr_stream_set_decode_strings(ScrStream *s, bool decode); ScrStream *scr_stream_decode_strings(ScrStream *s, bool decode); +/* Runtime decodeStrings/defaultEncoding option values: checked before + * construction (ERR_UNKNOWN_ENCODING thrown), then applied to the fresh + * stream, answering it (+1). Values borrowed. */ +void scr_stream_check_encoding_option(struct ScrDyn *v); +ScrStream *scr_stream_option_values(ScrStream *s, struct ScrDyn *decode_strings, struct ScrDyn *default_encoding); void scr_stream_set_write_encoding(ScrStream *s, ScrStr *enc); ScrStream *scr_stream_write_encoding(ScrStream *s, ScrStr *enc); ScrStream *scr_stream_set_default_encoding(ScrStream *s, struct ScrDyn *enc); diff --git a/packages/runtime/test/test_json.c b/packages/runtime/test/test_json.c index 2c072006f..865f1c757 100644 --- a/packages/runtime/test/test_json.c +++ b/packages/runtime/test/test_json.c @@ -551,6 +551,91 @@ static void test_class_values(void) { scr_str_release(name); } +/* The class-view copy (scr_dyn_copy_property_descriptors) gives each + * property of the bag the attributes, value or accessors, and own-key + * position that defining it from its own descriptor would. */ +static ScrDyn *define_from(ScrDyn *target, const char *key, ScrDyn *value, bool writable, + bool enumerable, bool configurable) { + ScrDyn *descriptor = scr_dyn_new_obj(); + scr_dyn_obj_set(descriptor, "value", 5, value); + scr_dyn_obj_set(descriptor, "writable", 8, scr_dyn_new_bool(writable)); + scr_dyn_obj_set(descriptor, "enumerable", 10, scr_dyn_new_bool(enumerable)); + scr_dyn_obj_set(descriptor, "configurable", 12, scr_dyn_new_bool(configurable)); + ScrStr *name = S(key); + ScrDyn *k = scr_dyn_new_str(name); + ScrDyn *result = scr_dyn_define_property(target, k, descriptor); + scr_dyn_release(k); + scr_str_release(name); + scr_dyn_release(descriptor); + return result; +} + +static const ScrDynEntry *entry_of(const ScrDyn *obj, const char *key) { + for (size_t i = 0; i < obj->v.obj.len; i++) + if (obj->v.obj.entries[i].key_len == strlen(key) && + memcmp(obj->v.obj.entries[i].key, key, strlen(key)) == 0) + return &obj->v.obj.entries[i]; + return NULL; +} + +static void test_copy_property_descriptors(void) { + ScrDyn *bag = scr_dyn_new_obj(); + ScrDyn *getter = scr_dyn_new_obj(); + scr_dyn_release(define_from(bag, "label", scr_dyn_new_num(1), true, true, true)); + scr_dyn_release(define_from(bag, "hidden", scr_dyn_new_num(2), false, false, true)); + scr_dyn_obj_set(bag, "7", 1, scr_dyn_new_num(7)); + scr_dyn_obj_set(bag, "3", 1, scr_dyn_new_num(3)); + /* An accessor entry (the getter object stands in for its function). */ + scr_dyn_obj_set(bag, "computed", 8, scr_dyn_retain(scr_dyn_undefined())); + ScrDynEntry *accessor = (ScrDynEntry *)entry_of(bag, "computed"); + accessor->accessor = true; + accessor->getter = scr_dyn_retain(getter); + accessor->writable = false; + accessor->enumerable = false; + accessor->configurable = false; + + ScrDyn *view = scr_dyn_new_obj(); + scr_dyn_release(scr_dyn_copy_property_descriptors(view, bag)); + check(!scr_exc_pending() && view->v.obj.len == 5, "copy keeps every bag property"); + ScrDyn *keys = scr_dyn_obj_own_keys(view); + const char *order[] = {"3", "7", "label", "hidden", "computed"}; + bool ordered = keys->v.arr.len == 5; + for (size_t i = 0; ordered && i < 5; i++) ordered = str_is(keys->v.arr.items[i]->v.str, order[i]); + check(ordered, "copy keeps the bag's own-key order"); + scr_dyn_release(keys); + const ScrDynEntry *hidden = entry_of(view, "hidden"); + check(hidden && !hidden->accessor && hidden->value->v.num == 2 && !hidden->writable && + !hidden->enumerable && hidden->configurable, + "copy keeps data attributes"); + const ScrDynEntry *copied = entry_of(view, "computed"); + check(copied && copied->accessor && copied->getter == getter && !copied->setter && + !copied->writable && !copied->enumerable && !copied->configurable && + copied->value->kind == SCR_DYN_UNDEF, + "copy keeps an accessor without calling it"); + + /* A property the target already has is redefined like defineProperty. */ + ScrDyn *existing = scr_dyn_new_obj(); + scr_dyn_obj_set(existing, "label", 5, scr_dyn_new_num(9)); + scr_dyn_release(scr_dyn_copy_property_descriptors(existing, bag)); + const ScrDynEntry *label = entry_of(existing, "label"); + check(!scr_exc_pending() && existing->v.obj.len == 5 && existing->v.obj.entries[0].key_len == 5 && + label->value->v.num == 1, + "copy redefines an existing property in place"); + + /* A target that takes no new properties throws as defineProperty does. */ + ScrDyn *closed = scr_dyn_new_obj(); + scr_dyn_release(scr_dyn_prevent_extensions(closed)); + ScrDyn *refused = scr_dyn_copy_property_descriptors(closed, bag); + check(!refused && scr_exc_pending() && closed->v.obj.len == 0, "copy onto a non-extensible object throws"); + scr_exc_clear(); + + scr_dyn_release(closed); + scr_dyn_release(existing); + scr_dyn_release(view); + scr_dyn_release(bag); + scr_dyn_release(getter); +} + static void test_primitive_owners(void) { ScrDyn *values[] = {scr_dyn_new_null(), scr_dyn_new_bool(false), scr_dyn_new_bool(true)}; const char *texts[] = {"null", "false", "true"}; @@ -579,6 +664,7 @@ int main(void) { test_class_values(); scr_init(); test_primitive_owners(); + test_copy_property_descriptors(); /* ── primitives ─────────────────────────────────────────────────── */ ScrDyn *d = parse_ok("null", "parse null"); diff --git a/tests/corpus/7447-union-record-arm-width.mjs b/tests/corpus/7447-union-record-arm-width.mjs new file mode 100644 index 000000000..dc12f8a59 --- /dev/null +++ b/tests/corpus/7447-union-record-arm-width.mjs @@ -0,0 +1,37 @@ +// Values read back from arrays whose elements mix object literals of +// different widths, an empty object among them, keep every field: the +// record arm with more fields is tried before `{}`, which matches every +// object. +const pairs = [ + ["a", { publicKeyEncoding: 5 }], + ["b", {}], + ["c", { publicKeyEncoding: 6, extra: true }], +]; +for (const [label, value] of pairs) console.log(label, Object.keys(value), JSON.stringify(value)); + +const options = [ + ["rsa", { modulusLength: 1024 }], + ["ec", { namedCurve: "P-256", paramEncoding: "named" }], + ["none", {}], +]; +for (const [type, value] of options) { + const { modulusLength, namedCurve, paramEncoding } = value; + console.log(type, modulusLength, namedCurve, paramEncoding); +} + +const mixed = [{ a: 1 }, {}, "s", { a: 2, b: "two" }]; +for (const value of mixed) { + console.log(typeof value, JSON.stringify(value), typeof value === "object" ? Object.keys(value).length : -1); +} + +function describe(value) { + return Object.entries(value) + .map(([key, field]) => `${key}=${field}`) + .join(";"); +} +const rows = [ + [1, { x: 1, y: 2 }], + [2, {}], + [3, { x: 3 }], +]; +for (const [id, value] of rows) console.log(id, describe(value), value.x, value.y); diff --git a/tests/corpus/7448-union-record-arm-width.ts b/tests/corpus/7448-union-record-arm-width.ts new file mode 100644 index 000000000..1518dd575 --- /dev/null +++ b/tests/corpus/7448-union-record-arm-width.ts @@ -0,0 +1,21 @@ +// A checked cast to a union of record types keeps every field of the +// value: arms with more required fields, then more fields, are tried +// before an arm whose fields are all optional, which matches every object. +type Full = { p: number; q: string }; +type Bare = { p?: number }; +type Other = { r: boolean }; + +const values = JSON.parse('[{"p":5,"q":"x"},{},{"r":true},{"p":7}]') as (Full | Bare | Other)[]; +for (const value of values) { + const q = "q" in value ? value.q : "-"; + const r = "r" in value ? value.r : "-"; + console.log(JSON.stringify(value), "q" in value, "r" in value, q, r); +} + +const one = JSON.parse('{"p":1,"q":"y"}') as Bare | Full; +console.log(JSON.stringify(one), "q" in one ? one.q : "no q"); + +type Point = { x: number; y?: number }; +type Labeled = { x: number; label: string }; +const points = JSON.parse('[{"x":1,"label":"a"},{"x":2,"y":3},{"x":4}]') as (Point | Labeled)[]; +for (const point of points) console.log(point.x, "label" in point ? point.label : "none", "y" in point ? point.y : "none"); diff --git a/tests/corpus/7460-transform-writev.mjs b/tests/corpus/7460-transform-writev.mjs new file mode 100644 index 000000000..d78bdfc77 --- /dev/null +++ b/tests/corpus/7460-transform-writev.mjs @@ -0,0 +1,33 @@ +// writev on a Transform and a PassThrough (the writev option): a batch of +// buffered writes reaches writev instead of the transform, a lone write +// still reaches the transform, and the readable side gets what writev +// pushes. +import { Transform, PassThrough } from "node:stream"; +const t = new Transform({ + transform(chunk, enc, cb) { console.log("transform", String(chunk), enc); cb(null, chunk); }, + writev(chunks, cb) { + console.log("writev", chunks.map((c) => `${c.chunk}/${c.encoding}`).join(",")); + for (const { chunk } of chunks) this.push(String(chunk).toUpperCase()); + cb(); + }, +}); +t.on("data", (d) => console.log("data", String(d))); +t.on("finish", () => console.log("finish")); +t.on("end", () => console.log("end")); +t.write("a"); +t.cork(); +t.write("b"); +t.write("c", "latin1"); +process.nextTick(() => { t.uncork(); t.end("d"); }); + +const p = new PassThrough({ + objectMode: true, + writev(chunks, cb) { console.log("pt writev", JSON.stringify(chunks.map((c) => c.chunk))); cb(); }, +}); +p.on("data", (d) => console.log("pt data", JSON.stringify(d))); +p.cork(); +p.write({ x: 1 }); +p.write([2]); +p.uncork(); +p.write("solo"); +p.end(); diff --git a/tests/corpus/7461-transform-writev.cjs b/tests/corpus/7461-transform-writev.cjs new file mode 100644 index 000000000..dd45cf09d --- /dev/null +++ b/tests/corpus/7461-transform-writev.cjs @@ -0,0 +1,33 @@ +// Constructor-function Transform and PassThrough streams with _writev: +// the prototype's _writev takes the batches, and a writev option shadows +// it with the stream as this. +'use strict'; +const { Transform, PassThrough } = require('node:stream'); +function T(opts) { Transform.call(this, opts); } +Object.setPrototypeOf(T.prototype, Transform.prototype); +Object.setPrototypeOf(T, Transform); +T.prototype._transform = function (chunk, enc, cb) { cb(null, 'T(' + chunk + ')'); }; +T.prototype._writev = function (chunks, cb) { + this.push('V(' + chunks.map((c) => String(c.chunk)).join('+') + ')'); + cb(); +}; +const t = Object.create(T.prototype); +T.call(t, {}); +t.on('data', (d) => console.log('t data', String(d))); +t.on('end', () => console.log('t end')); +t.write('a'); +t.cork(); +t.write('b'); +t.write('c'); +process.nextTick(() => { t.uncork(); t.end(); }); +function P(opts) { PassThrough.call(this, opts); } +Object.setPrototypeOf(P.prototype, PassThrough.prototype); +Object.setPrototypeOf(P, PassThrough); +const p = Object.create(P.prototype); +P.call(p, { writev(chunks, cb) { console.log('p writev', chunks.length, this === p); cb(); } }); +p.on('data', (d) => console.log('p data', String(d))); +p.cork(); +p.write('x'); +p.write('y'); +p.uncork(); +p.write('z'); diff --git a/tests/corpus/7462-transform-writev.ts b/tests/corpus/7462-transform-writev.ts new file mode 100644 index 000000000..ca079b3c9 --- /dev/null +++ b/tests/corpus/7462-transform-writev.ts @@ -0,0 +1,109 @@ +// Typed writev on Transform and PassThrough: an options writev with +// decodeStrings: false and flush, a Transform subclass's _writev, a +// batch completing with an error, and object-mode entries whose +// encoding is declared as a string or as Node's string | null | +// undefined. +import { Transform, PassThrough, Writable } from "node:stream"; +type WritevChunks = Array<{ chunk: any; encoding: BufferEncoding }>; +type Cb = (error?: Error | null) => void; +type TransformCallback = (error?: Error | null, data?: any) => void; +const log: string[] = []; +const t = new Transform({ + decodeStrings: false, + transform(chunk: string, enc: BufferEncoding, cb: TransformCallback): void { + log.push(`transform ${chunk} ${enc}`); + cb(null, chunk); + }, + flush(cb: TransformCallback): void { + log.push("flush"); + cb(null, "!"); + }, + writev(chunks: WritevChunks, cb: Cb): void { + log.push(`writev ${chunks.map((c) => `${c.chunk}/${c.encoding}`).join(",")}`); + this.push(chunks.map((c) => String(c.chunk)).join("")); + cb(); + }, +}); +t.on("data", (d: Buffer) => log.push(`data ${d.toString()}`)); +t.on("end", () => log.push("end")); +t.on("finish", () => log.push("finish")); +t.on("close", () => { + console.log(log.join("\n")); + next(); +}); +t.write("one"); +t.cork(); +t.write("two", "latin1"); +t.write("three"); +t.uncork(); +t.end("four"); + +class Batcher extends Transform { + _transform(chunk: Buffer, _enc: BufferEncoding, cb: TransformCallback): void { + console.log("batcher transform", chunk.toString()); + cb(null, chunk); + } + _writev(chunks: Array<{ chunk: any; encoding: BufferEncoding }>, cb: Cb): void { + console.log("batcher writev", chunks.length); + this.push(Buffer.concat(chunks.map((c) => c.chunk as Buffer))); + cb(); + } +} + +function next(): void { + const b = new Batcher(); + b.on("data", (d: Buffer) => console.log("batcher data", d.toString())); + b.write("x"); + b.cork(); + b.write("y"); + b.write("z"); + process.nextTick(() => { + b.uncork(); + b.end(); + }); + b.on("finish", () => { + const p = new PassThrough({ + writev(chunks: WritevChunks, cb: Cb): void { + console.log("pt writev", chunks.length, String(chunks[0]!.chunk)); + cb(new Error("batch failed")); + }, + }); + p.on("error", (e: Error) => { + console.log("pt error", e.message); + objectEntries(); + }); + p.cork(); + p.write("1"); + p.write("2"); + p.uncork(); + }); +} + +function objectEntries(): void { + const declared = new Writable({ + objectMode: true, + write(_chunk: any, _enc: BufferEncoding, cb: Cb): void { cb(); }, + writev(chunks: WritevChunks, cb: Cb): void { + console.log("string encodings", JSON.stringify(chunks.map((c) => c.chunk))); + cb(); + }, + }); + declared.write({ n: 0 }); + declared.cork(); + declared.write({ n: 1 }); + declared.write({ n: 2 }, () => {}); + declared.uncork(); + const nodeTyped = new Writable({ + objectMode: true, + write(_chunk: any, _enc: BufferEncoding, cb: Cb): void { cb(); }, + writev(chunks: Array<{ chunk: any; encoding: string | null | undefined }>, cb: Cb): void { + console.log("node encodings", JSON.stringify(chunks.map((c) => [c.chunk, c.encoding === undefined ? "undefined" : c.encoding]))); + cb(); + }, + }); + nodeTyped.write({ n: 0 }); + nodeTyped.cork(); + nodeTyped.write({ n: 1 }); + nodeTyped.write({ n: 2 }, () => {}); + nodeTyped.uncork(); +} diff --git a/tests/corpus/7463-stream-option-values.ts b/tests/corpus/7463-stream-option-values.ts new file mode 100644 index 000000000..04f497b6b --- /dev/null +++ b/tests/corpus/7463-stream-option-values.ts @@ -0,0 +1,58 @@ +// Typed decodeStrings and defaultEncoding options whose values are known +// only at run time: decodeStrings === false keeps string writes, a known +// defaultEncoding is the write default as given and how push(string) +// decodes, null and utf8 are the default, an unknown name throws +// ERR_UNKNOWN_ENCODING before construction, and a subclass's super() +// options take them the same way. +import { Writable, Readable, Duplex } from "node:stream"; +type Cb = (error?: Error | null) => void; +const flags: boolean[] = JSON.parse("[false, true]"); +const encodings: Array = JSON.parse('["hex", "latin1", null, "UTF8"]'); +for (const decode of flags) { + const w = new Writable({ + decodeStrings: decode, + defaultEncoding: encodings[0] as BufferEncoding, + write(chunk: any, enc: string, cb: Cb): void { + console.log("write", decode, typeof chunk, enc, String(chunk)); + cb(); + }, + }); + w.write("6869"); +} +const r = new Readable({ defaultEncoding: encodings[1] as BufferEncoding, read(): void {} }); +r.push("é"); +r.push(null); +r.on("data", (d: Buffer) => console.log("readable", d.toString("hex"))); +const dflt = new Writable({ + defaultEncoding: encodings[2] as any, + write(chunk: Buffer, enc: string, cb: Cb): void { + console.log("null default", enc, chunk.toString()); + cb(); + }, +}); +dflt.write("ok"); +const upper = new Duplex({ + defaultEncoding: encodings[3] as BufferEncoding, + decodeStrings: flags[0], + read(): void {}, + write(chunk: any, enc: string, cb: Cb): void { + console.log("duplex", typeof chunk, enc); + cb(); + }, +}); +upper.write("x"); +try { + new Writable({ defaultEncoding: JSON.parse('"nope"') }); +} catch (e) { + console.log("bad", (e as Error & { code?: string }).code, (e as Error).message); +} +class Sink extends Writable { + constructor(decode: boolean, encoding: BufferEncoding) { + super({ decodeStrings: decode, defaultEncoding: encoding }); + } + _write(chunk: any, enc: string, cb: Cb): void { + console.log("sink", typeof chunk, enc); + cb(); + } +} +new Sink(flags[0]!, encodings[1] as BufferEncoding).write("y"); diff --git a/tests/corpus/7464-stream-option-values.mjs b/tests/corpus/7464-stream-option-values.mjs new file mode 100644 index 000000000..dfeb7a447 --- /dev/null +++ b/tests/corpus/7464-stream-option-values.mjs @@ -0,0 +1,28 @@ +// Option values known only at run time in JavaScript: decodeStrings and +// defaultEncoding read from an object, a write callback whose chunk is +// then a string, decodeStrings: 0 (only false turns decoding off), a +// defaultEncoding that is not a string, and a _construct assigned after +// construction, which Node's constructor never reads. +import { Writable, Readable } from "node:stream"; +const opts = { decode: false, enc: "base64" }; +const w = new Writable({ + decodeStrings: opts.decode, + defaultEncoding: opts.enc, + write(c, e, cb) { console.log("js", typeof c, e, String(c)); cb(); }, +}); +w.write("aGk="); +try { new Writable({ defaultEncoding: 5 }); } catch (e) { console.log("bad", e.code, e.message); } +const zero = new Writable({ decodeStrings: 0, write(c, e, cb) { console.log("zero", Buffer.isBuffer(c), e); cb(); } }); +zero.write("z"); +const late = new Writable({ write(chunk, enc, cb) { console.log("write", chunk.toString()); cb(); } }); +let ran = false; +late._construct = (cb) => { ran = true; cb(); }; +late.write("a"); +late.end(() => console.log("ended, construct ran:", ran)); +const r = new Readable({ read() {} }); +const order = []; +(order.push("receiver"), r)._construct = (order.push("value"), function (cb) { console.log("never"); cb(); }); +console.log(order.join(",")); +r.push("x"); +r.push(null); +r.on("data", (d) => console.log("data", d.toString())); diff --git a/tests/corpus/7465-transform-callback-values.ts b/tests/corpus/7465-transform-callback-values.ts new file mode 100644 index 000000000..e7003e894 --- /dev/null +++ b/tests/corpus/7465-transform-callback-values.ts @@ -0,0 +1,41 @@ +// A transform or flush callback passing a value that is not a Buffer or a +// string: the push reports Node's ERR_INVALID_ARG_TYPE through 'error' +// and destroys the stream, for option callbacks and _transform methods, +// while Buffer and string values take the typed path. +import { Transform } from "node:stream"; +type TransformCallback = (error?: Error | null, data?: any) => void; +const t = new Transform({ + transform(chunk: Buffer, _enc: BufferEncoding, cb: TransformCallback): void { + console.log("transform", chunk.toString()); + cb(null, 42); + }, +}); +t.on("error", (e: Error & { code?: string }) => console.log("error", e.name, e.code, e.message)); +t.on("data", (d: Buffer) => console.log("data", d.toString())); +t.on("close", () => console.log("close", t.destroyed)); +console.log("write", t.write("a")); +const f = new Transform({ + transform(chunk: Buffer, _enc: BufferEncoding, cb: TransformCallback): void { cb(null, chunk); }, + flush(cb: TransformCallback): void { cb(null, { tail: true }); }, +}); +f.on("error", (e: Error & { code?: string }) => console.log("flush error", e.code, e.message)); +f.on("data", (d: Buffer) => console.log("flush data", d.toString())); +f.on("close", () => console.log("flush close")); +f.end("z"); +class Bad extends Transform { + _transform(chunk: Buffer, _enc: BufferEncoding, cb: TransformCallback): void { + cb(null, chunk.length); + } +} +const bad = new Bad(); +bad.on("error", (e: Error & { code?: string }) => console.log("error", e.code, e.message)); +bad.on("close", () => console.log("close")); +bad.write("abc"); +class Good extends Transform { + _transform(chunk: Buffer, _enc: BufferEncoding, cb: TransformCallback): void { + cb(null, chunk.toString().toUpperCase()); + } +} +const good = new Good(); +good.on("data", (d: Buffer) => console.log("good", d.toString())); +good.end("xyz"); diff --git a/tests/corpus/7466-stream-construct-assign.mjs b/tests/corpus/7466-stream-construct-assign.mjs new file mode 100644 index 000000000..37684afcd --- /dev/null +++ b/tests/corpus/7466-stream-construct-assign.mjs @@ -0,0 +1,46 @@ +// Assigning _construct after construction: Node reads it on the +// construction tick, so an assignment before that tick replaces the +// scheduled hook, and one after it, or on a stream without a hook, has no +// effect. +import { Readable, Writable } from "node:stream"; + +const w = new Writable({ + construct(cb) { + console.log("original construct"); + cb(); + }, + write(chunk, _enc, cb) { + console.log("write", chunk.toString()); + cb(); + }, +}); +w._construct = function (cb) { + console.log("replacement construct"); + cb(); +}; +w.write("a"); +w.end(); +w.on("finish", () => { + w._construct = function (cb) { + console.log("late construct"); + cb(); + }; + console.log("finish"); + readable(); +}); + +// A stream without a construct hook: the assignment has no effect. +function readable() { + const r = new Readable({ + read() { + this.push("x"); + this.push(null); + }, + }); + r._construct = function (cb) { + console.log("never"); + cb(); + }; + r.on("data", (d) => console.log("data", d.toString())); + r.on("end", () => console.log("end")); +} diff --git a/tests/corpus/7500-pipeline-callback-undefined.ts b/tests/corpus/7500-pipeline-callback-undefined.ts new file mode 100644 index 000000000..e1503ff6b --- /dev/null +++ b/tests/corpus/7500-pipeline-callback-undefined.ts @@ -0,0 +1,120 @@ +// pipeline() with an async function or async generator destination: a +// typed callback whose error parameter admits undefined receives +// undefined when the pipeline succeeds, as in Node, and the error when it +// fails. +import { pipeline, Readable } from "node:stream"; + +function source(): Readable { + return Readable.from(["a", "b", "c"]); +} + +async function join(src: AsyncIterable): Promise { + let s = ""; + for await (const c of src) s += c; + return s; +} + +function optional(): Promise { + return new Promise((resolve) => { + pipeline(source(), join, (err?: Error | null, value?: string) => { + console.log("optional", err, err === undefined, value); + resolve(); + }); + }); +} + +function declared(): Promise { + return new Promise((resolve) => { + pipeline( + source(), + async function (src: AsyncIterable) { + let n = 0; + for await (const c of src) n += c.length; + return n; + }, + (err: Error | null | undefined, value?: number) => { + console.log("declared", err, typeof err, value); + resolve(); + }, + ); + }); +} + +function unknownParam(): Promise { + return new Promise((resolve) => { + pipeline(source(), join, (err: unknown, value: unknown) => { + console.log("unknown", err, value); + resolve(); + }); + }); +} + +function rest(): Promise { + return new Promise((resolve) => { + pipeline( + source(), + async function* (src: AsyncIterable) { + for await (const c of src) yield c.toUpperCase(); + }, + async function (src: AsyncIterable) { + for await (const c of src) void c; + }, + (...args: unknown[]) => { + console.log("rest", args.length, args); + resolve(); + }, + ); + }); +} + +function generatorDestination(): Promise { + return new Promise((resolve) => { + pipeline( + source(), + async function* (src: AsyncIterable) { + for await (const c of src) yield c; + }, + (err: Error | null | undefined, value: unknown) => { + console.log("generator", err, value); + resolve(); + }, + ); + }); +} + +function named(): Promise { + return new Promise((resolve) => { + function done(err: Error | null | undefined, value: string | undefined): void { + console.log("named", err, value); + resolve(); + } + pipeline(source(), join, done); + }); +} + +function failing(): Promise { + return new Promise((resolve) => { + pipeline( + source(), + async function (src: AsyncIterable) { + for await (const c of src) if (c === "b") throw new Error("boom at " + c); + return "never"; + }, + (err?: Error | null, value?: string) => { + console.log("failing", err?.message, value); + resolve(); + }, + ); + }); +} + +async function main(): Promise { + await optional(); + await declared(); + await unknownParam(); + await rest(); + await generatorDestination(); + await named(); + await failing(); +} +void main(); diff --git a/tests/corpus/7501-pipeline-callback-undefined.mjs b/tests/corpus/7501-pipeline-callback-undefined.mjs new file mode 100644 index 000000000..c51738e77 --- /dev/null +++ b/tests/corpus/7501-pipeline-callback-undefined.mjs @@ -0,0 +1,22 @@ +// pipeline() with an async function destination in JavaScript: the +// callback receives undefined and the destination's value on success. +import { pipeline, Readable } from "node:stream"; + +pipeline( + Readable.from(["x", "y"]), + async function (src) { + const out = []; + for await (const c of src) out.push(c); + return out.join("-"); + }, + (err, value) => { + console.log("callback", err, typeof err, value); + pipeline( + Readable.from(["z"]), + async function (src) { + for await (const c of src) void c; + }, + (...args) => console.log("no value", args), + ); + }, +); diff --git a/tests/corpus/7502-readline-cursor-undefined.ts b/tests/corpus/7502-readline-cursor-undefined.ts new file mode 100644 index 000000000..abac985cb --- /dev/null +++ b/tests/corpus/7502-readline-cursor-undefined.ts @@ -0,0 +1,28 @@ +// rl.cursor answers undefined on an interface that is not a terminal and +// the cursor position on a terminal interface. +import * as readline from "node:readline"; +import { PassThrough, Writable } from "node:stream"; + +const input = new PassThrough(); +const rl = readline.createInterface({ input }); +const c = rl.cursor; +console.log("cursor", c, typeof rl.cursor, rl.cursor === undefined); +console.log("arithmetic", rl.cursor + 1, Number.isNaN(rl.cursor * 2)); +console.log("fallback", rl.cursor ?? -1, `${rl.cursor}`); + +const interfaces = [rl]; +function first(): readline.Interface { + return interfaces[0]!; +} +console.log("call receiver", first().cursor); +rl.close(); + +const sink = new Writable({ + write(_chunk: Buffer, _enc, cb) { + cb(); + }, +}); +const terminal = readline.createInterface({ input: new PassThrough(), output: sink, terminal: true }); +terminal.write("abc"); +console.log("terminal", terminal.cursor, typeof terminal.cursor, terminal.cursor + 1); +terminal.close(); diff --git a/tests/corpus/7543-var-list-declarator-order.js b/tests/corpus/7543-var-list-declarator-order.js new file mode 100644 index 000000000..8fae06ff9 --- /dev/null +++ b/tests/corpus/7543-var-list-declarator-order.js @@ -0,0 +1,66 @@ +// A `var` read in a later declarator of the statement that stores it runs +// after the store; a read in the same or an earlier declarator sees undefined. +var base = { a: 1, b: "x" }, + more = { ...base, c: 2 }, + keys = Object.keys({ ...more }).join(","), + first = base.a + 1; +console.log(JSON.stringify(more), keys, first); + +function describeBase() { + return base.b + more.c; +} +var rules = { space: /\s+/, word: /\w+/ }, + merged = { ...rules, digit: /\d/ }, + described = describeBase(); +console.log(Object.keys(merged).join(","), merged.word.test("ab"), described); + +var table = { one: 1, two: 2 }, + lookup = function (key) { + return table[key]; + }, + arrow = (key) => ({ ...table })[key]; +console.log(lookup("one"), arrow("two")); + +var readLater = function () { + return later; + }, + early = readLater(), + later = { k: 1 }, + after = readLater().k; +console.log(early, after); + +var self = self ? 1 : 2; +console.log(self); + +for (var i = 0, j = i + 10; i < 2; i++) console.log(i, j); + +function nested() { + var inner = [1, 2, 3], + copy = [...inner, 4], + total = copy.reduce((sum, n) => sum + n, 0); + return { copy, total }; +} +console.log(JSON.stringify(nested())); + +function earlyInFunction() { + show("before"); + var point = { n: 1 }, + next = { m: point.n + 1 }; + show("after"); + function show(when) { + console.log(when, typeof point, point === undefined ? "none" : point.n, next === undefined ? "none" : next.m); + } + var check = () => console.log("check", items === undefined); + check(); + var items = [1]; + check(); +} +earlyInFunction(); + +function loopLists() { + for (var k = 0, sum = k; k < 3; k++) sum += k; + var doubled = sum * 2, + again = doubled + 1; + return [sum, doubled, again]; +} +console.log(JSON.stringify(loopLists())); diff --git a/tests/corpus/7545-string-regex-checked-arguments.js b/tests/corpus/7545-string-regex-checked-arguments.js new file mode 100644 index 000000000..a7ace9cdb --- /dev/null +++ b/tests/corpus/7545-string-regex-checked-arguments.js @@ -0,0 +1,89 @@ +// String search, match and replace with untyped pattern values: a RegExp +// is used as is, and any other value is converted as Node converts it. +const rules = JSON.parse('{"other":{}}'); +rules.other.space = /\s+/g; +rules.other.word = /o+/; +rules.other.text = "l"; +rules.other.dot = "."; +rules.other.none = undefined; +rules.other.nul = null; +rules.other.one = /(l)(o)/; +rules.other.many = /l+/g; +const t = " hello world"; + +console.log( + t.search(rules.other.word), + t.search(rules.other.text), + t.search(rules.other.none), + t.search(rules.other.nul), + t.search(rules.other.space), + "a.b".search(rules.other.dot), +); +console.log(rules.other.space.lastIndex); + +let conversions = 0; +rules.other.obj = { + toString() { + conversions++; + return "w"; + }, +}; +console.log(t.search(rules.other.obj), t.match(rules.other.obj)[0], conversions); + +const all = t.match(rules.other.many); +console.log(JSON.stringify(all), rules.other.many.lastIndex); +const pair = t.match(rules.other.one); +console.log(pair[0], pair[1], pair[2], pair.index, pair.length, pair.input === t); +const literal = t.match(rules.other.dot); +console.log(literal[0], literal.index, literal.length); +console.log(JSON.stringify(t.match(rules.other.none)), t.match(rules.other.zz)); +rules.other.zz = "zz"; +console.log(t.match(rules.other.zz)); +function first(text, re) { + const m = text.match(re); + return m ? m[0] : "none"; +} +console.log(first("abc", rules.other.dot), first("", rules.other.dot)); + +console.log(t.replace(rules.other.space, (m) => "[" + m.length + "]")); +console.log(t.replace(rules.other.word, (m, offset) => m.toUpperCase() + offset)); +console.log(t.replace(rules.other.text, (m, offset, subject) => "<" + m + offset + subject.length + ">")); +console.log(t.replaceAll(rules.other.text, (m) => m.toUpperCase())); +console.log(t.replaceAll(rules.other.space, () => "_")); +console.log(t.replace(rules.other.one, (m, p1, p2, offset) => p1 + "|" + p2 + "|" + offset)); +console.log(t.replace(rules.other.text, (m, offset, subject, extra) => typeof extra)); +console.log(t.replace(rules.other.text, (m) => m.length)); +try { + t.replaceAll(rules.other.word, (m) => m); +} catch (e) { + console.log(e.constructor.name, e.message); +} + +function tabs(text, width) { + let column = 0; + let out = ""; + for (const ch of text) { + if (ch === "\t") { + const pad = 4 - (column % 4); + out += " ".repeat(pad); + column += pad; + } else { + out += ch; + column++; + } + } + return out + width; +} +class Lexer { + constructor(table) { + this.rules = table; + } + indent(line, width) { + return line.replace(this.rules.leading, (w) => tabs(w, width)); + } + firstText(line) { + return line.search(this.rules.nonSpace); + } +} +const lexer = new Lexer({ leading: /^( {1,4}| {0,3}\t)/, nonSpace: /[^ ]/ }); +console.log(JSON.stringify(lexer.indent("\tabc", 2)), lexer.firstText(" x")); diff --git a/tests/corpus/7548-cjs-receiverless-builtin-calls/lib.cjs b/tests/corpus/7548-cjs-receiverless-builtin-calls/lib.cjs new file mode 100644 index 000000000..248e66270 --- /dev/null +++ b/tests/corpus/7548-cjs-receiverless-builtin-calls/lib.cjs @@ -0,0 +1,10 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: !0 }); +exports.label = exports.describe = void 0; +const node_util_1 = require("node:util"); +const node_path_1 = require("node:path"); +function describe(name, size) { + return (0, node_util_1.format)("%s holds %d (%s)", name, size, (0, node_path_1.basename)("/a/b/c.txt")); +} +exports.describe = describe; +exports.label = "lib"; diff --git a/tests/corpus/7548-cjs-receiverless-builtin-calls/main.cjs b/tests/corpus/7548-cjs-receiverless-builtin-calls/main.cjs new file mode 100644 index 000000000..6fd0b78a9 --- /dev/null +++ b/tests/corpus/7548-cjs-receiverless-builtin-calls/main.cjs @@ -0,0 +1,19 @@ +"use strict"; +// tsc and esbuild output: the __esModule stamp spelled { value: !0 }, and +// calls of builtin module functions without a receiver, (0, ns.member)(...). +const lib = require("./lib.cjs"); +const dc = require("node:diagnostics_channel"); +const util = require("node:util"); +const os = require("node:os"); +const path = require("node:path"); +console.log(lib.label, lib.describe("cache", 3)); +const metrics = (0, dc.channel)("cache:metrics"); +console.log(metrics.name, metrics.hasSubscribers); +const seen = []; +dc.subscribe("cache:metrics", (message) => seen.push(message.key)); +metrics.publish({ key: "a" }); +console.log(seen.join(","), metrics.hasSubscribers); +const tracing = (0, dc.tracingChannel)("cache"); +console.log(tracing.hasSubscribers); +console.log((0, util.inspect)({ a: [1, 2] }), (0, util.isDeepStrictEqual)([1], [1])); +console.log((0, path.join)("a", "b"), typeof (0, os.platform)()); diff --git a/tests/corpus/7549-cjs-minified-esmodule-stamp/main.cjs b/tests/corpus/7549-cjs-minified-esmodule-stamp/main.cjs new file mode 100644 index 000000000..c619a847a --- /dev/null +++ b/tests/corpus/7549-cjs-minified-esmodule-stamp/main.cjs @@ -0,0 +1,7 @@ +"use strict"; +// Minifiers spell `true` as `!0`: Object.defineProperty(exports, +// "__esModule", { value: !0 }) is the same interop stamp as { value: true }. +const stamped = require("./stamped.cjs"); +const moduleStamped = require("./module-stamped.cjs"); +console.log(stamped.answer, stamped.twice(4), moduleStamped.label); +console.log(typeof stamped.twice, moduleStamped.label.length); diff --git a/tests/corpus/7549-cjs-minified-esmodule-stamp/module-stamped.cjs b/tests/corpus/7549-cjs-minified-esmodule-stamp/module-stamped.cjs new file mode 100644 index 000000000..96a2aae7b --- /dev/null +++ b/tests/corpus/7549-cjs-minified-esmodule-stamp/module-stamped.cjs @@ -0,0 +1,3 @@ +"use strict"; +Object.defineProperty(module.exports, "__esModule", { value: !0 }); +module.exports.label = "module"; diff --git a/tests/corpus/7549-cjs-minified-esmodule-stamp/stamped.cjs b/tests/corpus/7549-cjs-minified-esmodule-stamp/stamped.cjs new file mode 100644 index 000000000..cd3b9c400 --- /dev/null +++ b/tests/corpus/7549-cjs-minified-esmodule-stamp/stamped.cjs @@ -0,0 +1,5 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: !0 }); +exports.answer = void 0; +exports.answer = 42; +exports.twice = (n) => n * 2; diff --git a/tests/corpus/7553-var-for-in-of-zero-iterations.js b/tests/corpus/7553-var-for-in-of-zero-iterations.js new file mode 100644 index 000000000..1a439f715 --- /dev/null +++ b/tests/corpus/7553-var-for-in-of-zero-iterations.js @@ -0,0 +1,18 @@ +// A for-in or for-of loop stores its var binding only when it iterates: +// after a loop that ran zero times, closures read undefined. +function f(o) { + for (var key in o) {} + const read = () => typeof key; + const same = () => key === undefined; + return [read(), same()]; +} +console.log(JSON.stringify(f({})), JSON.stringify(f({ a: 1 }))); +function g(xs) { + for (var x of xs) {} + const show = () => [typeof x, String(x)]; + return show(); +} +console.log(JSON.stringify(g([])), JSON.stringify(g([4]))); +for (var top in {}) {} +const readTop = () => typeof top; +console.log(readTop()); diff --git a/tests/corpus/7620-js-class-view-descriptors.js b/tests/corpus/7620-js-class-view-descriptors.js new file mode 100644 index 000000000..a0dc42813 --- /dev/null +++ b/tests/corpus/7620-js-class-view-descriptors.js @@ -0,0 +1,91 @@ +// An instance of a JS class with public fields keeps its own properties in +// a property bag, and untyped code that calls a method on it sees a view +// built from that bag: data properties with their attributes, accessors, +// index-like keys (which come first, in ascending order), symbols, and +// properties added or removed after construction. Every refresh of the view +// must copy the bag exactly. +'use strict'; + +class Box { + label = 'box'; + count = 0; + constructor(n) { + this.count = n; + } + describe() { + return `${this.label}:${this.count}`; + } + bump() { + this.count++; + return this; + } +} + +function call(target, method, ...args) { + return target[method](...args); +} + +function put(target, key, value) { + target[key] = value; +} + +function drop(target, key) { + delete target[key]; +} + +function show(title, target) { + const lines = []; + for (const key of Reflect.ownKeys(target)) { + const d = Object.getOwnPropertyDescriptor(target, key); + const kind = 'value' in d ? `value=${String(d.value)} writable=${d.writable}` : `get=${typeof d.get} set=${typeof d.set}`; + lines.push(` ${String(key)}: ${kind} enumerable=${d.enumerable} configurable=${d.configurable}`); + } + console.log(`${title}\n${lines.join('\n')}`); +} + +const box = new Box(2); +Object.defineProperty(box, 'hidden', { value: 42, enumerable: false, writable: false, configurable: true }); +Object.defineProperty(box, 'computed', { + get() { + return this.count * 10; + }, + set(v) { + this.count = v / 10; + }, + enumerable: true, + configurable: true, +}); +Object.defineProperty(box, 'getterOnly', { get: () => 'g', enumerable: false, configurable: false }); +put(box, 2, 'two'); +put(box, 1, 'one'); +put(box, 'extra', 'x'); +const tag = Symbol('tag'); +put(box, tag, 'symbol'); + +console.log(call(box, 'describe')); +show('after construction', box); +console.log('keys', Object.keys(box).join(',')); + +call(box, 'bump'); +put(box, 'computed', 70); +drop(box, 'extra'); +put(box, 'later', true); +console.log(call(box, 'describe')); +show('after changes', box); +console.log('computed', box.computed, 'hidden', box.hidden, 'getterOnly', box.getterOnly); + +// Many views in a row see the same, current properties. +let total = 0; +for (let i = 0; i < 5; i++) { + call(box, 'bump'); + total += Object.keys(box).length; +} +console.log('total keys', total, call(box, 'describe')); + +const other = new Box(7); +console.log(call(other, 'describe'), Object.keys(other).join(',')); +try { + call(box, 'missing'); +} catch (error) { + console.log(error instanceof TypeError, error.message); +} diff --git a/tests/corpus/7661-node-call-modules/lib/entry.cjs b/tests/corpus/7661-node-call-modules/lib/entry.cjs new file mode 100644 index 000000000..3fa55af87 --- /dev/null +++ b/tests/corpus/7661-node-call-modules/lib/entry.cjs @@ -0,0 +1,14 @@ +"use strict"; +const { label } = require("./util"); +const settings = require("../settings.json"); + +exports.describe = (name) => `${label(name)} (${settings.mode})`; + +exports.hasOptional = () => { + try { + require("./not-shipped.cjs"); + return true; + } catch (error) { + return error.code !== "MODULE_NOT_FOUND"; + } +}; diff --git a/tests/corpus/7661-node-call-modules/lib/esm.mjs b/tests/corpus/7661-node-call-modules/lib/esm.mjs new file mode 100644 index 000000000..467987471 --- /dev/null +++ b/tests/corpus/7661-node-call-modules/lib/esm.mjs @@ -0,0 +1,5 @@ +import { factor } from "./factor.mjs"; + +export function twice(n) { + return n * factor; +} diff --git a/tests/corpus/7661-node-call-modules/lib/factor.mjs b/tests/corpus/7661-node-call-modules/lib/factor.mjs new file mode 100644 index 000000000..adb2e40cb --- /dev/null +++ b/tests/corpus/7661-node-call-modules/lib/factor.mjs @@ -0,0 +1 @@ +export const factor = 2; diff --git a/tests/corpus/7661-node-call-modules/lib/util.js b/tests/corpus/7661-node-call-modules/lib/util.js new file mode 100644 index 000000000..ea7711f79 --- /dev/null +++ b/tests/corpus/7661-node-call-modules/lib/util.js @@ -0,0 +1,5 @@ +// An ES module by the nearest package.json ("type": "module"), loaded with +// require() from a CommonJS module. +export function label(name) { + return `module of ${name}`; +} diff --git a/tests/corpus/7661-node-call-modules/main.ts b/tests/corpus/7661-node-call-modules/main.ts new file mode 100644 index 000000000..7f6f4ac76 --- /dev/null +++ b/tests/corpus/7661-node-call-modules/main.ts @@ -0,0 +1,12 @@ +// @requires-libnode +// The modules island calls load ship with the program: a module of the +// program and the modules it loads, and whole packages with their +// dependencies, including files a package reads or loads by computed names. +import { callSync } from "@scriptc/node"; + +console.log(callSync("./lib/entry.cjs", "describe", "scriptc")); +console.log(callSync("greeter", "greet", "world")); +console.log(callSync("greeter", "fromData")); +console.log(callSync("greeter", "byName", "shout", "computed")); +console.log(callSync("./lib/entry.cjs", "hasOptional")); +console.log(callSync("./lib/esm.mjs", "twice", 21)); diff --git a/tests/corpus/7661-node-call-modules/node_modules/greeter/data/message.txt b/tests/corpus/7661-node-call-modules/node_modules/greeter/data/message.txt new file mode 100644 index 000000000..8e4d1b0c2 --- /dev/null +++ b/tests/corpus/7661-node-call-modules/node_modules/greeter/data/message.txt @@ -0,0 +1 @@ +read from a file the package ships diff --git a/tests/corpus/7661-node-call-modules/node_modules/greeter/index.js b/tests/corpus/7661-node-call-modules/node_modules/greeter/index.js new file mode 100644 index 000000000..4e3f7bc55 --- /dev/null +++ b/tests/corpus/7661-node-call-modules/node_modules/greeter/index.js @@ -0,0 +1,8 @@ +"use strict"; +const fs = require("node:fs"); +const path = require("node:path"); +const shout = require("shout"); + +exports.greet = (who) => shout(`hello, ${who}`); +exports.fromData = () => fs.readFileSync(path.join(__dirname, "data", "message.txt"), "utf8").trim(); +exports.byName = (name, text) => require(name)(text); diff --git a/tests/corpus/7661-node-call-modules/node_modules/greeter/package.json b/tests/corpus/7661-node-call-modules/node_modules/greeter/package.json new file mode 100644 index 000000000..07b35f1af --- /dev/null +++ b/tests/corpus/7661-node-call-modules/node_modules/greeter/package.json @@ -0,0 +1 @@ +{ "name": "greeter", "version": "1.0.0", "main": "index.js", "dependencies": { "shout": "1.0.0" } } diff --git a/tests/corpus/7661-node-call-modules/node_modules/shout/index.js b/tests/corpus/7661-node-call-modules/node_modules/shout/index.js new file mode 100644 index 000000000..6401aa6da --- /dev/null +++ b/tests/corpus/7661-node-call-modules/node_modules/shout/index.js @@ -0,0 +1,2 @@ +"use strict"; +module.exports = (text) => `${text.toUpperCase()}!`; diff --git a/tests/corpus/7661-node-call-modules/node_modules/shout/package.json b/tests/corpus/7661-node-call-modules/node_modules/shout/package.json new file mode 100644 index 000000000..d32731e21 --- /dev/null +++ b/tests/corpus/7661-node-call-modules/node_modules/shout/package.json @@ -0,0 +1 @@ +{ "name": "shout", "version": "1.0.0", "main": "index.js" } diff --git a/tests/corpus/7661-node-call-modules/settings.json b/tests/corpus/7661-node-call-modules/settings.json new file mode 100644 index 000000000..42201c528 --- /dev/null +++ b/tests/corpus/7661-node-call-modules/settings.json @@ -0,0 +1 @@ +{ "mode": "shipped" } diff --git a/tests/corpus/7662-node-call-exit-events/listeners.cjs b/tests/corpus/7662-node-call-exit-events/listeners.cjs new file mode 100644 index 000000000..c21ad324f --- /dev/null +++ b/tests/corpus/7662-node-call-exit-events/listeners.cjs @@ -0,0 +1,13 @@ +"use strict"; +let beforeExit = 0; + +exports.register = () => { + process.on("beforeExit", (code) => { + beforeExit++; + console.log("beforeExit", code, beforeExit); + if (beforeExit === 1) setTimeout(() => console.log("work from beforeExit"), 1); + }); + process.on("exit", (code) => console.log("exit", code)); + process.exitCode = 4; + return true; +}; diff --git a/tests/corpus/7662-node-call-exit-events/main.ts b/tests/corpus/7662-node-call-exit-events/main.ts new file mode 100644 index 000000000..909550588 --- /dev/null +++ b/tests/corpus/7662-node-call-exit-events/main.ts @@ -0,0 +1,8 @@ +// @requires-libnode +// The program and the Node.js it loads end as one process: Node's +// 'beforeExit' and 'exit' run when the program ends, work that 'beforeExit' +// schedules runs first, and process.exitCode set in Node is the program's. +import { callSync } from "@scriptc/node"; + +console.log("registered", callSync("./listeners.cjs", "register")); +console.log("main done"); diff --git a/tests/diagnostics/stream-runtime-option-order.ts b/tests/diagnostics/stream-runtime-option-order.ts new file mode 100644 index 000000000..357907d5d --- /dev/null +++ b/tests/diagnostics/stream-runtime-option-order.ts @@ -0,0 +1,42 @@ +import { Writable } from "node:stream"; + +// Runtime decodeStrings and defaultEncoding values are read before the +// other option members are evaluated, while Node evaluates the members in +// order: a value after a member with effects refuses, and so does a value +// with effects after a member that reads a property those effects change. +let encoding = (process.env["STREAM_ENCODING"] ?? "utf8") as BufferEncoding; +let decode = process.env["STREAM_DECODE"] !== "0"; +function bump(): number { + encoding = "hex"; + decode = false; + return 16; +} +const sink = new Writable({ + highWaterMark: bump(), + defaultEncoding: encoding, + write(_chunk, _encoding, done) { + done(); + }, +}); +const other = new Writable({ + highWaterMark: bump(), + decodeStrings: decode, + write(_chunk, _encoding, done) { + done(); + }, +}); +const sizes = { hwm: 16 }; +function pickEncoding(): BufferEncoding { + sizes.hwm = 1; + return "hex"; +} +const third = new Writable({ + highWaterMark: sizes.hwm, + defaultEncoding: pickEncoding(), + write(_chunk, _encoding, done) { + done(); + }, +}); +sink.end(); +other.end(); +third.end(); diff --git a/tests/fixtures/node-island/gc-stress.ts b/tests/fixtures/node-island/gc-stress.ts new file mode 100644 index 000000000..8c71099d1 --- /dev/null +++ b/tests/fixtures/node-island/gc-stress.ts @@ -0,0 +1,24 @@ +// Full garbage collections in Node during island calls +// (tests/harness/node-island.test.ts): from the main stack, from deep native +// recursion, from async functions and from timers, with deep JavaScript +// recursion on Node's side. +import { call, callSync } from "@scriptc/node"; + +function nested(depth: number): number { + if (depth === 0) return callSync("./gc.cjs", "deep", 2000); + return nested(depth - 1) + 1; +} + +async function inFiber(n: number): Promise { + const later = await call("./gc.cjs", "later", n); + return later + callSync("./gc.cjs", "churn", 1); +} + +console.log("churn", callSync("./gc.cjs", "churn", 20)); +console.log("nested", nested(3000)); +console.log("deep", callSync("./gc.cjs", "deep", 8000)); +const results = await Promise.all([1, 2, 3, 4].map((n) => inFiber(n))); +console.log("fibers", results.join(",")); +setTimeout(() => { + console.log("timer", callSync("./gc.cjs", "churn", 5)); +}, 1); diff --git a/tests/fixtures/node-island/gc.cjs b/tests/fixtures/node-island/gc.cjs new file mode 100644 index 000000000..f8f5a32fa --- /dev/null +++ b/tests/fixtures/node-island/gc.cjs @@ -0,0 +1,41 @@ +"use strict"; +// Garbage-collection pressure inside Node island calls +// (tests/harness/node-island.test.ts): full collections while the call +// runs on Node's stack, at depth on both sides. +const v8 = require("node:v8"); +const vm = require("node:vm"); +v8.setFlagsFromString("--expose-gc"); +const gc = vm.runInNewContext("gc"); + +const kept = []; + +// Allocates `rounds` batches of objects that survive until the next batch, +// with a full collection after each. +exports.churn = (rounds) => { + let total = 0; + for (let r = 0; r < rounds; r++) { + const batch = []; + for (let i = 0; i < 100000; i++) batch.push({ i, s: `item ${i}`, tags: [i, i + 1] }); + kept[r % 4] = batch; + total += batch.length; + gc(); + } + return total; +}; + +// Recurses `depth` frames in JavaScript and collects at the bottom. +exports.deep = (depth) => { + if (depth === 0) { + gc(); + return 0; + } + const local = { depth, text: String(depth) }; + return 1 + exports.deep(depth - 1) + (local.depth > 0 ? 0 : 1); +}; + +// Collects after a timer, from Node's loop. +exports.later = async (n) => { + await new Promise((resolve) => setTimeout(resolve, 1)); + exports.churn(1); + return n; +}; diff --git a/tests/fixtures/node-island/helper.cjs b/tests/fixtures/node-island/helper.cjs index 903532b0c..92c4ac77b 100644 --- a/tests/fixtures/node-island/helper.cjs +++ b/tests/fixtures/node-island/helper.cjs @@ -1,3 +1,23 @@ "use strict"; exports.answer = () => 42; exports.echo = (value) => value; +// An error whose message cannot be read. +exports.unreadable = () => { + const error = new Error(); + Object.defineProperty(error, "message", { + get() { + throw Symbol("message"); + }, + }); + throw error; +}; +// A fork without an execPath of its own: in a program it would start the +// program again. +exports.forkSelf = () => { + try { + require("node:child_process").fork(__filename); + return "forked"; + } catch (error) { + return error.message; + } +}; diff --git a/tests/fixtures/node-island/lifecycle.ts b/tests/fixtures/node-island/lifecycle.ts index f18ec640d..262ab7a6b 100644 --- a/tests/fixtures/node-island/lifecycle.ts +++ b/tests/fixtures/node-island/lifecycle.ts @@ -16,4 +16,12 @@ if (mode !== "never") { console.log("mapped after", mapped()); } if (mode === "exit") process.exit(5); +if (mode === "fork") console.log(callSync("./helper.cjs", "forkSelf")); +if (mode === "unreadable") { + try { + callSync("./helper.cjs", "unreadable"); + } catch (error) { + console.log("caught", error instanceof Error, JSON.stringify((error as Error).message)); + } +} console.log("done"); diff --git a/tests/fixtures/node-island/worker.ts b/tests/fixtures/node-island/worker.ts new file mode 100644 index 000000000..c96196c8f --- /dev/null +++ b/tests/fixtures/node-island/worker.ts @@ -0,0 +1,18 @@ +// A call into Node from a worker thread (tests/harness/node-island.test.ts): +// Node islands run on the main thread only, so the worker's call throws. +import { Worker, isMainThread, parentPort } from "node:worker_threads"; +import { callSync } from "@scriptc/node"; + +if (isMainThread) { + console.log("main", callSync("./helper.cjs", "answer")); + const worker = new Worker(new URL(import.meta.url)); + worker.on("message", (message: string) => console.log("worker", message)); + worker.on("exit", (code: number) => console.log("exit", code)); +} else { + try { + callSync("./helper.cjs", "answer"); + parentPort!.postMessage("called"); + } catch (error) { + parentPort!.postMessage((error as Error).message); + } +} diff --git a/tests/fixtures/npm/cases/7546-minified-esm-rules/main.ts b/tests/fixtures/npm/cases/7546-minified-esm-rules/main.ts new file mode 100644 index 000000000..0c94acb0d --- /dev/null +++ b/tests/fixtures/npm/cases/7546-minified-esm-rules/main.ts @@ -0,0 +1,22 @@ +// A one-line minified ES module. Its module-level var statement spreads +// earlier declarators into later ones, its tokenizer reads regexes through +// an untyped rules table (search, match and replace with callbacks), and +// one function receives both typed and untyped link objects. +import { render } from "gtminimark"; + +const doc = [ + "# Title \\*one\\*", + "## Sub", + '[a](/x "T\\*")', + "[b](/y)", + '[r]: /ref "Ref"', + "[n]: /none", + "[R]", + "[n]", + "[z]", + "\tcode *em* line", + " three spaces *x* and *y*", + "plain words here", +].join("\n"); +console.log(render(doc)); +console.log(render(doc, { pedantic: true })); diff --git a/tests/fixtures/npm/cases/7547-minified-cjs-chunks/main.ts b/tests/fixtures/npm/cases/7547-minified-cjs-chunks/main.ts new file mode 100644 index 000000000..ad2303f47 --- /dev/null +++ b/tests/fixtures/npm/cases/7547-minified-cjs-chunks/main.ts @@ -0,0 +1,12 @@ +// An identifier-minified esbuild CommonJS bundle (lru-cache's shape): its +// __commonJS helper and chunk parameters lost their names, the chunks +// stamp __esModule with { value: !0 }, and they call node:util and +// node:diagnostics_channel functions in the (0, ns.member)(...) form. +import { Cache } from "gtminicache"; + +const cache = new Cache(2); +cache.set("a", 1).set("b", 2); +cache.get("a"); +cache.set("c", 3); +console.log(cache.keys().join(","), cache.get("b"), cache.get("c")); +console.log(cache.describe("cache")); diff --git a/tests/fixtures/npm/node_modules/gtminicache/dist/index.d.ts b/tests/fixtures/npm/node_modules/gtminicache/dist/index.d.ts new file mode 100644 index 000000000..5d12cad7e --- /dev/null +++ b/tests/fixtures/npm/node_modules/gtminicache/dist/index.d.ts @@ -0,0 +1,7 @@ +export declare class Cache { + constructor(max: number); + set(key: K, value: V): this; + get(key: K): V | undefined; + keys(): K[]; + describe(label: string): string; +} diff --git a/tests/fixtures/npm/node_modules/gtminicache/dist/index.js b/tests/fixtures/npm/node_modules/gtminicache/dist/index.js new file mode 100644 index 000000000..31c6deef9 --- /dev/null +++ b/tests/fixtures/npm/node_modules/gtminicache/dist/index.js @@ -0,0 +1 @@ +"use strict";var n=(s,e)=>()=>(e||s((e={exports:{}}).exports,e),e.exports);var u=n(i=>{"use strict";Object.defineProperty(i,"__esModule",{value:!0});i.metrics=void 0;var o=require("node:diagnostics_channel");i.metrics=(0,o.channel)("gtminicache:metrics")});var d=n(r=>{"use strict";Object.defineProperty(r,"__esModule",{value:!0});r.describe=a;var l=require("node:util");function a(s,e){return(0,l.format)("%s holds %d",s,e)}});Object.defineProperty(exports,"__esModule",{value:!0});exports.Cache=void 0;var h=u(),_=d(),c=class{#e=new Map;#t;constructor(e){this.#t=e}set(e,t){for(this.#e.delete(e),this.#e.set(e,t);this.#e.size>this.#t;)this.#e.delete(this.#e.keys().next().value);return h.metrics.hasSubscribers&&h.metrics.publish({key:e}),this}get(e){let t=this.#e.get(e);return t!==void 0&&(this.#e.delete(e),this.#e.set(e,t)),t}keys(){return[...this.#e.keys()]}describe(e){return(0,_.describe)(e,this.#e.size)}};exports.Cache=c; diff --git a/tests/fixtures/npm/node_modules/gtminicache/package.json b/tests/fixtures/npm/node_modules/gtminicache/package.json new file mode 100644 index 000000000..3f57e1ae3 --- /dev/null +++ b/tests/fixtures/npm/node_modules/gtminicache/package.json @@ -0,0 +1,7 @@ +{ + "name": "gtminicache", + "version": "1.0.0", + "description": "tsc-style CommonJS modules bundled by esbuild 0.27.0 with --bundle --platform=node --format=cjs --minify: the __commonJS helper and the chunk parameters lose their names, the __esModule stamps read { value: !0 }, and calls into node:util and node:diagnostics_channel use the (0, ns.member)(...) form", + "main": "./dist/index.js", + "types": "./dist/index.d.ts" +} diff --git a/tests/fixtures/npm/node_modules/gtminimark/dist/index.d.ts b/tests/fixtures/npm/node_modules/gtminimark/dist/index.d.ts new file mode 100644 index 000000000..295430b86 --- /dev/null +++ b/tests/fixtures/npm/node_modules/gtminimark/dist/index.d.ts @@ -0,0 +1 @@ +export declare function render(src: string, options?: { pedantic?: boolean }): string; diff --git a/tests/fixtures/npm/node_modules/gtminimark/dist/index.js b/tests/fixtures/npm/node_modules/gtminimark/dist/index.js new file mode 100644 index 000000000..c5e1daa31 --- /dev/null +++ b/tests/fixtures/npm/node_modules/gtminimark/dist/index.js @@ -0,0 +1 @@ +var m = { heading: /^(#{1,6}) +(.*)$/, link: /^\[([^\]]*)\]\(([^ )]*)(?: "([^"]*)")?\)/, ref: /^\[([^\]]*)\]$/, lead: /^( {1,4}| {0,3}\t)/, nonSpace: /[^ ]/, punct: /\\([\\*_\[\]])/g, word: /\w+/g, def: /^\[([^\]]*)\]: (\S+)(?: "([^"]*)")?$/ },B = { ...m, emph: /\*([^*]+)\*/g },P = { ...B, word: /[a-z]+/g },K = { normal: { other: m, inline: B }, pedantic: { other: m, inline: P, strict: !0 } };function X(a, e) { let t = 0, n = ""; for (let i of a) if (i === "\t") { let r = 4 - t % 4; n += " ".repeat(r), t += r } else n += i, t++; return n + e; }function Se(a, e, t, n) { let r = e.href, o = e.title || null; n.links++; return { type: "link", raw: t, href: r, title: o, text: a[1] }; }class T {constructor() { this.state = { links: 0 }; }heading(s) { let c = s.match(this.rules.other.heading); return c ? "h" + c[1].length + ":" + c[2].replace(this.rules.other.punct, "$1") : null; }indent(s, w) { return s.replace(this.rules.other.lead, W => X(W, w)); }first(s) { return s.search(this.rules.other.nonSpace); }emph(s) { return s.replace(this.rules.inline.emph, (W, x) => "" + x + ""); }words(s) { let c = s.match(this.rules.inline.word); return c ? c.length : 0; }link(s) { let c = this.rules.other.link.exec(s); if (!c) return null; let r = c[2], o = ""; o = c[3] ? c[3] : ""; return Se(c, { href: r && r.replace(this.rules.other.punct, "$1"), title: o && o.replace(this.rules.other.punct, "$1") }, c[0], this.state); }ref(s, l) { let c = this.rules.other.ref.exec(s); if (!c) return null; let o = l[c[1].toLowerCase()]; return o ? Se(c, o, c[0], this.state) : { type: "text", raw: c[0] }; }}class L {constructor(e) { this.options = e || {}; this.links = {}; this.tokenizer = new T(); this.tokenizer.rules = this.options.pedantic ? K.pedantic : K.normal; }define(s) { let c = s.match(this.tokenizer.rules.other.def); if (c) this.links[c[1].toLowerCase()] = { href: c[2], title: c[3] || "" }; return !!c; }line(s) {let t = this.tokenizer, h = t.heading(s);if (h) return h;let k = t.link(s);if (k) return JSON.stringify(k);let f = t.ref(s, this.links);if (f) return JSON.stringify(f);return t.first(s) + "|" + JSON.stringify(t.indent(s, t.first(s))) + "|" + t.emph(s) + "|" + t.words(s);}}export function render(src, options) {let l = new L(options), out = [];for (let s of src.split("\n")) if (!l.define(s)) out.push(l.line(s));return out.join("\n") + "\nlinks " + l.tokenizer.state.links;} diff --git a/tests/fixtures/npm/node_modules/gtminimark/package.json b/tests/fixtures/npm/node_modules/gtminimark/package.json new file mode 100644 index 000000000..ff188c682 --- /dev/null +++ b/tests/fixtures/npm/node_modules/gtminimark/package.json @@ -0,0 +1,8 @@ +{ + "name": "gtminimark", + "version": "1.0.0", + "description": "A one-line minified ES module: a module-level var statement whose later declarators spread earlier ones, regexes read through an untyped rules table, and a function called with typed and untyped link objects", + "type": "module", + "main": "./dist/index.js", + "types": "./dist/index.d.ts" +} diff --git a/tests/harness/__snapshots__/stream-runtime-option-order.ts.txt b/tests/harness/__snapshots__/stream-runtime-option-order.ts.txt new file mode 100644 index 000000000..ff7f10be1 --- /dev/null +++ b/tests/harness/__snapshots__/stream-runtime-option-order.ts.txt @@ -0,0 +1,26 @@ +stream-runtime-option-order.ts:16:3 - error SC2020: 'the Writable option 'defaultEncoding' with a runtime value that an earlier option value could observe or change' is part of the standard library types but has no scriptc lowering yet + + 15 | highWaterMark: bump(), + 16 | defaultEncoding: encoding, + | ^~~~~~~~~~~~~~~~~~~~~~~~~ + 17 | write(_chunk, _encoding, done) { + + hint: compute the option values into consts before the options object (the runtime value is read before the other options run) + +stream-runtime-option-order.ts:23:3 - error SC2020: 'the Writable option 'decodeStrings' with a runtime value that an earlier option value could observe or change' is part of the standard library types but has no scriptc lowering yet + + 22 | highWaterMark: bump(), + 23 | decodeStrings: decode, + | ^~~~~~~~~~~~~~~~~~~~~ + 24 | write(_chunk, _encoding, done) { + + hint: compute the option values into consts before the options object (the runtime value is read before the other options run) + +stream-runtime-option-order.ts:35:3 - error SC2020: 'the Writable option 'defaultEncoding' with a runtime value that an earlier option value could observe or change' is part of the standard library types but has no scriptc lowering yet + + 34 | highWaterMark: sizes.hwm, + 35 | defaultEncoding: pickEncoding(), + | ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + 36 | write(_chunk, _encoding, done) { + + hint: compute the option values into consts before the options object (the runtime value is read before the other options run) \ No newline at end of file diff --git a/tests/harness/ci-stack-scope.test.ts b/tests/harness/ci-stack-scope.test.ts index 4abd321d0..9408433c4 100644 --- a/tests/harness/ci-stack-scope.test.ts +++ b/tests/harness/ci-stack-scope.test.ts @@ -62,6 +62,22 @@ test.each([ ["bottom layer", "pull_request", stacked("main", 1), true], ["new bottom after a merge", "pull_request", stacked("main", 2), true], ["upper layer", "pull_request", upper, false], + [ + "a PR into the default branch whose stack names another base", + "pull_request", + { + ...stacked("main"), + repository: { full_name: repository, default_branch: "main" }, + pull_request: { ...stacked("main").pull_request, stack: { base: { ref: "ni/base" } } }, + }, + true, + ], + [ + "upper layer with the default branch known", + "pull_request", + { ...upper, repository: { full_name: repository, default_branch: "main" } }, + false, + ], ["main push", "push", {}, true], ["push with stack metadata", "push", upper, true], ["merge queue group", "merge_group", upper, true], diff --git a/tests/harness/corpus-costs/7447-union-record-arm-width.mjs.json b/tests/harness/corpus-costs/7447-union-record-arm-width.mjs.json new file mode 100644 index 000000000..5648bd255 --- /dev/null +++ b/tests/harness/corpus-costs/7447-union-record-arm-width.mjs.json @@ -0,0 +1 @@ +{ "plain": 5785, "san": 5722 } diff --git a/tests/harness/corpus-costs/7448-union-record-arm-width.ts.json b/tests/harness/corpus-costs/7448-union-record-arm-width.ts.json new file mode 100644 index 000000000..c60a813c2 --- /dev/null +++ b/tests/harness/corpus-costs/7448-union-record-arm-width.ts.json @@ -0,0 +1 @@ +{ "plain": 3204, "san": 4052 } diff --git a/tests/harness/corpus-costs/7460-transform-writev.mjs.json b/tests/harness/corpus-costs/7460-transform-writev.mjs.json new file mode 100644 index 000000000..f6c21fd3d --- /dev/null +++ b/tests/harness/corpus-costs/7460-transform-writev.mjs.json @@ -0,0 +1 @@ +{ "plain": 2266, "san": 2991 } diff --git a/tests/harness/corpus-costs/7461-transform-writev.cjs.json b/tests/harness/corpus-costs/7461-transform-writev.cjs.json new file mode 100644 index 000000000..b5868c90b --- /dev/null +++ b/tests/harness/corpus-costs/7461-transform-writev.cjs.json @@ -0,0 +1 @@ +{ "plain": 1828, "san": 2413 } diff --git a/tests/harness/corpus-costs/7462-transform-writev.ts.json b/tests/harness/corpus-costs/7462-transform-writev.ts.json new file mode 100644 index 000000000..f8fd66c2a --- /dev/null +++ b/tests/harness/corpus-costs/7462-transform-writev.ts.json @@ -0,0 +1 @@ +{ "plain": 2178, "san": 2875 } diff --git a/tests/harness/corpus-costs/7463-stream-option-values.ts.json b/tests/harness/corpus-costs/7463-stream-option-values.ts.json new file mode 100644 index 000000000..0f66a4266 --- /dev/null +++ b/tests/harness/corpus-costs/7463-stream-option-values.ts.json @@ -0,0 +1 @@ +{ "plain": 1786, "san": 4192 } diff --git a/tests/harness/corpus-costs/7464-stream-option-values.mjs.json b/tests/harness/corpus-costs/7464-stream-option-values.mjs.json new file mode 100644 index 000000000..b609b128f --- /dev/null +++ b/tests/harness/corpus-costs/7464-stream-option-values.mjs.json @@ -0,0 +1 @@ +{ "plain": 1678, "san": 4502 } diff --git a/tests/harness/corpus-costs/7465-transform-callback-values.ts.json b/tests/harness/corpus-costs/7465-transform-callback-values.ts.json new file mode 100644 index 000000000..1c67bb43f --- /dev/null +++ b/tests/harness/corpus-costs/7465-transform-callback-values.ts.json @@ -0,0 +1 @@ +{ "plain": 1571, "san": 2074 } diff --git a/tests/harness/corpus-costs/7466-stream-construct-assign.mjs.json b/tests/harness/corpus-costs/7466-stream-construct-assign.mjs.json new file mode 100644 index 000000000..1c67bb43f --- /dev/null +++ b/tests/harness/corpus-costs/7466-stream-construct-assign.mjs.json @@ -0,0 +1 @@ +{ "plain": 1571, "san": 2074 } diff --git a/tests/harness/corpus-costs/7500-pipeline-callback-undefined.ts.json b/tests/harness/corpus-costs/7500-pipeline-callback-undefined.ts.json new file mode 100644 index 000000000..122c66b27 --- /dev/null +++ b/tests/harness/corpus-costs/7500-pipeline-callback-undefined.ts.json @@ -0,0 +1 @@ +{ "plain": 3783, "san": 4994 } diff --git a/tests/harness/corpus-costs/7501-pipeline-callback-undefined.mjs.json b/tests/harness/corpus-costs/7501-pipeline-callback-undefined.mjs.json new file mode 100644 index 000000000..d9c00a7de --- /dev/null +++ b/tests/harness/corpus-costs/7501-pipeline-callback-undefined.mjs.json @@ -0,0 +1 @@ +{ "plain": 2457, "san": 3243 } diff --git a/tests/harness/corpus-costs/7502-readline-cursor-undefined.ts.json b/tests/harness/corpus-costs/7502-readline-cursor-undefined.ts.json new file mode 100644 index 000000000..7a36db8f3 --- /dev/null +++ b/tests/harness/corpus-costs/7502-readline-cursor-undefined.ts.json @@ -0,0 +1 @@ +{ "plain": 2592, "san": 3421 } diff --git a/tests/harness/corpus-costs/7543-var-list-declarator-order.js.json b/tests/harness/corpus-costs/7543-var-list-declarator-order.js.json new file mode 100644 index 000000000..cc47657bd --- /dev/null +++ b/tests/harness/corpus-costs/7543-var-list-declarator-order.js.json @@ -0,0 +1 @@ +{ "plain": 4300, "san": 5900 } diff --git a/tests/harness/corpus-costs/7545-string-regex-checked-arguments.js.json b/tests/harness/corpus-costs/7545-string-regex-checked-arguments.js.json new file mode 100644 index 000000000..83b82ee18 --- /dev/null +++ b/tests/harness/corpus-costs/7545-string-regex-checked-arguments.js.json @@ -0,0 +1 @@ +{ "plain": 4900, "san": 4600 } diff --git a/tests/harness/corpus-costs/7548-cjs-receiverless-builtin-calls/main.cjs.json b/tests/harness/corpus-costs/7548-cjs-receiverless-builtin-calls/main.cjs.json new file mode 100644 index 000000000..8a603b2f4 --- /dev/null +++ b/tests/harness/corpus-costs/7548-cjs-receiverless-builtin-calls/main.cjs.json @@ -0,0 +1 @@ +{ "plain": 12300, "san": 13600 } diff --git a/tests/harness/corpus-costs/7549-cjs-minified-esmodule-stamp/main.cjs.json b/tests/harness/corpus-costs/7549-cjs-minified-esmodule-stamp/main.cjs.json new file mode 100644 index 000000000..d00a49ec8 --- /dev/null +++ b/tests/harness/corpus-costs/7549-cjs-minified-esmodule-stamp/main.cjs.json @@ -0,0 +1 @@ +{ "plain": 16000, "san": 21200 } diff --git a/tests/harness/corpus-costs/7553-var-for-in-of-zero-iterations.js.json b/tests/harness/corpus-costs/7553-var-for-in-of-zero-iterations.js.json new file mode 100644 index 000000000..566096d24 --- /dev/null +++ b/tests/harness/corpus-costs/7553-var-for-in-of-zero-iterations.js.json @@ -0,0 +1 @@ +{ "plain": 19400, "san": 25900 } diff --git a/tests/harness/corpus-costs/7620-js-class-view-descriptors.js.json b/tests/harness/corpus-costs/7620-js-class-view-descriptors.js.json new file mode 100644 index 000000000..ef439d096 --- /dev/null +++ b/tests/harness/corpus-costs/7620-js-class-view-descriptors.js.json @@ -0,0 +1 @@ +{ "plain": 3000, "san": 4500 } diff --git a/tests/harness/corpus-costs/7661-node-call-modules/main.ts.json b/tests/harness/corpus-costs/7661-node-call-modules/main.ts.json new file mode 100644 index 000000000..a970c2f16 --- /dev/null +++ b/tests/harness/corpus-costs/7661-node-call-modules/main.ts.json @@ -0,0 +1 @@ +{ "plain": 6000, "san": 9000 } diff --git a/tests/harness/corpus-costs/7662-node-call-exit-events/main.ts.json b/tests/harness/corpus-costs/7662-node-call-exit-events/main.ts.json new file mode 100644 index 000000000..a970c2f16 --- /dev/null +++ b/tests/harness/corpus-costs/7662-node-call-exit-events/main.ts.json @@ -0,0 +1 @@ +{ "plain": 6000, "san": 9000 } diff --git a/tests/harness/library-mode.test.ts b/tests/harness/library-mode.test.ts index 0d4052952..db4fabb3e 100644 --- a/tests/harness/library-mode.test.ts +++ b/tests/harness/library-mode.test.ts @@ -50,8 +50,8 @@ * no-teachings default is pinned by K5/K7 over * the teachings-free traps profile * K13 npm-posture bare npm specifiers are static-or-refuse: an - * eligible package (own .d.ts, unminified JS, - * no transform markers) compiles statically + * eligible package (own .d.ts, no transform + * markers) compiles statically * into the library graph, its own npm deps * judged by the same bar; every ineligible one * refuses SC4020 naming the failed bar — never @@ -746,11 +746,11 @@ describe("K9: library-mode refusals", () => { * synthesized per test like K9's. Emission never engages — SC4020 is a * frontend verdict — so one lane suffices. */ let npmRefusalCounter = 0; -async function npmRefusal( +async function compileNpmFixture( entryFile: string, exports: unknown[], profilePatch: Record = {}, -): Promise<{ code: string; message: string; hint?: string; note?: string; file: string }[]> { +): Promise>> { const outDir = join(cacheDir, `npm-refusal-${npmRefusalCounter++}`); mkdirSync(outDir, { recursive: true }); const profile = { @@ -770,7 +770,15 @@ async function npmRefusal( }; const profilePath = join(outDir, "profile.json"); writeFileSync(profilePath, JSON.stringify(profile)); - const result = await compileLibrary({ profilePath, outDir }); + return compileLibrary({ profilePath, outDir }); +} + +async function npmRefusal( + entryFile: string, + exports: unknown[], + profilePatch: Record = {}, +): Promise<{ code: string; message: string; hint?: string; note?: string; file: string }[]> { + const result = await compileNpmFixture(entryFile, exports, profilePatch); expect(result.ok).toBe(false); if (result.ok) throw new Error("unreachable"); return result.diagnostics.map((d) => ({ @@ -795,13 +803,11 @@ describe("K13: library-mode npm refusals (SC4020, static-or-refuse)", () => { expect(diags[0]!.file.endsWith("lib-markers.ts")).toBe(true); }); - test("minified shipped JS refuses", async () => { - const diags = await npmRefusal("lib-minified.ts", [ + test("minified shipped JS compiles", async () => { + const result = await compileNpmFixture("lib-minified.ts", [ { export: "f", symbol: "kx_f", params: ["f64", "f64"], returns: "f64" }, ]); - expect(diags.map((d) => d.code)).toEqual(["SC4020"]); - expect(diags[0]!.message).toContain("'slimmed'"); - expect(diags[0]!.message).toContain("looks minified"); + expect(result.ok ? [] : result.diagnostics.map((d) => d.message)).toEqual([]); }); test("no own .d.ts refuses by name — never the generic import fence", async () => { diff --git a/tests/harness/library-static-types.test.ts b/tests/harness/library-static-types.test.ts index dfbd70bf9..36e2befac 100644 --- a/tests/harness/library-static-types.test.ts +++ b/tests/harness/library-static-types.test.ts @@ -81,7 +81,8 @@ test.each(["sample", "@scope/sample"])( expect(actual).toBe(reference); expect(actual).toBe("22\n"); } - // Opting in to one runtime package does not admit its untyped dependency. + // A dependency the package's JavaScript imports compiles without + // declarations of its own: no declaration checks that import. const dependency = join(dir, "node_modules/unchecked-dependency"); await mkdir(dependency, { recursive: true }); await writeFile( @@ -96,14 +97,18 @@ test.each(["sample", "@scope/sample"])( join(pkg, "index.js"), "import { adjust } from 'unchecked-dependency'; export function calculate(value) { return adjust(value) * 3 + 1; }\n", ); - const refused = await compileLibrary({ profilePath, outDir: join(dir, "refused") }); - expect(refused.ok).toBe(false); - if (!refused.ok) - expect( - refused.diagnostics.some( - (d) => d.code === "SC4020" && d.message.includes("unchecked-dependency"), - ), - ).toBe(true); + const withDependency = await compileLibrary({ profilePath, outDir: join(dir, "dependency") }); + if (!withDependency.ok) throw new Error(JSON.stringify(withDependency.diagnostics)); + if (process.env["SCRIPTC_PORTABLE_ONLY"] !== "1") { + const source = join(dir, "dependency-host.c"), + binary = join(dir, "dependency-host"); + await writeFile( + source, + '#include \nextern void st_init(void); extern double st_result(void);\nint main(void) { st_init(); printf("%.0f\\n", st_result()); }\n', + ); + execFileSync("clang", [source, withDependency.archivePath, "-lm", "-o", binary]); + expect(execFileSync(binary, { encoding: "utf8" })).toBe("22\n"); + } expect(await readFile(join(types, "index.d.ts"), "utf8")).toBe(declaration); } finally { await rm(dir, { recursive: true, force: true }); diff --git a/tests/harness/node-island.test.ts b/tests/harness/node-island.test.ts index 47a68e290..132af41f0 100644 --- a/tests/harness/node-island.test.ts +++ b/tests/harness/node-island.test.ts @@ -6,8 +6,19 @@ * (6800-6803) pins the call semantics against Node. Needs a libnode pack * in SCRIPTC_LIBNODE; required in the Linux x64 CI lanes (libnode.ts). */ import { execFile } from "node:child_process"; -import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; -import { join } from "node:path"; +import { + chmodSync, + copyFileSync, + cpSync, + existsSync, + mkdirSync, + readFileSync, + readdirSync, + rmSync, + statSync, + writeFileSync, +} from "node:fs"; +import { basename, join } from "node:path"; import { promisify } from "node:util"; import { beforeAll, describe, expect, test } from "vitest"; import { compile, deserializeModule } from "@scriptc/compiler"; @@ -36,10 +47,13 @@ let binary = ""; const before = `mapped before ${process.platform === "linux" ? "false" : "unknown"}`; const after = `mapped after ${process.platform === "linux" ? "true" : "unknown"}`; +// The island module cache of these runs (scr_node_island.c). +const cacheHome = join(outDir, "cache"); + async function run( path: string, args: string[], - env: NodeJS.ProcessEnv = process.env, + env: NodeJS.ProcessEnv = { ...process.env, XDG_CACHE_HOME: cacheHome }, ): Promise { try { const { stdout, stderr } = await execFileAsync(path, args, { env, encoding: "utf8" }); @@ -61,11 +75,11 @@ function comparable(stderr: string): string { .join("\n"); } -async function build(name: string, libnode?: string): Promise { +async function build(name: string, libnode?: string, entry = fixture): Promise { const dir = join(outDir, name); mkdirSync(dir, { recursive: true }); - const result = await compile(fixture, { - outPath: join(dir, "lifecycle"), + const result = await compile(entry, { + outPath: join(dir, basename(entry, ".ts")), outDir: dir, sanitize, ...(libnode === undefined ? {} : { libnode }), @@ -125,18 +139,57 @@ describe.skipIf(skip)("Node islands in-process", () => { expect(result.stdout).toBe(`${before}\n42\n${after}\n`); }); - test("beside packaging carries its own pack", async () => { + test("beside packaging carries its own pack and island modules", async () => { const beside = await build("beside", "beside"); expect(existsSync(join(outDir, "beside/lifecycle.libnode/manifest.json"))).toBe(true); - const env = { ...process.env }; + const helper = join(outDir, "beside/lifecycle.libnode/modules/helper.cjs"); + expect(readFileSync(helper, "utf8")).toBe( + readFileSync(join(repoRoot, "tests/fixtures/node-island/helper.cjs"), "utf8"), + ); + const env = { ...process.env, XDG_CACHE_HOME: cacheHome }; delete env["SCRIPTC_LIBNODE"]; const result = await run(beside, [], env); expect(result.stdout).toBe(`${before}\n42\n${after}\ndone\n`); expect(result.code).toBe(0); + // Modules that no longer match the executable are refused. + writeFileSync(helper, `${readFileSync(helper, "utf8")}\n// edited\n`); + const edited = await run(beside, [], env); + expect(edited.code).toBe(1); + expect(edited.stderr).toBe( + `scriptc: the Node island modules at ${join(outDir, "beside/lifecycle.libnode/modules")} do not match this program\n`, + ); }, 300_000); + test("the executable carries its island modules and no path of the build machine", async () => { + const bytes = readFileSync(binary); + expect(bytes.includes(Buffer.from(repoRoot))).toBe(false); + // A copy elsewhere runs with the modules it carries, extracted into a + // private cache directory named by the manifest's digest. + const moved = join(outDir, "moved"); + mkdirSync(moved, { recursive: true }); + copyFileSync(binary, join(moved, "app")); + chmodSync(join(moved, "app"), 0o755); + const cache = join(outDir, "moved-cache"); + rmSync(cache, { recursive: true, force: true }); + const env = { ...process.env, XDG_CACHE_HOME: cache }; + expect((await run(join(moved, "app"), [], env)).stdout).toBe(`${before}\n42\n${after}\ndone\n`); + const islands = join(cache, "scriptc/node-islands"); + const [digest] = readdirSync(islands); + expect(digest).toMatch(/^[0-9a-f]{64}$/); + expect(statSync(islands).mode & 0o777).toBe(0o700); + expect(statSync(join(islands, digest!)).mode & 0o777).toBe(0o700); + // An entry that no longer matches is not used: the run extracts the + // modules into a directory of its own instead. + writeFileSync(join(islands, digest!, "helper.cjs"), "exports.answer = () => 0;\n"); + expect((await run(join(moved, "app"), [], env)).stdout).toBe(`${before}\n42\n${after}\ndone\n`); + }); + test("a missing pack fails the first call", async () => { - const result = await run(binary, [], { ...process.env, SCRIPTC_LIBNODE: "/nonexistent/pack" }); + const result = await run(binary, [], { + ...process.env, + XDG_CACHE_HOME: cacheHome, + SCRIPTC_LIBNODE: "/nonexistent/pack", + }); expect(result.code).toBe(1); expect(result.stdout).toBe(`${before}\n`); expect(result.stderr).toMatch( @@ -144,9 +197,89 @@ describe.skipIf(skip)("Node islands in-process", () => { ); }); + test("SCRIPTC_LIBNODE must be an absolute path", async () => { + const result = await run(binary, [], { + ...process.env, + XDG_CACHE_HOME: cacheHome, + SCRIPTC_LIBNODE: "relative/pack", + }); + expect(result.code).toBe(1); + expect(result.stdout).toBe(`${before}\n`); + expect(result.stderr).toBe( + "scriptc: SCRIPTC_LIBNODE must be an absolute path to a libnode pack directory\n", + ); + }); + + test("an error whose message cannot be read crosses with an empty message", async () => { + expect(await run(binary, ["unreadable"])).toEqual({ + stdout: `${before}\n42\n${after}\ncaught true ""\ndone\n`, + stderr: "", + code: 0, + }); + }); + + test("a call from a worker thread throws", async () => { + const worker = await build( + "worker", + undefined, + join(repoRoot, "tests/fixtures/node-island/worker.ts"), + ); + expect(await run(worker, [])).toEqual({ + stdout: + "main 42\nworker Node islands run on the main thread only; this call came from a worker thread\nexit 0\n", + stderr: "", + code: 0, + }); + }, 300_000); + + test("full garbage collections during calls, at depth on both sides", async () => { + const stress = await build( + "gc-stress", + undefined, + join(repoRoot, "tests/fixtures/node-island/gc-stress.ts"), + ); + expect(await run(stress, [])).toEqual({ + stdout: + "churn 2000000\nnested 5000\ndeep 8000\nfibers 100001,100002,100003,100004\ntimer 500000\n", + stderr: "", + code: 0, + }); + }, 300_000); + + test("Node ignores NODE_OPTIONS from the program's environment", async () => { + const result = await run(binary, [], { + ...process.env, + XDG_CACHE_HOME: cacheHome, + NODE_OPTIONS: "--require /nonexistent/preload.cjs", + }); + expect(result).toEqual({ stdout: `${before}\n42\n${after}\ndone\n`, stderr: "", code: 0 }); + }); + + test("child_process.fork() needs an execPath of its own", async () => { + expect((await run(binary, ["fork"])).stdout).toBe( + `${before}\n42\n${after}\nchild_process.fork() in a Node island needs options.execPath (a node executable): process.execPath is this program\ndone\n`, + ); + }); + + test("a cached build follows a change of the recorded libnode pack", async () => { + // The same program built against a copy of the pack elsewhere records + // the copy: the executable cache does not serve the first build's entry. + const copy = join(outDir, "pack-copy"); + rmSync(copy, { recursive: true, force: true }); + cpSync(pack!, copy, { recursive: true }); + const first = readFileSync(await build("cached-pack", pack!)); + const second = readFileSync(await build("cached-pack", copy)); + expect(first.includes(Buffer.from(copy))).toBe(false); + expect(second.includes(Buffer.from(copy))).toBe(true); + }, 300_000); + test("a pack of another Node version fails the first call", async () => { const dir = await fakePack("old-node", "SCR_NODE_BRIDGE_ABI", "22.11.0"); - const result = await run(binary, [], { ...process.env, SCRIPTC_LIBNODE: dir }); + const result = await run(binary, [], { + ...process.env, + XDG_CACHE_HOME: cacheHome, + SCRIPTC_LIBNODE: dir, + }); expect(result.code).toBe(1); expect(result.stderr).toMatch( new RegExp( @@ -157,10 +290,14 @@ describe.skipIf(skip)("Node islands in-process", () => { test("a bridge of another ABI fails the first call", async () => { const dir = await fakePack("other-abi", "999u", "24.22.0"); - const result = await run(binary, [], { ...process.env, SCRIPTC_LIBNODE: dir }); + const result = await run(binary, [], { + ...process.env, + XDG_CACHE_HOME: cacheHome, + SCRIPTC_LIBNODE: dir, + }); expect(result.code).toBe(1); expect(result.stderr).toBe( - `scriptc: the libnode pack at ${dir} has an incompatible bridge (ABI 999; this program needs 2)\n`, + `scriptc: the libnode pack at ${dir} has an incompatible bridge (ABI 999; this program needs 3)\n`, ); }); diff --git a/tests/harness/npm-static.test.ts b/tests/harness/npm-static.test.ts index 7ee76307f..388da2138 100644 --- a/tests/harness/npm-static.test.ts +++ b/tests/harness/npm-static.test.ts @@ -1490,6 +1490,24 @@ describe(`npm-static pilots${sanitize ? " (sanitized)" : ""}`, () => { ]); }, 180_000); + // 7546: a one-line minified ES module: a var statement whose later + // declarators spread earlier ones, regexes read through an untyped rules + // table, and a function instantiated with typed and untyped arguments. + test("the minified module 7546-minified-esm-rules compiles statically and matches Node", async () => { + await expectNodeParity(join(fixturesRoot, "npm/cases/7546-minified-esm-rules/main.ts"), [ + "gtminimark", + ]); + }, 180_000); + + // 7547: an identifier-minified esbuild CommonJS bundle: the __commonJS + // helper and chunk parameters renamed, { value: !0 } stamps, and + // (0, ns.member)(...) calls of builtin module functions. + test("the identifier-minified bundle 7547-minified-cjs-chunks compiles statically and matches Node", async () => { + await expectNodeParity(join(fixturesRoot, "npm/cases/7547-minified-cjs-chunks/main.ts"), [ + "gtminicache", + ]); + }, 180_000); + // A whitespace/syntax-minified bundle whose helpers keep their names is // judged by the recognizers, not refused up front as minified. test("--npm-static=auto opts a minified bundle with named helpers in", () => { diff --git a/tests/harness/self-hosting-native-toolchain.test.ts b/tests/harness/self-hosting-native-toolchain.test.ts index 091d3f0b5..fb15e80a4 100644 --- a/tests/harness/self-hosting-native-toolchain.test.ts +++ b/tests/harness/self-hosting-native-toolchain.test.ts @@ -108,7 +108,7 @@ for (const backend of ["llvm"] as const) { object_format: target.objectFormat, minimum_os: target.minimumOs, }, - runtime_abi: { version: 36, marker: "scr_runtime_abi_v36" }, + runtime_abi: { version: 37, marker: "scr_runtime_abi_v37" }, compiler: { command: "fixture", identity: "fixture", target: target.llvmTriple }, macros: { executable: [], excluded: [], sanitizer: "external-toolchain-required" }, flavors: { diff --git a/tests/library-mode/npm-refuse/lib-minified.ts b/tests/library-mode/npm-refuse/lib-minified.ts index 9e5c16e75..492cd5369 100644 --- a/tests/library-mode/npm-refuse/lib-minified.ts +++ b/tests/library-mode/npm-refuse/lib-minified.ts @@ -1,5 +1,5 @@ -// SC4020 fixture: slimmed ships a one-long-line dist — minified JS fails -// the eligibility bar. +// slimmed ships a one-long-line dist: minified JS compiles like any other +// eligible package. import { pick } from "slimmed"; export function f(a: number, b: number): number { diff --git a/tests/library-mode/npm-static/lib.ts b/tests/library-mode/npm-static/lib.ts index 27107f29d..15bad17a3 100644 --- a/tests/library-mode/npm-static/lib.ts +++ b/tests/library-mode/npm-static/lib.ts @@ -1,5 +1,5 @@ // Library-mode npm posture fixture (static-or-refuse): a bare specifier -// naming an ELIGIBLE package (own .d.ts, unminified shipped JS, no +// naming an ELIGIBLE package (own .d.ts, no // build-transform markers) compiles statically as part of the library // graph — no flag, no island — and the package's own top-level require // ("mathdep") joins through the same bar. The node:path import alongside diff --git a/tests/library-mode/npm-static/node_modules/mathkit/index.js b/tests/library-mode/npm-static/node_modules/mathkit/index.js index f9f9242c3..eadfefba9 100644 --- a/tests/library-mode/npm-static/node_modules/mathkit/index.js +++ b/tests/library-mode/npm-static/node_modules/mathkit/index.js @@ -1,6 +1,6 @@ "use strict"; // A vendored fake package for the library-mode npm posture: eligible by -// every bar (own .d.ts, unminified CJS, no build-transform markers), with +// every bar (own .d.ts, no build-transform markers), with // its OWN npm dep edge (mathdep) so the transitive fixpoint is exercised. const steps = require("mathdep");