From b5cb8b397acb2612fe90462e837cabcd629a4730 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Wed, 14 Oct 2026 20:58:41 -0500 Subject: [PATCH 1/7] Add the Readable operators, the Readable.from options and stream.compose The Readable operators (map, filter, forEach, toArray, some, find, every, flatMap, drop, take, reduce) run over string and Buffer chunk streams with Node's ordering, errors, and concurrency, highWaterMark and signal options. Readable.from reads arrays lazily and accepts the objectMode and highWaterMark options, and its JavaScript form follows from.js's checks. stream.compose returns a Duplex over a pipeline of stream instances. pipeline now follows pipeline.js: per-stream destroyers and eos watchers, and the callback on a tick once every watcher finished. 'finish' waits for a pending _final or _flush callback, and a failed synchronous write emits 'error' one tick later, as in Node. --- docs/content/docs/limitations.mdx | 2 +- .../node-v24-compatibility-meta.json | 2 +- .../src/generated/node-v24-compatibility.json | 94 +- .../generated/node-v24-backlog.json | 251 +- .../generated/node-v24-internal.json | 156 +- internal/compatibility/static-support.json | 3 + .../ambient/scriptc-node-fallback.d.ts | 57 + .../compiler/src/backend/llvm/lib-shared.ts | 26 + .../compiler/src/backend/llvm/lib-stream.ts | 21 + .../compiler/src/backend/native/executable.ts | 1 + .../lowering/class-dynamic-dispatch.ts | 52 +- .../lowering/lower-async-iteration.ts | 15 +- .../src/frontend/lowering/lower-calls.ts | 3 +- .../src/frontend/lowering/lower-stream.ts | 403 +++- .../compiler/src/frontend/lowering/lowerer.ts | 17 +- packages/compiler/src/frontend/tsc-codes.ts | 1 + packages/compiler/src/ir/builtin-effects.ts | 8 + .../compiler/src/ir/builtin-signatures.ts | 31 +- packages/compiler/src/ir/runtime-features.ts | 3 + packages/compiler/src/ir/validate.ts | 51 +- .../5335-stream-operators-values.ts.json | 6 + ...36-stream-operators-promise-timing.ts.json | 6 + .../5337-stream-operators-concurrency.ts.json | 6 + .../5338-stream-operators-checks.cjs.json | 6 + .../corpus/5339-stream-for-await-hops.ts.json | 6 + .../corpus/5340-readable-from-options.ts.json | 6 + .../5341-readable-from-dynamic.cjs.json | 6 + .../tests/corpus/5342-stream-compose.ts.json | 6 + .../corpus/5343-stream-compose-edges.ts.json | 6 + .../5344-stream-pipeline-timing.ts.json | 6 + .../runtime-pack-matrix.mjs | 2 + .../runtime-pack-matrix.mjs | 2 + packages/runtime/src/scr_json.c | 17 + packages/runtime/src/scr_runtime.h | 68 +- packages/runtime/src/scr_stream.c | 1278 ++++++++-- packages/runtime/src/scr_stream_ops.c | 2057 +++++++++++++++++ tests/corpus/5335-stream-operators-values.ts | 21 + .../5336-stream-operators-promise-timing.ts | 14 + .../5337-stream-operators-concurrency.ts | 40 + tests/corpus/5338-stream-operators-checks.cjs | 38 + tests/corpus/5339-stream-for-await-hops.ts | 17 + tests/corpus/5340-readable-from-options.ts | 48 + tests/corpus/5341-readable-from-dynamic.cjs | 34 + tests/corpus/5342-stream-compose.ts | 72 + tests/corpus/5343-stream-compose-edges.ts | 51 + tests/corpus/5344-stream-pipeline-timing.ts | 71 + 46 files changed, 4702 insertions(+), 385 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5335-stream-operators-values.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5336-stream-operators-promise-timing.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5337-stream-operators-concurrency.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5338-stream-operators-checks.cjs.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5339-stream-for-await-hops.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5340-readable-from-options.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5341-readable-from-dynamic.cjs.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5342-stream-compose.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5343-stream-compose-edges.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5344-stream-pipeline-timing.ts.json create mode 100644 packages/runtime/src/scr_stream_ops.c create mode 100644 tests/corpus/5335-stream-operators-values.ts create mode 100644 tests/corpus/5336-stream-operators-promise-timing.ts create mode 100644 tests/corpus/5337-stream-operators-concurrency.ts create mode 100644 tests/corpus/5338-stream-operators-checks.cjs create mode 100644 tests/corpus/5339-stream-for-await-hops.ts create mode 100644 tests/corpus/5340-readable-from-options.ts create mode 100644 tests/corpus/5341-readable-from-dynamic.cjs create mode 100644 tests/corpus/5342-stream-compose.ts create mode 100644 tests/corpus/5343-stream-compose-edges.ts create mode 100644 tests/corpus/5344-stream-pipeline-timing.ts diff --git a/docs/content/docs/limitations.mdx b/docs/content/docs/limitations.mdx index d398c86f0..c44f3c8eb 100644 --- a/docs/content/docs/limitations.mdx +++ b/docs/content/docs/limitations.mdx @@ -52,7 +52,7 @@ A declaration in the standard library does not guarantee that an API compiles. U - **Text encoding:** `TextEncoder` and `TextDecoder` can be stored in fields, arguments, return values, arrays, and closures. Encoding supports UTF-8, including `encodeInto` into a Uint8Array or Buffer view. Codec properties expose the canonical encoding and decoder flags. Decoding accepts recognized WHATWG labels, including runtime labels and omitted or undefined encodings. Literal constructor options support `fatal` and `ignoreBOM`; literal decode options support streaming UTF-8, UTF-16, and single-byte encodings. Other legacy streaming encodings, runtime options objects, inspection, JSON conversion, and module-scope declarations using `var` or lacking an initializer are unsupported. Use initialized `let` or `const` bindings for stored codecs. - **EventEmitter:** Literal event names use fixed argument tuples. Computed string names require a constant prefix or suffix that excludes `error`, meta events, and stream-internal events; their arguments must match the exact arity and support checked conversion. Unrestricted names, symbols, names that may reach internal events, and typed `listeners()`/`rawListeners()` results for names with possible computed registrations are unsupported. -- **Streams and readline:** Stream chunks are Buffers or strings. Object-mode streams of other values, `Readable.from` over arrays of other values, the `Readable` iterator helpers (`map`, `filter`, `toArray`, and the others), `stream.compose`, and generator stages in `pipeline` are unsupported. `pipeline` from `node:stream/promises` accepts a trailing `{ signal, end }` object, and `finished` accepts `readable`, `writable`, `error`, `cleanup`, and `signal` options, with boolean literals for the flags. Stream constructors accept a `signal` option in an options object literal, and `stream.addAbortSignal` accepts any stream. `node:readline` and `node:readline/promises` interfaces read `process.stdin` or a readable stream without terminal mode: `question`, `prompt`, `write`, `pause`, `resume`, `'line'`/`'close'`/`'pause'`/`'resume'`/`'error'` listeners, and `for await` compile. Terminal mode, `completer`, history options, and chained listener registration are unsupported. +- **Streams and readline:** Stream chunks are Buffers or strings. Object-mode streams of other values, `Readable.from` over arrays of other values or over generators and other iterables, and generator stages in `pipeline` are unsupported. `Readable.from` accepts `objectMode` and `highWaterMark` options. The `Readable` operators (`map`, `filter`, `forEach`, `toArray`, `some`, `find`, `every`, `flatMap`, `drop`, `take`, and `reduce`) accept `concurrency`, `highWaterMark`, and `signal` options; their callbacks take only the chunk, `map` returns strings or Buffers, and `flatMap` returns strings or arrays of them. `readable.compose` is unsupported. `stream.compose` accepts two or more streams, or one `Duplex`; functions, iterables, and web streams as stages are unsupported, and the `ERR_INVALID_ARG_VALUE` message for a stream that is no longer readable or writable names the stream's class instead of printing the stream. `pipeline` from `node:stream/promises` accepts a trailing `{ signal, end }` object, and `finished` accepts `readable`, `writable`, `error`, `cleanup`, and `signal` options, with boolean literals for the flags. Stream constructors accept a `signal` option in an options object literal, and `stream.addAbortSignal` accepts any stream. `node:readline` and `node:readline/promises` interfaces read `process.stdin` or a readable stream without terminal mode: `question`, `prompt`, `write`, `pause`, `resume`, `'line'`/`'close'`/`'pause'`/`'resume'`/`'error'` listeners, and `for await` compile. Terminal mode, `completer`, history options, and chained listener registration are unsupported. - **CommonJS metadata:** The module graph is fixed at build time. Cache deletion or reloading, metadata writes, `module.paths` mutation, and `require.extensions` are unsupported. Read-only access to `module.id`, `filename`, `path`, `paths`, `loaded`, `isPreloading`, `parent`, `children`, `require.main`, and `require.cache` lookup/enumeration compiles. - **ESM metadata:** Native `import.meta.url`, `filename`, `dirname`, and `main` support dot access, statically known keys, and destructuring selected fields. Storing or mutating the metadata object, rest bindings, runtime keys, and stored `resolve` callbacks are unsupported. Native `import.meta.resolve()` requires a literal specifier. Module instances distinguished by URL query strings or fragments are unsupported. - **Date:** Date unions, year/month constructors, setters, statically typed TypeScript identity comparisons, thrown Date values, and locale/string formatters are unsupported. Zero-argument and single number/string constructors, storage and passing, `getTime`, `valueOf`, `toISOString`, calendar getters, `getTimezoneOffset`, and single-string `Date.parse` compile. JavaScript Dates retain identity through checked storage and support Date-copy construction and JSON serialization. See [Dates and timezones](#dates-and-timezones) for parsing restrictions. diff --git a/docs/src/generated/node-v24-compatibility-meta.json b/docs/src/generated/node-v24-compatibility-meta.json index 8f5b36279..c1090e7d9 100644 --- a/docs/src/generated/node-v24-compatibility-meta.json +++ b/docs/src/generated/node-v24-compatibility-meta.json @@ -2,6 +2,6 @@ "schemaVersion": 3, "nodeVersion": "24.22.0", "nodeCommit": "f14cd3aa7bf9ca8df2cc5b8bb14beb0f335b1d93", - "artifactVersion": "7e9684fe4f97a27c6a16", + "artifactVersion": "0d70b859bd11aa1d4ffe", "rowCount": 3880 } diff --git a/docs/src/generated/node-v24-compatibility.json b/docs/src/generated/node-v24-compatibility.json index a6abba8e6..4f857252a 100644 --- a/docs/src/generated/node-v24-compatibility.json +++ b/docs/src/generated/node-v24-compatibility.json @@ -659,9 +659,9 @@ "apiEntries": 140, "static": { "supported": 0, - "partial": 38, + "partial": 53, "refused": 0, - "not-implemented": 102, + "not-implemented": 87, "by-design": 0, "unreviewed": 0, "not-applicable": 0 @@ -61525,9 +61525,9 @@ "inherited": true }, "static": { - "status": "not-implemented", - "detail": "No static compiler lowering is registered for this API yet.", - "verification": "registry-gap" + "status": "partial", + "detail": "Implemented for the documented static async-iteration subset.", + "verification": "test-backed" } }, { @@ -61640,9 +61640,9 @@ "inherited": true }, "static": { - "status": "not-implemented", - "detail": "No static compiler lowering is registered for this API yet.", - "verification": "registry-gap" + "status": "partial", + "detail": "Implemented for the documented static async-iteration subset.", + "verification": "test-backed" } }, { @@ -62031,9 +62031,9 @@ "inherited": true }, "static": { - "status": "not-implemented", - "detail": "No static compiler lowering is registered for this API yet.", - "verification": "registry-gap" + "status": "partial", + "detail": "Implemented for the documented static async-iteration subset.", + "verification": "test-backed" } }, { @@ -62146,9 +62146,9 @@ "inherited": true }, "static": { - "status": "not-implemented", - "detail": "No static compiler lowering is registered for this API yet.", - "verification": "registry-gap" + "status": "partial", + "detail": "Implemented for the documented static async-iteration subset.", + "verification": "test-backed" } }, { @@ -63388,9 +63388,9 @@ "inherited": false }, "static": { - "status": "not-implemented", - "detail": "No static compiler lowering is registered for this API yet.", - "verification": "registry-gap" + "status": "partial", + "detail": "Implemented for the documented static async-iteration subset.", + "verification": "test-backed" } }, { @@ -63411,9 +63411,9 @@ "inherited": false }, "static": { - "status": "not-implemented", - "detail": "No static compiler lowering is registered for this API yet.", - "verification": "registry-gap" + "status": "partial", + "detail": "Implemented for the documented static async-iteration subset.", + "verification": "test-backed" } }, { @@ -63434,9 +63434,9 @@ "inherited": false }, "static": { - "status": "not-implemented", - "detail": "No static compiler lowering is registered for this API yet.", - "verification": "registry-gap" + "status": "partial", + "detail": "Implemented for the documented static async-iteration subset.", + "verification": "test-backed" } }, { @@ -63457,9 +63457,9 @@ "inherited": false }, "static": { - "status": "not-implemented", - "detail": "No static compiler lowering is registered for this API yet.", - "verification": "registry-gap" + "status": "partial", + "detail": "Implemented for the documented static async-iteration subset.", + "verification": "test-backed" } }, { @@ -63480,9 +63480,9 @@ "inherited": false }, "static": { - "status": "not-implemented", - "detail": "No static compiler lowering is registered for this API yet.", - "verification": "registry-gap" + "status": "partial", + "detail": "Implemented for the documented static async-iteration subset.", + "verification": "test-backed" } }, { @@ -63503,9 +63503,9 @@ "inherited": false }, "static": { - "status": "not-implemented", - "detail": "No static compiler lowering is registered for this API yet.", - "verification": "registry-gap" + "status": "partial", + "detail": "Implemented for the documented static async-iteration subset.", + "verification": "test-backed" } }, { @@ -63526,9 +63526,9 @@ "inherited": false }, "static": { - "status": "not-implemented", - "detail": "No static compiler lowering is registered for this API yet.", - "verification": "registry-gap" + "status": "partial", + "detail": "Implemented for the documented static async-iteration subset.", + "verification": "test-backed" } }, { @@ -63549,9 +63549,9 @@ "inherited": false }, "static": { - "status": "not-implemented", - "detail": "No static compiler lowering is registered for this API yet.", - "verification": "registry-gap" + "status": "partial", + "detail": "Implemented for the documented static async-iteration subset.", + "verification": "test-backed" } }, { @@ -63572,9 +63572,9 @@ "inherited": false }, "static": { - "status": "not-implemented", - "detail": "No static compiler lowering is registered for this API yet.", - "verification": "registry-gap" + "status": "partial", + "detail": "Implemented for the documented static async-iteration subset.", + "verification": "test-backed" } }, { @@ -63595,9 +63595,9 @@ "inherited": false }, "static": { - "status": "not-implemented", - "detail": "No static compiler lowering is registered for this API yet.", - "verification": "registry-gap" + "status": "partial", + "detail": "Implemented for the documented static async-iteration subset.", + "verification": "test-backed" } }, { @@ -63618,9 +63618,9 @@ "inherited": false }, "static": { - "status": "not-implemented", - "detail": "No static compiler lowering is registered for this API yet.", - "verification": "registry-gap" + "status": "partial", + "detail": "Implemented for the documented static async-iteration subset.", + "verification": "test-backed" } }, { diff --git a/internal/compatibility/generated/node-v24-backlog.json b/internal/compatibility/generated/node-v24-backlog.json index f184a371f..e5179e4be 100644 --- a/internal/compatibility/generated/node-v24-backlog.json +++ b/internal/compatibility/generated/node-v24-backlog.json @@ -8,8 +8,8 @@ "tiers": { "static": { "replace-refusal": 40, - "verify-gap": 2489, - "audit-partial": 870, + "verify-gap": 2474, + "audit-partial": 885, "classify": 2, "implement": 85 } @@ -61849,12 +61849,15 @@ "priority": "high", "tiers": { "static": { - "status": "not-implemented", - "action": "verify-gap", - "verification": "registry-gap", - "confidence": "low", - "source": "compiler-unmatched:stream", - "tests": [] + "status": "partial", + "action": "audit-partial", + "verification": "test-backed", + "confidence": "high", + "source": "compiler-feature:stream.stream.compose", + "tests": [ + "tests/corpus/5342-stream-compose.ts", + "tests/corpus/5343-stream-compose-edges.ts" + ] } } }, @@ -61924,12 +61927,15 @@ "priority": "high", "tiers": { "static": { - "status": "not-implemented", - "action": "verify-gap", - "verification": "registry-gap", - "confidence": "low", - "source": "compiler-unmatched:stream", - "tests": [] + "status": "partial", + "action": "audit-partial", + "verification": "test-backed", + "confidence": "high", + "source": "compiler-feature:stream.stream.Readable.from", + "tests": [ + "tests/corpus/5340-readable-from-options.ts", + "tests/corpus/5341-readable-from-dynamic.cjs" + ] } } }, @@ -62438,12 +62444,15 @@ "priority": "high", "tiers": { "static": { - "status": "not-implemented", - "action": "verify-gap", - "verification": "registry-gap", - "confidence": "low", - "source": "compiler-unmatched:stream", - "tests": [] + "status": "partial", + "action": "audit-partial", + "verification": "test-backed", + "confidence": "high", + "source": "compiler-feature:stream.stream.compose", + "tests": [ + "tests/corpus/5342-stream-compose.ts", + "tests/corpus/5343-stream-compose-edges.ts" + ] } } }, @@ -62513,12 +62522,15 @@ "priority": "high", "tiers": { "static": { - "status": "not-implemented", - "action": "verify-gap", - "verification": "registry-gap", - "confidence": "low", - "source": "compiler-unmatched:stream", - "tests": [] + "status": "partial", + "action": "audit-partial", + "verification": "test-backed", + "confidence": "high", + "source": "compiler-feature:stream.stream.Readable.from", + "tests": [ + "tests/corpus/5340-readable-from-options.ts", + "tests/corpus/5341-readable-from-dynamic.cjs" + ] } } }, @@ -64019,12 +64031,17 @@ "priority": "low", "tiers": { "static": { - "status": "not-implemented", - "action": "verify-gap", - "verification": "registry-gap", - "confidence": "low", - "source": "compiler-unmatched:stream", - "tests": [] + "status": "partial", + "action": "audit-partial", + "verification": "test-backed", + "confidence": "high", + "source": "compiler-feature:stream.readable.map", + "tests": [ + "tests/corpus/5335-stream-operators-values.ts", + "tests/corpus/5336-stream-operators-promise-timing.ts", + "tests/corpus/5337-stream-operators-concurrency.ts", + "tests/corpus/5338-stream-operators-checks.cjs" + ] } } }, @@ -64044,12 +64061,17 @@ "priority": "low", "tiers": { "static": { - "status": "not-implemented", - "action": "verify-gap", - "verification": "registry-gap", - "confidence": "low", - "source": "compiler-unmatched:stream", - "tests": [] + "status": "partial", + "action": "audit-partial", + "verification": "test-backed", + "confidence": "high", + "source": "compiler-feature:stream.readable.filter", + "tests": [ + "tests/corpus/5335-stream-operators-values.ts", + "tests/corpus/5336-stream-operators-promise-timing.ts", + "tests/corpus/5337-stream-operators-concurrency.ts", + "tests/corpus/5338-stream-operators-checks.cjs" + ] } } }, @@ -64069,12 +64091,17 @@ "priority": "low", "tiers": { "static": { - "status": "not-implemented", - "action": "verify-gap", - "verification": "registry-gap", - "confidence": "low", - "source": "compiler-unmatched:stream", - "tests": [] + "status": "partial", + "action": "audit-partial", + "verification": "test-backed", + "confidence": "high", + "source": "compiler-feature:stream.readable.forEach", + "tests": [ + "tests/corpus/5335-stream-operators-values.ts", + "tests/corpus/5336-stream-operators-promise-timing.ts", + "tests/corpus/5337-stream-operators-concurrency.ts", + "tests/corpus/5338-stream-operators-checks.cjs" + ] } } }, @@ -64094,12 +64121,17 @@ "priority": "low", "tiers": { "static": { - "status": "not-implemented", - "action": "verify-gap", - "verification": "registry-gap", - "confidence": "low", - "source": "compiler-unmatched:stream", - "tests": [] + "status": "partial", + "action": "audit-partial", + "verification": "test-backed", + "confidence": "high", + "source": "compiler-feature:stream.readable.toArray", + "tests": [ + "tests/corpus/5335-stream-operators-values.ts", + "tests/corpus/5336-stream-operators-promise-timing.ts", + "tests/corpus/5337-stream-operators-concurrency.ts", + "tests/corpus/5338-stream-operators-checks.cjs" + ] } } }, @@ -64119,12 +64151,17 @@ "priority": "low", "tiers": { "static": { - "status": "not-implemented", - "action": "verify-gap", - "verification": "registry-gap", - "confidence": "low", - "source": "compiler-unmatched:stream", - "tests": [] + "status": "partial", + "action": "audit-partial", + "verification": "test-backed", + "confidence": "high", + "source": "compiler-feature:stream.readable.some", + "tests": [ + "tests/corpus/5335-stream-operators-values.ts", + "tests/corpus/5336-stream-operators-promise-timing.ts", + "tests/corpus/5337-stream-operators-concurrency.ts", + "tests/corpus/5338-stream-operators-checks.cjs" + ] } } }, @@ -64144,12 +64181,17 @@ "priority": "low", "tiers": { "static": { - "status": "not-implemented", - "action": "verify-gap", - "verification": "registry-gap", - "confidence": "low", - "source": "compiler-unmatched:stream", - "tests": [] + "status": "partial", + "action": "audit-partial", + "verification": "test-backed", + "confidence": "high", + "source": "compiler-feature:stream.readable.find", + "tests": [ + "tests/corpus/5335-stream-operators-values.ts", + "tests/corpus/5336-stream-operators-promise-timing.ts", + "tests/corpus/5337-stream-operators-concurrency.ts", + "tests/corpus/5338-stream-operators-checks.cjs" + ] } } }, @@ -64169,12 +64211,17 @@ "priority": "low", "tiers": { "static": { - "status": "not-implemented", - "action": "verify-gap", - "verification": "registry-gap", - "confidence": "low", - "source": "compiler-unmatched:stream", - "tests": [] + "status": "partial", + "action": "audit-partial", + "verification": "test-backed", + "confidence": "high", + "source": "compiler-feature:stream.readable.every", + "tests": [ + "tests/corpus/5335-stream-operators-values.ts", + "tests/corpus/5336-stream-operators-promise-timing.ts", + "tests/corpus/5337-stream-operators-concurrency.ts", + "tests/corpus/5338-stream-operators-checks.cjs" + ] } } }, @@ -64194,12 +64241,17 @@ "priority": "low", "tiers": { "static": { - "status": "not-implemented", - "action": "verify-gap", - "verification": "registry-gap", - "confidence": "low", - "source": "compiler-unmatched:stream", - "tests": [] + "status": "partial", + "action": "audit-partial", + "verification": "test-backed", + "confidence": "high", + "source": "compiler-feature:stream.readable.flatMap", + "tests": [ + "tests/corpus/5335-stream-operators-values.ts", + "tests/corpus/5336-stream-operators-promise-timing.ts", + "tests/corpus/5337-stream-operators-concurrency.ts", + "tests/corpus/5338-stream-operators-checks.cjs" + ] } } }, @@ -64219,12 +64271,17 @@ "priority": "low", "tiers": { "static": { - "status": "not-implemented", - "action": "verify-gap", - "verification": "registry-gap", - "confidence": "low", - "source": "compiler-unmatched:stream", - "tests": [] + "status": "partial", + "action": "audit-partial", + "verification": "test-backed", + "confidence": "high", + "source": "compiler-feature:stream.readable.drop", + "tests": [ + "tests/corpus/5335-stream-operators-values.ts", + "tests/corpus/5336-stream-operators-promise-timing.ts", + "tests/corpus/5337-stream-operators-concurrency.ts", + "tests/corpus/5338-stream-operators-checks.cjs" + ] } } }, @@ -64244,12 +64301,17 @@ "priority": "low", "tiers": { "static": { - "status": "not-implemented", - "action": "verify-gap", - "verification": "registry-gap", - "confidence": "low", - "source": "compiler-unmatched:stream", - "tests": [] + "status": "partial", + "action": "audit-partial", + "verification": "test-backed", + "confidence": "high", + "source": "compiler-feature:stream.readable.take", + "tests": [ + "tests/corpus/5335-stream-operators-values.ts", + "tests/corpus/5336-stream-operators-promise-timing.ts", + "tests/corpus/5337-stream-operators-concurrency.ts", + "tests/corpus/5338-stream-operators-checks.cjs" + ] } } }, @@ -64269,12 +64331,17 @@ "priority": "low", "tiers": { "static": { - "status": "not-implemented", - "action": "verify-gap", - "verification": "registry-gap", - "confidence": "low", - "source": "compiler-unmatched:stream", - "tests": [] + "status": "partial", + "action": "audit-partial", + "verification": "test-backed", + "confidence": "high", + "source": "compiler-feature:stream.readable.reduce", + "tests": [ + "tests/corpus/5335-stream-operators-values.ts", + "tests/corpus/5336-stream-operators-promise-timing.ts", + "tests/corpus/5337-stream-operators-concurrency.ts", + "tests/corpus/5338-stream-operators-checks.cjs" + ] } } }, diff --git a/internal/compatibility/generated/node-v24-internal.json b/internal/compatibility/generated/node-v24-internal.json index df0b892fa..cafca7ddb 100644 --- a/internal/compatibility/generated/node-v24-internal.json +++ b/internal/compatibility/generated/node-v24-internal.json @@ -716,9 +716,9 @@ "apiEntries": 140, "static": { "supported": 0, - "partial": 38, + "partial": 53, "refused": 0, - "not-implemented": 102, + "not-implemented": 87, "by-design": 0, "unreviewed": 0, "not-applicable": 0 @@ -85274,7 +85274,15 @@ "tests/corpus/5300-stream-add-abort-signal.ts", "tests/corpus/5301-stream-add-abort-signal-errors.cjs", "tests/corpus/5302-stream-promises-options.ts", - "tests/corpus/5303-stream-finished-callback-opts.cjs" + "tests/corpus/5303-stream-finished-callback-opts.cjs", + "tests/corpus/5335-stream-operators-values.ts", + "tests/corpus/5336-stream-operators-promise-timing.ts", + "tests/corpus/5337-stream-operators-concurrency.ts", + "tests/corpus/5338-stream-operators-checks.cjs", + "tests/corpus/5340-readable-from-options.ts", + "tests/corpus/5341-readable-from-dynamic.cjs", + "tests/corpus/5342-stream-compose.ts", + "tests/corpus/5343-stream-compose-edges.ts" ] }, "anchorSource": "chapter" @@ -85592,8 +85600,12 @@ "inherited": true }, "static": { - "status": "not-implemented", - "evidence": "compiler-unmatched:stream" + "status": "partial", + "evidence": "compiler-feature:stream.stream.compose", + "tests": [ + "tests/corpus/5342-stream-compose.ts", + "tests/corpus/5343-stream-compose-edges.ts" + ] }, "anchorSource": "exact" }, @@ -85661,8 +85673,12 @@ "inherited": true }, "static": { - "status": "not-implemented", - "evidence": "compiler-unmatched:stream" + "status": "partial", + "evidence": "compiler-feature:stream.stream.Readable.from", + "tests": [ + "tests/corpus/5340-readable-from-options.ts", + "tests/corpus/5341-readable-from-dynamic.cjs" + ] }, "anchorSource": "exact" }, @@ -86167,8 +86183,12 @@ "inherited": true }, "static": { - "status": "not-implemented", - "evidence": "compiler-unmatched:stream" + "status": "partial", + "evidence": "compiler-feature:stream.stream.compose", + "tests": [ + "tests/corpus/5342-stream-compose.ts", + "tests/corpus/5343-stream-compose-edges.ts" + ] }, "anchorSource": "ancestor" }, @@ -86236,8 +86256,12 @@ "inherited": true }, "static": { - "status": "not-implemented", - "evidence": "compiler-unmatched:stream" + "status": "partial", + "evidence": "compiler-feature:stream.stream.Readable.from", + "tests": [ + "tests/corpus/5340-readable-from-options.ts", + "tests/corpus/5341-readable-from-dynamic.cjs" + ] }, "anchorSource": "ancestor" }, @@ -87766,8 +87790,14 @@ "inherited": false }, "static": { - "status": "not-implemented", - "evidence": "compiler-unmatched:stream" + "status": "partial", + "evidence": "compiler-feature:stream.readable.map", + "tests": [ + "tests/corpus/5335-stream-operators-values.ts", + "tests/corpus/5336-stream-operators-promise-timing.ts", + "tests/corpus/5337-stream-operators-concurrency.ts", + "tests/corpus/5338-stream-operators-checks.cjs" + ] }, "anchorSource": "exact" }, @@ -87789,8 +87819,14 @@ "inherited": false }, "static": { - "status": "not-implemented", - "evidence": "compiler-unmatched:stream" + "status": "partial", + "evidence": "compiler-feature:stream.readable.filter", + "tests": [ + "tests/corpus/5335-stream-operators-values.ts", + "tests/corpus/5336-stream-operators-promise-timing.ts", + "tests/corpus/5337-stream-operators-concurrency.ts", + "tests/corpus/5338-stream-operators-checks.cjs" + ] }, "anchorSource": "exact" }, @@ -87812,8 +87848,14 @@ "inherited": false }, "static": { - "status": "not-implemented", - "evidence": "compiler-unmatched:stream" + "status": "partial", + "evidence": "compiler-feature:stream.readable.forEach", + "tests": [ + "tests/corpus/5335-stream-operators-values.ts", + "tests/corpus/5336-stream-operators-promise-timing.ts", + "tests/corpus/5337-stream-operators-concurrency.ts", + "tests/corpus/5338-stream-operators-checks.cjs" + ] }, "anchorSource": "exact" }, @@ -87835,8 +87877,14 @@ "inherited": false }, "static": { - "status": "not-implemented", - "evidence": "compiler-unmatched:stream" + "status": "partial", + "evidence": "compiler-feature:stream.readable.toArray", + "tests": [ + "tests/corpus/5335-stream-operators-values.ts", + "tests/corpus/5336-stream-operators-promise-timing.ts", + "tests/corpus/5337-stream-operators-concurrency.ts", + "tests/corpus/5338-stream-operators-checks.cjs" + ] }, "anchorSource": "exact" }, @@ -87858,8 +87906,14 @@ "inherited": false }, "static": { - "status": "not-implemented", - "evidence": "compiler-unmatched:stream" + "status": "partial", + "evidence": "compiler-feature:stream.readable.some", + "tests": [ + "tests/corpus/5335-stream-operators-values.ts", + "tests/corpus/5336-stream-operators-promise-timing.ts", + "tests/corpus/5337-stream-operators-concurrency.ts", + "tests/corpus/5338-stream-operators-checks.cjs" + ] }, "anchorSource": "exact" }, @@ -87881,8 +87935,14 @@ "inherited": false }, "static": { - "status": "not-implemented", - "evidence": "compiler-unmatched:stream" + "status": "partial", + "evidence": "compiler-feature:stream.readable.find", + "tests": [ + "tests/corpus/5335-stream-operators-values.ts", + "tests/corpus/5336-stream-operators-promise-timing.ts", + "tests/corpus/5337-stream-operators-concurrency.ts", + "tests/corpus/5338-stream-operators-checks.cjs" + ] }, "anchorSource": "exact" }, @@ -87904,8 +87964,14 @@ "inherited": false }, "static": { - "status": "not-implemented", - "evidence": "compiler-unmatched:stream" + "status": "partial", + "evidence": "compiler-feature:stream.readable.every", + "tests": [ + "tests/corpus/5335-stream-operators-values.ts", + "tests/corpus/5336-stream-operators-promise-timing.ts", + "tests/corpus/5337-stream-operators-concurrency.ts", + "tests/corpus/5338-stream-operators-checks.cjs" + ] }, "anchorSource": "exact" }, @@ -87927,8 +87993,14 @@ "inherited": false }, "static": { - "status": "not-implemented", - "evidence": "compiler-unmatched:stream" + "status": "partial", + "evidence": "compiler-feature:stream.readable.flatMap", + "tests": [ + "tests/corpus/5335-stream-operators-values.ts", + "tests/corpus/5336-stream-operators-promise-timing.ts", + "tests/corpus/5337-stream-operators-concurrency.ts", + "tests/corpus/5338-stream-operators-checks.cjs" + ] }, "anchorSource": "exact" }, @@ -87950,8 +88022,14 @@ "inherited": false }, "static": { - "status": "not-implemented", - "evidence": "compiler-unmatched:stream" + "status": "partial", + "evidence": "compiler-feature:stream.readable.drop", + "tests": [ + "tests/corpus/5335-stream-operators-values.ts", + "tests/corpus/5336-stream-operators-promise-timing.ts", + "tests/corpus/5337-stream-operators-concurrency.ts", + "tests/corpus/5338-stream-operators-checks.cjs" + ] }, "anchorSource": "exact" }, @@ -87973,8 +88051,14 @@ "inherited": false }, "static": { - "status": "not-implemented", - "evidence": "compiler-unmatched:stream" + "status": "partial", + "evidence": "compiler-feature:stream.readable.take", + "tests": [ + "tests/corpus/5335-stream-operators-values.ts", + "tests/corpus/5336-stream-operators-promise-timing.ts", + "tests/corpus/5337-stream-operators-concurrency.ts", + "tests/corpus/5338-stream-operators-checks.cjs" + ] }, "anchorSource": "exact" }, @@ -87996,8 +88080,14 @@ "inherited": false }, "static": { - "status": "not-implemented", - "evidence": "compiler-unmatched:stream" + "status": "partial", + "evidence": "compiler-feature:stream.readable.reduce", + "tests": [ + "tests/corpus/5335-stream-operators-values.ts", + "tests/corpus/5336-stream-operators-promise-timing.ts", + "tests/corpus/5337-stream-operators-concurrency.ts", + "tests/corpus/5338-stream-operators-checks.cjs" + ] }, "anchorSource": "exact" }, diff --git a/internal/compatibility/static-support.json b/internal/compatibility/static-support.json index 298b983ab..5fd8f6bd3 100644 --- a/internal/compatibility/static-support.json +++ b/internal/compatibility/static-support.json @@ -173,6 +173,9 @@ {"chapter": "readline", "symbols": ["readlinePromises.Interface", "readlinePromises.createInterface", "rl.question"], "status": "partial", "evidence": ["tests/corpus/5295-readline-promises-question.ts", "tests/corpus/5296-readline-lines.ts", "tests/corpus/5298-readline-abort.cjs", "tests/corpus/5299-readline-stdin.ts"]}, {"chapter": "readline", "symbols": ["readline.Interface", "rl.pause", "rl.resume", "rl.prompt", "rl.setPrompt", "rl.getPrompt", "rl.write", "rl", "'close'", "'error'", "'line'", "'pause'", "'resume'"], "status": "partial", "evidence": ["tests/corpus/5295-readline-promises-question.ts", "tests/corpus/5296-readline-lines.ts", "tests/corpus/5297-readline-callback-events.cjs", "tests/corpus/5299-readline-stdin.ts"]}, {"chapter": "stream", "symbols": ["stream.addAbortSignal"], "status": "partial", "evidence": ["tests/corpus/5300-stream-add-abort-signal.ts", "tests/corpus/5301-stream-add-abort-signal-errors.cjs"]}, + {"chapter": "stream", "symbols": ["readable.map", "readable.filter", "readable.forEach", "readable.toArray", "readable.some", "readable.find", "readable.every", "readable.flatMap", "readable.drop", "readable.take", "readable.reduce"], "status": "partial", "evidence": ["tests/corpus/5335-stream-operators-values.ts", "tests/corpus/5336-stream-operators-promise-timing.ts", "tests/corpus/5337-stream-operators-concurrency.ts", "tests/corpus/5338-stream-operators-checks.cjs"]}, + {"chapter": "stream", "symbols": ["stream.Readable.from"], "status": "partial", "evidence": ["tests/corpus/5340-readable-from-options.ts", "tests/corpus/5341-readable-from-dynamic.cjs"]}, + {"chapter": "stream", "symbols": ["stream.compose"], "status": "partial", "evidence": ["tests/corpus/5342-stream-compose.ts", "tests/corpus/5343-stream-compose-edges.ts"]}, {"chapter": "worker_threads", "symbols": ["Worker"], "status": "partial", "evidence": ["tests/corpus/native-worker-messages.ts", "tests/corpus/native-worker-commonjs.cjs", "tests/corpus/native-worker-environment.ts", "tests/corpus/native-worker-roots/main.ts"]}, {"chapter": "worker_threads", "symbols": ["MessagePort", "parentPort", "worker_threads.workerData", "isMainThread", "threadId"], "status": "partial", "evidence": ["tests/corpus/native-worker-commonjs.cjs", "tests/corpus/native-worker-port-lifetime.ts", "tests/corpus/native-worker-references.ts"]}, {"chapter": "worker_threads", "symbols": ["worker.postMessage", "port.postMessage"], "status": "partial", "evidence": ["tests/corpus/native-worker-message-graphs.ts", "tests/corpus/native-worker-shared-views.ts", "tests/corpus/native-worker-clone-errors.ts", "tests/corpus/native-worker-delayed-messages.ts"]}, diff --git a/packages/compiler/ambient/scriptc-node-fallback.d.ts b/packages/compiler/ambient/scriptc-node-fallback.d.ts index 166d7e57b..8d2d1172e 100644 --- a/packages/compiler/ambient/scriptc-node-fallback.d.ts +++ b/packages/compiler/ambient/scriptc-node-fallback.d.ts @@ -4873,6 +4873,14 @@ declare module "stream" { signal?: AbortSignal; } + /* The operators' options: concurrency and highWaterMark bound map's + * in-flight callbacks and queued results. */ + interface ArrayOptions { + concurrency?: number; + highWaterMark?: number; + signal?: AbortSignal; + } + class Readable extends EventEmitter { constructor(opts?: ReadableOptions); /* The underscore-method surface (@types/node declares these on the @@ -4903,6 +4911,51 @@ declare module "stream" { readonly destroyed: boolean; readonly errored: Error | null; static from(iterable: unknown, options?: ReadableOptions): Readable; + /* The Readable operators: chunks are strings or Buffers, map and + * flatMap callbacks return strings or Buffers (flatMap: arrays of + * them, or strings), and callbacks that declare the options argument + * are refused. */ + map( + fn: (data: any, options?: Pick) => any, + options?: ArrayOptions, + ): Readable; + filter( + fn: (data: any, options?: Pick) => boolean | Promise, + options?: ArrayOptions, + ): Readable; + forEach( + fn: (data: any, options?: Pick) => void | Promise, + options?: ArrayOptions, + ): Promise; + toArray(options?: Pick): Promise; + some( + fn: (data: any, options?: Pick) => boolean | Promise, + options?: ArrayOptions, + ): Promise; + find( + fn: (data: any, options?: Pick) => boolean | Promise, + options?: ArrayOptions, + ): Promise; + every( + fn: (data: any, options?: Pick) => boolean | Promise, + options?: ArrayOptions, + ): Promise; + flatMap( + fn: (data: any, options?: Pick) => any, + options?: ArrayOptions, + ): Readable; + drop(limit: number, options?: Pick): Readable; + take(limit: number, options?: Pick): Readable; + reduce( + fn: (previous: any, data: any, options?: Pick) => T, + initial?: undefined, + options?: Pick, + ): Promise; + reduce( + fn: (previous: T, data: any, options?: Pick) => T, + initial: T, + options?: Pick, + ): Promise; on(event: "data", listener: (chunk: any) => void): this; on(event: "end" | "close" | "readable" | "pause" | "resume", listener: () => void): this; on(event: "error", listener: (err: Error) => void): this; @@ -5119,6 +5172,9 @@ declare module "stream" { function addAbortSignal(signal: AbortSignal, stream: T): T; /* Declared so requires typecheck; each fences at its use site. */ function pipeline(...streams: unknown[]): unknown; + /* compose(...streams): a Duplex that writes to the first stream and + * reads from the last, with the streams running as a pipeline. */ + function compose(...streams: Array): Duplex; /* finished's options: readable/writable choose which ends count, * error: false ignores 'error' events, signal aborts the wait. */ interface FinishedOptions { @@ -5154,6 +5210,7 @@ declare module "stream" { Stream, addAbortSignal, pipeline, + compose, finished, getDefaultHighWaterMark, promises, diff --git a/packages/compiler/src/backend/llvm/lib-shared.ts b/packages/compiler/src/backend/llvm/lib-shared.ts index 55197ad8d..e6e5ca571 100644 --- a/packages/compiler/src/backend/llvm/lib-shared.ts +++ b/packages/compiler/src/backend/llvm/lib-shared.ts @@ -877,6 +877,19 @@ export const LIB_FN_SYMS: Record = { "readable.nextChunk": "scr_stream_next_chunk", "readable.nextChunkDyn": "scr_stream_next_chunk_dyn", "readable.fromArr": "scr_stream_from_arr", + "readable.fromDyn": "scr_stream_from_dyn", + // The Readable operators: borrowed operands; +1 stream or promise. + "readable.opMap": "scr_stream_op_map", + "readable.opFilter": "scr_stream_op_filter", + "readable.opFlatMap": "scr_stream_op_flat_map", + "readable.opDrop": "scr_stream_op_drop", + "readable.opTake": "scr_stream_op_take", + "readable.opToArray": "scr_stream_op_to_array", + "readable.opForEach": "scr_stream_op_for_each", + "readable.opSome": "scr_stream_op_some", + "readable.opEvery": "scr_stream_op_every", + "readable.opFind": "scr_stream_op_find", + "readable.opReduce": "scr_stream_op_reduce", "readable.pipe": "scr_stream_pipe", "writable.cork": "scr_stream_cork", "writable.uncork": "scr_stream_uncork", @@ -1240,6 +1253,18 @@ export const USES_TIMERS_LIB_FNS = new Set([ "readable.nextChunk", "readable.nextChunkDyn", "readable.fromArr", + "readable.fromDyn", + "readable.opMap", + "readable.opFilter", + "readable.opFlatMap", + "readable.opDrop", + "readable.opTake", + "readable.opToArray", + "readable.opForEach", + "readable.opSome", + "readable.opEvery", + "readable.opFind", + "readable.opReduce", "readable.resume", "readable.pipe", "readable.unpipe", @@ -1264,6 +1289,7 @@ export const USES_TIMERS_LIB_FNS = new Set([ "stream.finishedDyn", "stream.pipeline", "stream.pipelineDyn", + "stream.compose", "sp.finished", "sp.pipeline", "sp.pipelineOpts", diff --git a/packages/compiler/src/backend/llvm/lib-stream.ts b/packages/compiler/src/backend/llvm/lib-stream.ts index 4bf1df6fc..ce8ac6d6a 100644 --- a/packages/compiler/src/backend/llvm/lib-stream.ts +++ b/packages/compiler/src/backend/llvm/lib-stream.ts @@ -612,6 +612,27 @@ export function emitStreamLibCall(host: LlvmEmitterContext, e: LibCallExpr): LlV host.emitPendingCheck(); return out; } + if (e.fn === "stream.compose") { + // compose(count, s1..sn): the stream list rides a stack array. + const countArg = e.args[0]!; + if (countArg.kind !== "numLit") + throw new InternalCompilerError(`llvm emitter bug: ${e.fn} count not a literal`); + const n = countArg.value; + const args = e.args.map((a) => host.emitExpr(a)); + const arr = B.slot(); + B.entryAllocas.push(`${arr} = alloca [${n} x ptr]`); + for (let i = 0; i < n; i++) { + const p = B.tmp(); + B.line(`${p} = getelementptr inbounds [${n} x ptr], ptr ${arr}, i64 0, i64 ${i}`); + B.line(`store ptr ${args[1 + i]!.name}, ptr ${p}`); + } + const t = B.tmp(); + host.declare(`declare ptr @scr_stream_compose(double, ptr)`); + B.line(`${t} = call ptr @scr_stream_compose(double ${f64Lit(n)}, ptr ${arr})`); + const out = host.own({ name: t, type: e.type }); + host.emitPendingCheck(); + return out; + } if (e.fn === "stream.pipeline" || e.fn === "stream.pipelineDyn") { // pipeline(count, s1..sn, cb): the destination answers +1. The // stream list rides a stack array (the C compound literal). diff --git a/packages/compiler/src/backend/native/executable.ts b/packages/compiler/src/backend/native/executable.ts index 833fd2596..fbc1a5927 100644 --- a/packages/compiler/src/backend/native/executable.ts +++ b/packages/compiler/src/backend/native/executable.ts @@ -543,6 +543,7 @@ export async function compileCInternal( ? [rt(join(rtDir, "scr_util_callable.c"))] : []), ...(opts.stream ? [rt(join(rtDir, "scr_stream.c"))] : []), + ...(opts.stream && opts.dynAsync ? [rt(join(rtDir, "scr_stream_ops.c"))] : []), // The readiness-poller backends (scr_platform.h): kqueue on macOS/BSD, // epoll on Linux, WSAPoll on Windows — each TU is empty off its // platform, so all three link whenever a poller-using unit does and diff --git a/packages/compiler/src/frontend/lowering/class-dynamic-dispatch.ts b/packages/compiler/src/frontend/lowering/class-dynamic-dispatch.ts index fd2725868..1219b5fbb 100644 --- a/packages/compiler/src/frontend/lowering/class-dynamic-dispatch.ts +++ b/packages/compiler/src/frontend/lowering/class-dynamic-dispatch.ts @@ -58,13 +58,14 @@ import { import { classStaticData } from "./class-static-data.js"; import { isJsSourceFile } from "../program.js"; import * as ts from "../ts7/adapter.js"; -import { SYMBOL_T } from "../../ir/ir.js"; +import { SYMBOL_T, type IrLibFn } from "../../ir/ir.js"; import { isClassCallback } from "./class-callbacks.js"; import { CaughtValueDispatch } from "./caught-value-dispatch.js"; import { refreshDescriptorGuards } from "./class-descriptors.js"; import { ClassConstructionDispatch } from "./class-construction.js"; import { emitterRooted } from "./lower-event-emitter.js"; import { protocolArm, protocolFieldArm } from "./class-protocol-dispatch.js"; +import { STREAM_OPERATORS } from "./lower-stream.js"; const STREAM_METHODS = new Set([ "on", @@ -1088,7 +1089,12 @@ export class ClassDynamicDispatch { const matching = this.propertyReceivers(lowerer, true).flatMap(({ info }) => { if (!info || info.fields.has(method) || info.builtinError) return []; if (emitterRooted(lowerer, info) && EMITTER_METHODS.has(method)) return [info]; - if (info.builtinStream) return STREAM_METHODS.has(method) ? [info] : []; + if (info.builtinStream) { + return STREAM_METHODS.has(method) || + (STREAM_OPERATORS.has(method) && info.builtinStream !== "w") + ? [info] + : []; + } return findMethodOn(lowerer, info, method) || findGenericMethodOn(lowerer, info, method) ? [info] : []; @@ -3962,6 +3968,48 @@ export class ClassDynamicDispatch { type: method === "isPaused" ? BOOL : receiver.type, loc, }); + if (STREAM_OPERATORS.has(method) && info.builtinStream && info.builtinStream !== "w") { + // The Readable operators over a checked-dynamic receiver: every + // argument is already a checked value. + const arg = (i: number): IrExpr => incoming[i] ?? dynUndefinedExpr(loc); + const readableT: IrType = { kind: "object", className: "%Readable" }; + const op = (fn: string, args: IrExpr[], type: IrType): IrStmt[] => + ret({ kind: "libCall", fn: fn as IrLibFn, args: [receiver, ...args], type, loc }); + switch (method) { + case "map": + return op("readable.opMap", [arg(0), arg(1)], readableT); + case "filter": + return op("readable.opFilter", [arg(0), arg(1)], readableT); + case "flatMap": + return op("readable.opFlatMap", [arg(0), arg(1)], readableT); + case "drop": + return op("readable.opDrop", [arg(0), arg(1)], readableT); + case "take": + return op("readable.opTake", [arg(0), arg(1)], readableT); + case "toArray": + return op("readable.opToArray", [arg(0)], { kind: "promise", inner: DYN }); + case "forEach": + return op("readable.opForEach", [arg(0), arg(1)], { kind: "promise", inner: VOID }); + case "some": + return op("readable.opSome", [arg(0), arg(1)], { kind: "promise", inner: BOOL }); + case "every": + return op("readable.opEvery", [arg(0), arg(1)], { kind: "promise", inner: BOOL }); + case "find": + return op("readable.opFind", [arg(0), arg(1)], { kind: "promise", inner: DYN }); + case "reduce": + return op( + "readable.opReduce", + [ + arg(0), + arg(1), + { kind: "boolLit", value: dispatch.source.args.length >= 2, type: BOOL, loc }, + arg(2), + { kind: "numLit", value: 0, type: F64, loc }, + ], + { kind: "promise", inner: DYN }, + ); + } + } if (method === "destroy" && (!incoming.length || incoming[0]?.kind === "unitLit")) return ret({ kind: "libCall", diff --git a/packages/compiler/src/frontend/lowering/lower-async-iteration.ts b/packages/compiler/src/frontend/lowering/lower-async-iteration.ts index 62b7a89a5..fb6a8be22 100644 --- a/packages/compiler/src/frontend/lowering/lower-async-iteration.ts +++ b/packages/compiler/src/frontend/lowering/lower-async-iteration.ts @@ -315,6 +315,19 @@ function boolRef(localId: string, loc: ReturnType): IrExpr { return { kind: "varRef", localId, type: BOOL, loc }; } +/** The loop's stream: a JavaScript value typed as a stream can arrive as a + * checked-dynamic value (returned through an untyped function), which + * checks back into the stream class. */ +function streamSource( + lowerer: Lowerer, + plan: NodeReadablePlan, + loc: ReturnType, +): IrExpr { + const v = lowerer.lowerExpr(plan.source); + if (v.type.kind !== "dyn") return v; + return { kind: "dynCheck", value: v, type: plan.type, loc }; +} + function lowerNodeReadable( lowerer: Lowerer, stmt: ts.ForOfStatement, @@ -421,7 +434,7 @@ function lowerNodeReadable( return { kind: "block", body: [ - { kind: "varDecl", localId: stream.id, init: lowerer.lowerExpr(plan.source), loc }, + { kind: "varDecl", localId: stream.id, init: streamSource(lowerer, plan, loc), loc }, { kind: "varDecl", localId: active.id, diff --git a/packages/compiler/src/frontend/lowering/lower-calls.ts b/packages/compiler/src/frontend/lowering/lower-calls.ts index 11a59f70b..cd5f95295 100644 --- a/packages/compiler/src/frontend/lowering/lower-calls.ts +++ b/packages/compiler/src/frontend/lowering/lower-calls.ts @@ -142,6 +142,7 @@ import { } from "./lower-console.js"; import { STREAM_API_MEMBERS, + STREAM_OPERATORS, lowerStreamMethodCall, lowerStreamStaticCall, streamSidesOf, @@ -8488,7 +8489,7 @@ export function lowerObjectMethodCall( if ( info && !found && - STREAM_API_MEMBERS.has(access.name.text) && + (STREAM_API_MEMBERS.has(access.name.text) || STREAM_OPERATORS.has(access.name.text)) && streamSidesOf(lowerer, info) !== null ) { const stream = lowerStreamMethodCall(lowerer, call, access, info); diff --git a/packages/compiler/src/frontend/lowering/lower-stream.ts b/packages/compiler/src/frontend/lowering/lower-stream.ts index 7c8eb075f..c72913e1c 100644 --- a/packages/compiler/src/frontend/lowering/lower-stream.ts +++ b/packages/compiler/src/frontend/lowering/lower-stream.ts @@ -38,6 +38,8 @@ import { appendImplicitUndefinedReturn } from "./function-returns.js"; import { bufEncoding, knownBufEncoding } from "./containers/bytes.js"; import { tryLowerExpression } from "./expressions/try-lower-expression.js"; import { sideEffectFreeOptionValue } from "./surfaces.js"; +import { lowerDynObjectLiteral } from "./expressions/object-literals.js"; +import { dynUndefinedExpr } from "../../ir/build.js"; import { BOOL, DYN, @@ -1346,12 +1348,14 @@ export function lowerStreamUnderscoreAssign( /* ── the static surface (Readable.from) ───────────────────────────────── */ -/** `Readable.from(iterable)` — the array-seeded object-entry form: - * string[] / Buffer[] values (or literals) become a fully-seeded - * readable delivering one WHOLE chunk per element (Node's objectMode - * accounting; hwm 1), and a single string/Buffer argument is a - * one-chunk stream (Node's special case). Called from the method-call - * chain once the receiver resolved to a stream class value. */ +/** `Readable.from(iterable, options)`: string[] / Buffer[] sources + * become a readable that pushes one element per _read (whole entries in + * object mode, hwm 1), and a single string/Buffer argument is a + * one-chunk stream (from.js's special case, the mode's default hwm). The + * options lower when they are a literal of `objectMode` and + * `highWaterMark`; the JS lane and `any` sources take the runtime's + * checked-dynamic form. Called from the method-call chain once the + * receiver resolved to a stream class value. */ export function lowerStreamStaticCall( lowerer: Lowerer, call: ts.CallExpression, @@ -1410,49 +1414,97 @@ export function lowerStreamStaticCall( lowerer.noLowering(`${cls.slice(1)}.from`, call, "Readable.from is the lowered form"); } const args = call.arguments; - if (args.length !== 1) { + if (args.length > 2) { lowerer.noLowering( `Readable.from with ${args.length} arguments`, call, - "the one-argument form (an array, string, or Buffer) is supported", + "Readable.from(iterable, options) is the supported form", ); } - const srcT = lowerer.mapTypeOf(lowerer.typeOf(args[0]!)); const type: IrType = { kind: "object", className: "%Readable" }; - if ( - srcT?.kind === "array" && - (srcT.elem.kind === "string" || (srcT.elem.kind === "bytes" && srcT.elem.elem === "u8")) - ) { - const arr = lowerer.lowerExpr(args[0]!); - const strings = srcT.elem.kind === "string"; + const js = isJsSourceFile(call.getSourceFile()); + const srcNode = args[0]; + const optsNode = args[1]; + const srcT = srcNode ? lowerer.mapTypeOf(lowerer.typeOf(srcNode)) : null; + const typedElem = (t: IrType | null | undefined): "string" | "bytes" | null => + t?.kind === "string" ? "string" : t?.kind === "bytes" && t.elem === "u8" ? "bytes" : null; + const single = typedElem(srcT); + const elem = srcT?.kind === "array" ? typedElem(srcT.elem) : null; + const opts = optsNode ? fromOptions(lowerer, optsNode) : { flags: 0, hwm: null }; + if (srcNode && (single || elem) && opts) { + const v = lowerer.lowerExpr(srcNode); + const hwm = opts.hwm ? lowerer.lowerExprExpecting(opts.hwm, F64) : numLit(0, loc); + const arr: IrExpr = single ? { kind: "arrayLit", elems: [v], type: arrayOf(v.type), loc } : v; + // SCR_FROM_STRINGS 1, SCR_FROM_SINGLE 2 (scr_runtime.h); the options + // add SCR_FROM_BYTE_MODE 4 and SCR_FROM_HAS_HWM 8. + const flags = ((single ?? elem) === "string" ? 1 : 0) | (single ? 2 : 0) | opts.flags; return { kind: "libCall", fn: "readable.fromArr", - args: [arr, boolLit(strings, loc)], + args: [arr, numLit(flags, loc), hwm], type, loc, }; } - if (srcT?.kind === "string" || (srcT?.kind === "bytes" && srcT.elem === "u8")) { - // Node special-cases a single string/Buffer: ONE whole chunk. - const v = lowerer.lowerExpr(args[0]!); - const strings = srcT.kind === "string"; - const arr: IrExpr = { kind: "arrayLit", elems: [v], type: arrayOf(v.type), loc }; - return { - kind: "libCall", - fn: "readable.fromArr", - args: [arr, boolLit(strings, loc)], - type, - loc, - }; + const anySrc = + srcT?.kind === "dyn" || + (srcNode !== undefined && + srcT === null && + (lowerer.typeOf(srcNode).flags & ts.TypeFlags.Any) !== 0); + if (js || anySrc) { + // The checked-dynamic form: the runtime takes from.js's ladder + // (ERR_INVALID_ARG_TYPE for a non-iterable, the options, null values). + const src = srcNode ? lowerer.lowerExprExpecting(srcNode, DYN) : dynUndefinedExpr(loc); + const o = lowerOperatorValue(lowerer, optsNode, "Readable.from options", loc, true); + return { kind: "libCall", fn: "readable.fromDyn", args: [src, o], type, loc }; + } + if (srcNode && (single || elem)) { + lowerer.noLowering( + "Readable.from with these options", + optsNode!, + "an object literal with 'objectMode' (true or false) and a number 'highWaterMark' is supported", + ); } lowerer.noLowering( `Readable.from over a '${srcT ? lowerer.fmt(srcT) : "?"}' source`, - args[0]!, + srcNode ?? call, "string[] / Buffer[] arrays (and a single string or Buffer) are the supported sources — generators and async iterables have no lowering yet", ); } +/** Readable.from's typed options: an object literal of `objectMode` + * (a boolean literal) and `highWaterMark` (a number), as SCR_FROM_* flag + * bits and the hwm expression; null for any other shape. */ +function fromOptions( + lowerer: Lowerer, + node: ts.Expression, +): { flags: number; hwm: ts.Expression | null } | null { + if (!ts.isObjectLiteralExpression(node)) return null; + let flags = 0; + let hwm: ts.Expression | null = null; + for (const prop of node.properties) { + if ( + !ts.isPropertyAssignment(prop) || + !(ts.isIdentifier(prop.name) || ts.isStringLiteral(prop.name)) + ) { + return null; + } + const value = prop.initializer; + if (prop.name.text === "objectMode") { + if (value.kind === ts.SyntaxKind.FalseKeyword) flags |= 4; + else if (value.kind === ts.SyntaxKind.TrueKeyword) flags &= ~4; + else return null; + } else if (prop.name.text === "highWaterMark") { + if (lowerer.mapTypeOf(lowerer.typeOf(value))?.kind !== "f64") return null; + flags |= 8; + hwm = value; + } else { + return null; + } + } + return { flags, hwm }; +} + /** `const Writable = stream.Writable` — a runtime stream CLASS bound * through a namespace member: alias plumbing with no storage (both * declaration walks skip it; uses resolve through builtinStreamInfoOf's @@ -1491,7 +1543,7 @@ const PIPELINE_STAGE_EXPECTED = function lowerStreamArg( lowerer: Lowerer, node: ts.Expression, - what: "finished" | "pipeline" | "addAbortSignal", + what: "finished" | "pipeline" | "addAbortSignal" | "compose", ): IrExpr { const v = lowerer.lowerExpr(node); const info = v.type.kind === "object" ? lowerer.classes.get(v.type.className) : undefined; @@ -1546,7 +1598,7 @@ function lowerStreamArg( lowerer.noLowering( `${what} over a '${lowerer.fmt(v.type)}'`, node, - what === "pipeline" + what === "pipeline" || what === "compose" ? "stream stages are the lowered surface — Node also accepts iterables (strings, arrays), generators, plain functions, and { readable, writable } pairs, but those have no lowering yet" : "stream arguments are the lowered surface (iterables, generators, and plain functions have no lowering yet)", ); @@ -2005,11 +2057,295 @@ export function lowerStreamModuleCall( }; } + if (bi.member === "compose") return lowerStreamCompose(lowerer, call, loc); + return null; } +/** `stream.compose(...streams)` over stream instances: a Duplex writing + * to the first stream and reading from the last (scr_stream_compose). The + * classes fix which positions can be readable and writable; a stream in + * the wrong position, a single Readable or Writable (Duplex.from wraps + * those), and functions, iterables or web streams have no lowering. */ +function lowerStreamCompose(lowerer: Lowerer, call: ts.CallExpression, loc: SrcLoc): IrExpr { + const args = call.arguments; + const duplex: IrType = { kind: "object", className: "%Duplex" }; + if (args.some((a) => ts.isSpreadElement(a))) { + lowerer.noLowering("compose with spread arguments", call, "pass the streams directly"); + } + if (args.length === 0) { + return nodeThrowExpr( + 1, + "ERR_MISSING_ARGS", + 'The "streams" argument must be specified', + duplex, + loc, + ); + } + let streams: IrExpr[]; + try { + streams = args.map((a) => lowerStreamArg(lowerer, a, "compose")); + } catch (e) { + if (e instanceof StreamArgTypeThrow) return e.expr; + throw e; + } + const sides = streams.map((s) => + s.type.kind === "object" ? streamSidesOf(lowerer, lowerer.classes.get(s.type.className)) : null, + ); + if (streams.length === 1) { + // Duplex.from answers a duplex itself. + if (sides[0] === "rw") return streams[0]!; + lowerer.noLowering( + `compose of a single ${sides[0] === "r" ? "Readable" : "Writable"}`, + args[0]!, + "Duplex.from wraps a lone Readable or Writable, which has no lowering yet; one Duplex, Transform or PassThrough answers itself", + ); + } + for (let i = 0; i < streams.length; i++) { + const s = sides[i]; + const want = + i < streams.length - 1 && s === "w" ? "readable" : i > 0 && s === "r" ? "writable" : null; + if (want) { + lowerer.noLowering( + `compose with a stream that is not ${want} at position ${i}`, + args[i]!, + `Node throws ERR_INVALID_ARG_VALUE ('streams[${i}]' must be ${want}); every stream but the last must be readable and every stream but the first writable`, + ); + } + } + return { + kind: "libCall", + fn: "stream.compose", + args: [numLit(streams.length, loc), ...streams], + type: duplex, + loc, + }; +} + /* ── the method surface ───────────────────────────────────────────────── */ +/* ── the Readable operators ──────────────────────────────────────────── */ + +/** The Readable operators (scr_stream_ops.c — Node's operators.js). */ +export const STREAM_OPERATORS: ReadonlySet = new Set([ + "map", + "filter", + "flatMap", + "drop", + "take", + "toArray", + "forEach", + "some", + "every", + "find", + "reduce", +]); + +/** An operator callback as a checked-dynamic function value. Callbacks + * that declare the options argument ({ signal }) are refused: the + * AbortSignal Node passes there has no representation. map and flatMap + * callbacks must produce strings or Buffers (flatMap: arrays of them, or + * strings) — or checked-dynamic values, which the runtime checks. */ +function lowerOperatorCallback( + lowerer: Lowerer, + node: ts.Expression, + member: string, + maxParams: number, +): IrExpr { + const loc = locOf(node); + const optionsParam = (at: ts.Node): never => + lowerer.noLowering( + `${member} callbacks that declare the options argument`, + at, + "the { signal } argument has no lowering — take only the chunk", + ); + if ( + (ts.isArrowFunction(node) || ts.isFunctionExpression(node)) && + node.parameters.length > maxParams + ) { + optionsParam(node.parameters[maxParams]!); + } + const v = lowerer.lowerExpr(node); + if (v.type.kind === "dyn") return v; + if ( + v.type.kind !== "func" && + isJsSourceFile(node.getSourceFile()) && + lowerer.dynConvertible(v.type) + ) { + // JavaScript passes any value; Node's validateFunction throws. + return { kind: "dynFrom", value: v, type: DYN, loc }; + } + if (v.type.kind !== "func") { + lowerer.noLowering( + `${member} with a '${lowerer.fmt(v.type)}' callback`, + node, + "pass a function", + ); + } + if (v.type.params.length > maxParams) optionsParam(node); + if (member === "map" || member === "flatMap") { + const ret = v.type.ret.kind === "promise" ? v.type.ret.inner : v.type.ret; + const chunk = (t: IrType): boolean => + t.kind === "string" || t.kind === "dyn" || (t.kind === "bytes" && t.elem === "u8"); + const ok = + member === "map" + ? chunk(ret) + : ret.kind === "dyn" || ret.kind === "string" || (ret.kind === "array" && chunk(ret.elem)); + if (!ok) { + lowerer.noLowering( + `${member} callbacks that return '${lowerer.fmt(ret)}'`, + node, + member === "map" + ? "map results are stream chunks: strings or Buffers (object-mode streams of other values are not lowered)" + : "flatMap results are arrays of strings or Buffers, or strings", + ); + } + } + const getRecord = (id: string) => lowerer.shapes.get(id); + const getUnion = (id: string) => lowerer.unions.get(id); + if (!canBoxFuncIntoDyn(v.type, getRecord, getUnion)) { + lowerer.noLowering( + `${member} with this callback`, + node, + "the callback's parameter and result types must cross the checked-dynamic boundary", + ); + } + return { kind: "dynFrom", value: v, type: DYN, loc }; +} + +/** An operator's options (or reduce's initial value): an object literal + * becomes a checked-dynamic object, checked-dynamic values pass through, + * absent arguments are undefined. */ +function lowerOperatorValue( + lowerer: Lowerer, + node: ts.Expression | undefined, + what: string, + loc: SrcLoc, + literalObject: boolean, +): IrExpr { + if (node === undefined) return dynUndefinedExpr(loc); + if (literalObject && ts.isObjectLiteralExpression(node)) + return lowerDynObjectLiteral(lowerer, node); + const v = lowerer.lowerExpr(node); + if (v.type.kind === "dyn") return v; + if (literalObject && v.type.kind !== "undefinedT" && v.type.kind !== "nullT") { + lowerer.noLowering( + `${what} that are not an object literal`, + node, + "write the options inline, e.g. { concurrency: 2 }", + ); + } + if (!lowerer.dynConvertible(v.type)) { + lowerer.noLowering(`${what} of type '${lowerer.fmt(v.type)}'`, node); + } + return { kind: "dynFrom", value: v, type: DYN, loc: locOf(node) }; +} + +function lowerStreamOperator( + lowerer: Lowerer, + call: ts.CallExpression, + access: ts.PropertyAccessExpression, + member: string, +): IrExpr { + const loc = locOf(call); + const args = call.arguments; + const recv = lowerer.lowerExpr(access.expression); + const readable: IrType = { kind: "object", className: "%Readable" }; + const opts = (i: number): IrExpr => + lowerOperatorValue(lowerer, args[i], `${member} options`, loc, true); + const fnArg = (maxParams: number): IrExpr => { + if (args[0] === undefined) return dynUndefinedExpr(loc); + return lowerOperatorCallback(lowerer, args[0], member, maxParams); + }; + const lib = (fn: string, a: IrExpr[], type: IrType): IrExpr => ({ + kind: "libCall", + fn: fn as IrLibFn, + args: a, + type, + loc, + }); + switch (member) { + case "map": + case "filter": + case "flatMap": { + const fn = fnArg(1); + const fnName = + member === "map" + ? "readable.opMap" + : member === "filter" + ? "readable.opFilter" + : "readable.opFlatMap"; + return lib(fnName, [recv, fn, opts(1)], readable); + } + case "drop": + case "take": { + const n = lowerOperatorValue(lowerer, args[0], `${member}'s count`, loc, false); + return lib( + member === "drop" ? "readable.opDrop" : "readable.opTake", + [recv, n, opts(1)], + readable, + ); + } + case "toArray": + return lib("readable.opToArray", [recv, opts(0)], { kind: "promise", inner: DYN }); + case "forEach": + return lib("readable.opForEach", [recv, fnArg(1), opts(1)], { kind: "promise", inner: VOID }); + case "some": + case "every": { + const fn = fnArg(1); + return lib(member === "some" ? "readable.opSome" : "readable.opEvery", [recv, fn, opts(1)], { + kind: "promise", + inner: BOOL, + }); + } + case "find": { + const t = lowerer.typeOf(call); + const inner = lowerer.checker.getTypeArguments(t as ts.TypeReference)[0]; + if (inner && !(inner.flags & (ts.TypeFlags.Any | ts.TypeFlags.Unknown))) { + lowerer.noLowering( + "find with a type-guard callback", + call, + "the result is a checked-dynamic value — use a boolean predicate and narrow the result", + ); + } + return lib("readable.opFind", [recv, fnArg(1), opts(1)], { kind: "promise", inner: DYN }); + } + case "reduce": { + const t = lowerer.typeOf(call); + const resultTs = lowerer.checker.getTypeArguments(t as ts.TypeReference)[0]; + let kind = 0; + let inner: IrType = DYN; + if (resultTs && !(resultTs.flags & (ts.TypeFlags.Any | ts.TypeFlags.Unknown))) { + const mapped = lowerer.mapTypeOf(resultTs); + if (mapped?.kind === "f64") { + kind = 1; + inner = F64; + } else if (mapped?.kind === "bool") { + kind = 2; + inner = BOOL; + } else if (mapped?.kind === "string") { + kind = 3; + inner = STRING; + } else if (mapped?.kind !== "dyn") { + lowerer.noLowering( + `reduce with a '${mapped ? lowerer.fmt(mapped) : lowerer.checker.typeToString(resultTs)}' result`, + call, + "reduce results are numbers, booleans, strings, or checked-dynamic values", + ); + } + } + const fn = fnArg(2); + const init = lowerOperatorValue(lowerer, args[1], "reduce's initial value", loc, false); + return lib( + "readable.opReduce", + [recv, fn, init, boolLit(args.length >= 2, loc), opts(2), numLit(kind, loc)], + { kind: "promise", inner }, + ); + } + } + throw new InternalCompilerError(`unknown stream operator ${member}`); +} + /** The one supported encoding argument: utf8 (Node's default), as a * compile-time literal or absent. */ function checkUtf8Encoding(lowerer: Lowerer, member: string, arg: ts.Expression | undefined): void { @@ -2032,6 +2368,11 @@ export function lowerStreamMethodCall( info: ClassInfo, ): IrExpr | null { const member = access.name.text; + if (STREAM_OPERATORS.has(member)) { + const opSides = streamSidesOf(lowerer, info); + if (opSides === "r" || opSides === "rw") + return lowerStreamOperator(lowerer, call, access, member); + } if (!STREAM_API_MEMBERS.has(member)) return null; const sides = streamSidesOf(lowerer, info); if (!sides) return null; diff --git a/packages/compiler/src/frontend/lowering/lowerer.ts b/packages/compiler/src/frontend/lowering/lowerer.ts index 9f1628023..3c87efadf 100644 --- a/packages/compiler/src/frontend/lowering/lowerer.ts +++ b/packages/compiler/src/frontend/lowering/lowerer.ts @@ -1142,8 +1142,21 @@ export function dynFallbackType(lowerer: Lowerer, node: ts.Node, t: ts.Type): Ir symbol && lowerer.isStdlibSymbol(symbol) ? lowerer.checker.getTypeArguments(t as ts.TypeReference) : []; - if (symbol?.name === "Promise" && args[0] && args[0].flags & ts.TypeFlags.Any) - return jsFallbackPromiseType(lowerer, t); + // Promise and Promise (an any[] is itself a checked + // value below): a promise of a checked value. + const anyArray = (x: ts.Type): boolean => + lowerer.checker.isArrayType(x) && + ((lowerer.checker.getTypeArguments(x as ts.TypeReference)[0]?.flags ?? 0) & + ts.TypeFlags.Any) !== + 0; + if ( + symbol?.name === "Promise" && + args[0] && + (args[0].flags & ts.TypeFlags.Any || anyArray(args[0])) + ) + return symbol.name === "Promise" && anyArray(args[0]) + ? { kind: "promise", inner: DYN } + : jsFallbackPromiseType(lowerer, t); if ( symbol && ["Generator", "AsyncGenerator", "IterableIterator"].includes(symbol.name) && diff --git a/packages/compiler/src/frontend/tsc-codes.ts b/packages/compiler/src/frontend/tsc-codes.ts index 49e4a59e1..fe48c8855 100644 --- a/packages/compiler/src/frontend/tsc-codes.ts +++ b/packages/compiler/src/frontend/tsc-codes.ts @@ -71,6 +71,7 @@ export const JS_RELAXED_TSC_CODES: ReadonlySet = new Set([ 2556, 2769, 2740, // 2322's elaborated "missing the following properties" form + 2559, // a value with no properties in common with an all-optional options type 2305, 2551, 2724, diff --git a/packages/compiler/src/ir/builtin-effects.ts b/packages/compiler/src/ir/builtin-effects.ts index dc10fcc54..33c313493 100644 --- a/packages/compiler/src/ir/builtin-effects.ts +++ b/packages/compiler/src/ir/builtin-effects.ts @@ -236,6 +236,7 @@ export const MAY_THROW_LIB_FNS: ReadonlySet = new Set([ "stream.finishedDyn", "stream.pipeline", "stream.pipelineDyn", + "stream.compose", "sp.finished", "sp.pipeline", "sc.text", @@ -378,6 +379,13 @@ export const MAY_THROW_LIB_FNS: ReadonlySet = new Set([ "https.requestFn", "https.requestFnCb", "rl.question", + "readable.fromArr", + "readable.fromDyn", + "readable.opMap", + "readable.opFilter", + "readable.opFlatMap", + "readable.opDrop", + "readable.opTake", "rl.createSignal", "stream.finishedOpts", "stream.finishedOptsSignal", diff --git a/packages/compiler/src/ir/builtin-signatures.ts b/packages/compiler/src/ir/builtin-signatures.ts index 62fe77221..bf452dcaa 100644 --- a/packages/compiler/src/ir/builtin-signatures.ts +++ b/packages/compiler/src/ir/builtin-signatures.ts @@ -3054,6 +3054,10 @@ export const LIB_FN_SIGS = defineLibFnSignatures({ "stream.finishedDyn": { argTypes: [null, DYN], result: VOID }, "stream.pipeline": { argTypes: [F64], result: VOID }, "stream.pipelineDyn": { argTypes: [F64], result: VOID }, + /** stream.compose(count, s1..sn): the Duplex over a pipeline of the + * streams (+1). MAY THROW (ERR_INVALID_ARG_VALUE for a stream that is + * no longer readable or writable). */ + "stream.compose": { argTypes: [F64], result: VOID }, /** node:stream/promises — the promise forms over the same machinery: * sp.finished: (s) → a pending void promise the terminal watcher * settles; sp.pipeline: (count, s1..sn) → the callback pipeline's @@ -3136,11 +3140,32 @@ export const LIB_FN_SIGS = defineLibFnSignatures({ * +1 promise of the next chunk — buffered content, the EOF sentinel * (empty Buffer / dyn undefined), or a rejection with the stream's * error. The Dyn twin boxes chunks by runtime tag (the JS lane). - * readable.fromArr is Readable.from(array): a fully-seeded object- - * entry stream (one whole chunk per element; strings per the flag). */ + * readable.fromArr is Readable.from over a typed array, string or + * Buffer: [array, SCR_FROM_* flags, highWaterMark]. readable.fromDyn is + * the JS lane's [source, options]. */ "readable.nextChunk": { argTypes: [null], result: VOID }, "readable.nextChunkDyn": { argTypes: [null], result: VOID }, - "readable.fromArr": { argTypes: [null, BOOL], result: VOID }, + "readable.fromArr": { argTypes: [null, F64, F64], result: VOID }, + "readable.fromDyn": { argTypes: [DYN, DYN], result: VOID }, + /** The Readable operators (scr_stream_ops.c): [stream, ...] with every + * other argument a checked-dynamic value (the callback, the count, the + * options). map/filter/flatMap/drop/take answer the object-mode output + * stream (validated as a stream class; THROW Node's validation errors); + * toArray settles a dyn array, find a dyn value, some/every a boolean, + * forEach nothing; reduce: [stream, fn, init, hasInit, options, kind] + * with the payload kind (0 dyn, 1 number, 2 boolean, 3 string). The + * promise-returning ones reject instead of throwing. */ + "readable.opMap": { argTypes: [null, DYN, DYN], result: VOID }, + "readable.opFilter": { argTypes: [null, DYN, DYN], result: VOID }, + "readable.opFlatMap": { argTypes: [null, DYN, DYN], result: VOID }, + "readable.opDrop": { argTypes: [null, DYN, DYN], result: VOID }, + "readable.opTake": { argTypes: [null, DYN, DYN], result: VOID }, + "readable.opToArray": { argTypes: [null, DYN], result: { kind: "promise", inner: DYN } }, + "readable.opForEach": { argTypes: [null, DYN, DYN], result: { kind: "promise", inner: VOID } }, + "readable.opSome": { argTypes: [null, DYN, DYN], result: { kind: "promise", inner: BOOL } }, + "readable.opEvery": { argTypes: [null, DYN, DYN], result: { kind: "promise", inner: BOOL } }, + "readable.opFind": { argTypes: [null, DYN, DYN], result: { kind: "promise", inner: DYN } }, + "readable.opReduce": { argTypes: [null, DYN, DYN, BOOL, DYN, F64], result: VOID }, "readable.resume": { argTypes: [null], result: VOID }, "readable.isPaused": { argTypes: [null], result: BOOL }, "readable.pipe": { argTypes: [null, null, BOOL], result: VOID }, diff --git a/packages/compiler/src/ir/runtime-features.ts b/packages/compiler/src/ir/runtime-features.ts index a0bb5b761..9fe33e832 100644 --- a/packages/compiler/src/ir/runtime-features.ts +++ b/packages/compiler/src/ir/runtime-features.ts @@ -210,6 +210,9 @@ function scanRuntimeFeatures(mod: IrModule, stopAt?: keyof RuntimeFeatures): Run ) features.stream = true; if (fn.startsWith("zlib.")) features.zlib = true; + // The Readable operators (scr_stream_ops.c) await through the + // dyn-async unit. + if (fn.startsWith("readable.op")) features.dynAsync = true; if (fn.startsWith("dc.")) features.dc = true; if (fn.startsWith("assert.")) features.assert = true; if ( diff --git a/packages/compiler/src/ir/validate.ts b/packages/compiler/src/ir/validate.ts index b435e4cb4..5875cb1b4 100644 --- a/packages/compiler/src/ir/validate.ts +++ b/packages/compiler/src/ir/validate.ts @@ -3959,6 +3959,7 @@ function validateFunction( e.fn === "passthrough.initDyn" || e.fn === "stream.pipeline" || e.fn === "stream.pipelineDyn" || + e.fn === "stream.compose" || e.fn === "sp.pipeline" || e.fn === "sp.pipelineOpts" || e.fn === "sp.pipelineOptsSignal" || @@ -5168,6 +5169,7 @@ function validateFunction( e.fn === "stream.finishedDyn" || e.fn === "stream.pipeline" || e.fn === "stream.pipelineDyn" || + e.fn === "stream.compose" || e.fn === "stream.addAbortSignal" || e.fn === "stream.addAbortSignalFn" || e.fn === "stream.finishedOpts" || @@ -5258,6 +5260,22 @@ function validateFunction( } return true; } + if (e.fn === "stream.compose") { + // (count, s1..sn) → a Duplex (n >= 2), or the lone stream. + const count = e.args[0]; + const n = count?.kind === "numLit" ? count.value : -1; + if (n < 1 || e.args.length !== n + 1) { + err(`libCall ${e.fn} count/arity mismatch`, e.loc); + return true; + } + for (let i = 1; i <= n; i++) { + if (!isStreamObject(e.args[i]?.type)) { + err(`libCall ${e.fn} stage ${i} must be a stream-hierarchy object`, e.loc); + } + } + if (!isStreamObject(e.type)) err(`libCall ${e.fn} must return a stream`, e.loc); + return true; + } if (e.fn === "stream.pipeline" || e.fn === "stream.pipelineDyn") { // (count, s1..sn, cb) → the destination's type. const count = e.args[0]; @@ -5283,10 +5301,35 @@ function validateFunction( } return true; } - if (e.fn === "readable.fromArr") { - // No receiver: the seed array leads; the result is the class. - if (e.args[0]?.type.kind !== "array" || !isStreamObject(e.type)) { - err(`libCall readable.fromArr must take an array and return a stream class`, e.loc); + if ( + e.fn === "readable.opMap" || + e.fn === "readable.opFilter" || + e.fn === "readable.opFlatMap" || + e.fn === "readable.opDrop" || + e.fn === "readable.opTake" + ) { + if (!isStreamObject(e.args[0]?.type) || !isStreamObject(e.type)) { + err(`libCall ${e.fn} must take a stream and return a stream class`, e.loc); + } + return true; + } + if (e.fn === "readable.opReduce") { + const inner = e.type.kind === "promise" ? e.type.inner : null; + const kind = e.args[5]?.kind === "numLit" ? e.args[5].value : -1; + const want = [DYN, F64, BOOL, STRING][kind]; + if (!isStreamObject(e.args[0]?.type) || !inner || !want || !typeEquals(inner, want)) { + err(`libCall readable.opReduce must return the promise its kind names`, e.loc); + } + return true; + } + if (e.fn === "readable.fromArr" || e.fn === "readable.fromDyn") { + // No receiver: the source leads; the result is the class. + const src = e.fn === "readable.fromArr" ? "array" : "dyn"; + if (e.args[0]?.type.kind !== src || !isStreamObject(e.type)) { + err(`libCall ${e.fn} must take a ${src} source and return a stream class`, e.loc); + } + if (e.fn === "readable.fromArr" && e.args[1]?.kind !== "numLit") { + err(`libCall readable.fromArr flags must be a number literal`, e.loc); } return true; } diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5335-stream-operators-values.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5335-stream-operators-values.ts.json new file mode 100644 index 000000000..16992748f --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5335-stream-operators-values.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/5335-stream-operators-values.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5336-stream-operators-promise-timing.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5336-stream-operators-promise-timing.ts.json new file mode 100644 index 000000000..047f67160 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5336-stream-operators-promise-timing.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/5336-stream-operators-promise-timing.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5337-stream-operators-concurrency.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5337-stream-operators-concurrency.ts.json new file mode 100644 index 000000000..940a8d8ad --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5337-stream-operators-concurrency.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/5337-stream-operators-concurrency.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5338-stream-operators-checks.cjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5338-stream-operators-checks.cjs.json new file mode 100644 index 000000000..d177f7d00 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5338-stream-operators-checks.cjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/5338-stream-operators-checks.cjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5339-stream-for-await-hops.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5339-stream-for-await-hops.ts.json new file mode 100644 index 000000000..6a408e78a --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5339-stream-for-await-hops.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/5339-stream-for-await-hops.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5340-readable-from-options.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5340-readable-from-options.ts.json new file mode 100644 index 000000000..e2391f004 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5340-readable-from-options.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/5340-readable-from-options.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5341-readable-from-dynamic.cjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5341-readable-from-dynamic.cjs.json new file mode 100644 index 000000000..ab5412d9d --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5341-readable-from-dynamic.cjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/5341-readable-from-dynamic.cjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5342-stream-compose.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5342-stream-compose.ts.json new file mode 100644 index 000000000..f556fd730 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5342-stream-compose.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/5342-stream-compose.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5343-stream-compose-edges.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5343-stream-compose-edges.ts.json new file mode 100644 index 000000000..284409138 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5343-stream-compose-edges.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/5343-stream-compose-edges.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5344-stream-pipeline-timing.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5344-stream-pipeline-timing.ts.json new file mode 100644 index 000000000..c264a1d5e --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5344-stream-pipeline-timing.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/5344-stream-pipeline-timing.ts" + ], + "diags": [] +} diff --git a/packages/runtime-darwin-arm64/runtime-pack-matrix.mjs b/packages/runtime-darwin-arm64/runtime-pack-matrix.mjs index e79bcc30c..c86706bab 100644 --- a/packages/runtime-darwin-arm64/runtime-pack-matrix.mjs +++ b/packages/runtime-darwin-arm64/runtime-pack-matrix.mjs @@ -85,6 +85,8 @@ const optional = [ ["scr_util_style.c", all("parseArgs", "inspect", "dynAsync")], ["scr_util_callable.c", all("parseArgs", "inspect", "dynAsync", "dynInvoke", "regex")], ["scr_stream.c", "stream"], + // The Readable operators await through the dyn-async unit. + ["scr_stream_ops.c", all("stream", "dynAsync")], ["scr_loop_kqueue.c", any("netEffective", "dgram")], ["scr_loop_epoll.c", any("netEffective", "dgram")], ["scr_loop_wsapoll.c", any("netEffective", "dgram")], diff --git a/packages/runtime-pack-common/runtime-pack-matrix.mjs b/packages/runtime-pack-common/runtime-pack-matrix.mjs index e9af952ab..abdbb7cad 100644 --- a/packages/runtime-pack-common/runtime-pack-matrix.mjs +++ b/packages/runtime-pack-common/runtime-pack-matrix.mjs @@ -75,6 +75,8 @@ const OPTIONAL = [ ["scr_util_style.c", all("parseArgs", "inspect", "dynAsync")], ["scr_util_callable.c", all("parseArgs", "inspect", "dynAsync", "dynInvoke", "regex")], ["scr_stream.c", "stream"], + // The Readable operators await through the dyn-async unit. + ["scr_stream_ops.c", all("stream", "dynAsync")], ["scr_loop_kqueue.c", any("netEffective", "dgram")], ["scr_loop_epoll.c", any("netEffective", "dgram")], ["scr_loop_wsapoll.c", any("netEffective", "dgram")], diff --git a/packages/runtime/src/scr_json.c b/packages/runtime/src/scr_json.c index e20a9a4c0..34dffc882 100644 --- a/packages/runtime/src/scr_json.c +++ b/packages/runtime/src/scr_json.c @@ -4108,6 +4108,23 @@ static ScrDyn *scr_iterator_method(const ScrDyn *source) { return scr_builtin_iterable(source) ? scr_builtin_iterator_method(source) : scr_dyn_retain(scr_dyn_undefined()); } +int scr_dyn_iterable_kind(const ScrDyn *value) { + if (value->kind == SCR_DYN_NULL || value->kind == SCR_DYN_UNDEF) return 0; + if (scr_dyn_generator(value)) return value->v.typed_ref.type_key[0] == 'a' ? 2 : 1; + if (scr_async_iterator_symbol) { + ScrDyn *method = scr_dyn_symbol_key_get(value, scr_async_iterator_symbol, false); + if (!method) return -1; + bool set = method->kind != SCR_DYN_UNDEF && method->kind != SCR_DYN_NULL; + scr_dyn_release(method); + if (set) return 2; + } + ScrDyn *method = scr_iterator_method(value); + if (!method) return -1; + bool set = method->kind != SCR_DYN_UNDEF && method->kind != SCR_DYN_NULL; + scr_dyn_release(method); + return set ? 1 : 0; +} + /* Compiled class methods are dispatched before this checked-object path. */ ScrDyn *scr_dyn_iterator_optional(const ScrDyn *source) { ScrDyn *method = scr_iterator_method(source); diff --git a/packages/runtime/src/scr_runtime.h b/packages/runtime/src/scr_runtime.h index d664842b3..691bb6d24 100644 --- a/packages/runtime/src/scr_runtime.h +++ b/packages/runtime/src/scr_runtime.h @@ -3136,16 +3136,22 @@ ScrClosure *scr_stream_finished(ScrStream *s, ScrClosure *cb, ScrStreamErrInv in * the slot): success calls with NO arguments, errors box through the * boundary encoding. cb's cap boxes the dyn callable. */ void scr_stream_finished_dyn_inv(ScrClosure *cb, ScrStream *s, ScrError *err); -/* stream.pipeline(s1..sn, cb) — the callback form over stream arguments: - * chains pipes (end: true), destroys every other stream with the first - * error (Node's destroyer, pipeline order), and calls cb once after the - * LAST 'close' with that error (or NULL). Streams borrowed; cb MOVES; - * answers the destination +1 (pipeline's return value). */ +/* stream.pipeline(s1..sn, cb) — the callback form over stream arguments + * (pipeline.js): pipes each pair, destroys the unfinished streams with + * the first error, and calls cb on a nextTick once every stream's eos + * watcher finished, with that error (or NULL). Streams borrowed; cb + * MOVES; answers the destination +1 (pipeline's return value), or NULL + * with ERR_STREAM_UNABLE_TO_PIPE / ERR_INVALID_ARG_TYPE pending. */ ScrStream *scr_stream_pipeline(double n, ScrStream **streams, ScrClosure *cb, ScrStreamErrInv inv); /* The dyn-valued twins (the callback as a checked-dynamic value — * mustCall wrappers; borrowed). */ ScrClosure *scr_stream_finished_dyn(ScrStream *s, struct ScrDyn *cb); ScrStream *scr_stream_pipeline_dyn(double n, ScrStream **streams, struct ScrDyn *cb); +/* stream.compose(s1..sn): a pipeline over the streams and a Duplex that + * writes to the first and reads from the last (compose.js). One stream + * answers itself (Duplex.from of a duplex). +1, or NULL with + * ERR_MISSING_ARGS / ERR_INVALID_ARG_VALUE pending. Streams borrowed. */ +ScrStream *scr_stream_compose(double n, ScrStream **streams); /* Emitted subclass RC/trace delegates for the state block (NULL-safe — * an exception before super(options) leaves st unset). The registry @@ -3223,10 +3229,23 @@ ScrPromise *scr_sc_text(ScrStream *s); ScrPromise *scr_sc_json(ScrStream *s); ScrPromise *scr_sc_buffer(ScrStream *s); ScrPromise *scr_sc_array_buffer(ScrStream *s); -/* Readable.from(array): +1 fully-seeded object-entry stream (one WHOLE - * chunk per element — strings or Buffers per the flag; hwm 1, already - * EOF'd). Borrows arr. */ -ScrStream *scr_stream_from_arr(ScrArr *arr, bool strings); +/* Readable.from over a typed source: arr holds the string or Buffer + * elements (one element under SCR_FROM_SINGLE, from.js's string/Buffer + * case). Elements push lazily, one _read at a time, as whole entries + * unless SCR_FROM_BYTE_MODE ({ objectMode: false }). hwm applies under + * SCR_FROM_HAS_HWM. +1, or NULL with ERR_INVALID_ARG_VALUE pending. + * Borrows arr. */ +enum { + SCR_FROM_STRINGS = 1, + SCR_FROM_SINGLE = 2, + SCR_FROM_BYTE_MODE = 4, + SCR_FROM_HAS_HWM = 8, +}; +ScrStream *scr_stream_from_arr(ScrArr *arr, double flags, double hwm); +/* Readable.from(iterable, options) in the JS lane: strings, Buffers and + * arrays of them; options objectMode and highWaterMark. +1, or NULL with + * the error pending (ERR_INVALID_ARG_TYPE for a non-iterable). Borrows. */ +ScrStream *scr_stream_from_dyn(ScrDyn *src, ScrDyn *opts); ScrStream *scr_stream_pause(ScrStream *s); /* recv +1 */ ScrStream *scr_stream_resume(ScrStream *s); /* recv +1 */ bool scr_stream_is_paused(ScrStream *s); @@ -3255,6 +3274,34 @@ ScrStream *scr_stream_iterator_close(ScrStream *s); * NULL with ERR_INVALID_ARG_TYPE pending. */ ScrStream *scr_stream_add_abort_signal(ScrStream *s, ScrDyn *signal, bool from_option, ScrAbortAddFn add); +/* Object-mode streams for the stream operators: Readable.from's stream + * (highWaterMark 1) around native callbacks (+1; callbacks move), and its + * push: whole string or Buffer entries, the first push fixing the kind. + * Answers push()'s result, -1 for another kind of value, -2 for a kind + * that differs from the first push's. */ +ScrStream *scr_stream_new_object_readable(ScrClosure *read, ScrStreamReadInv read_inv, + ScrClosure *destroy, ScrStreamErrInv destroy_inv); +int scr_stream_push_object(ScrStream *s, const ScrDyn *v); +/* The Readable operators (scr_stream_ops.c, the streamOps feature): the + * callbacks, counts and options are checked-dynamic values (borrowed). + * The stream-returning ones answer the +1 object-mode output stream, or + * NULL with Node's validation error pending; the promise-returning ones + * answer a +1 promise and reject instead of throwing. toArray settles a + * dyn array, find and reduce a dyn value — reduce's `kind` picks the + * payload instead: 0 dyn, 1 number, 2 boolean, 3 string — some and every + * a boolean, forEach nothing. */ +ScrStream *scr_stream_op_map(ScrStream *s, ScrDyn *fn, ScrDyn *options); +ScrStream *scr_stream_op_filter(ScrStream *s, ScrDyn *fn, ScrDyn *options); +ScrStream *scr_stream_op_flat_map(ScrStream *s, ScrDyn *fn, ScrDyn *options); +ScrStream *scr_stream_op_drop(ScrStream *s, ScrDyn *number, ScrDyn *options); +ScrStream *scr_stream_op_take(ScrStream *s, ScrDyn *number, ScrDyn *options); +ScrPromise *scr_stream_op_to_array(ScrStream *s, ScrDyn *options); +ScrPromise *scr_stream_op_for_each(ScrStream *s, ScrDyn *fn, ScrDyn *options); +ScrPromise *scr_stream_op_some(ScrStream *s, ScrDyn *fn, ScrDyn *options); +ScrPromise *scr_stream_op_every(ScrStream *s, ScrDyn *fn, ScrDyn *options); +ScrPromise *scr_stream_op_find(ScrStream *s, ScrDyn *fn, ScrDyn *options); +ScrPromise *scr_stream_op_reduce(ScrStream *s, ScrDyn *fn, ScrDyn *init, bool has_init, + ScrDyn *options, double kind); /* finished(s, options[, cb]) — Node's eos with its listeners and state * predicates: readable/writable are -1 (the default), 0 or 1; on_error is * options.error !== false; signal is NULL or borrowed (ERR_INVALID_ARG_TYPE @@ -5266,6 +5313,9 @@ void scr_dyn_arr_push_hole(ScrDyn *arr); ScrDyn *scr_dyn_new_obj(void); ScrDyn *scr_dyn_iterator(const ScrDyn *value, const ScrStr *spell); ScrDyn *scr_dyn_iterator_optional(const ScrDyn *value); +/* from.js's probe: 2 when value[Symbol.asyncIterator] is set, 1 when + * value[Symbol.iterator] is, 0 for neither, -1 with an error pending. */ +int scr_dyn_iterable_kind(const ScrDyn *value); void scr_dyn_install_iterator_symbol(ScrDyn *key); void scr_dyn_install_async_iterator_symbol(ScrDyn *key); ScrDyn *scr_dyn_array_values_function(void); diff --git a/packages/runtime/src/scr_stream.c b/packages/runtime/src/scr_stream.c index b459038ab..e96ada971 100644 --- a/packages/runtime/src/scr_stream.c +++ b/packages/runtime/src/scr_stream.c @@ -112,11 +112,17 @@ struct ScrStreamState { * toward length/hwm and delivers undivided — Node's objectMode * accounting for the from() surface; hwm is 1). */ bool object_entries; + /* An operator output's entry kind is decided by its first push. */ + bool object_kind_set; /* The parked for-await next() promise (at most one — the loop awaits * each chunk before asking again). next_dyn picks the resolution * shape: a dyn boxed by tag (the JS lane), or bytes. */ ScrPromise *next_waiter; bool next_dyn; + /* The iterator's own read() is calling _read: synchronous pushes + * buffer, and the waiter takes the whole buffer afterwards (Node's + * read() returns everything _read pushed). */ + bool next_reading; ScrClosure *read_cb; ScrStreamReadInv read_inv; } r; @@ -129,6 +135,7 @@ struct ScrStreamState { int corked; bool writing, wsync, need_drain, after_scheduled; bool ending, finished, prefinished, final_called, finish_scheduled; + bool final_pending; /* _final called, its callback not yet run */ ScrClosure *inflight_cb; /* owned or NULL */ ScrClosure *write_cb; ScrStreamChunkInv write_inv; @@ -144,6 +151,7 @@ struct ScrStreamState { ScrClosure *flush_cb; ScrStreamPlainInv flush_inv; bool flush_called; + bool flush_pending; /* _flush called, its callback not yet run */ /* destroy */ ScrClosure *destroy_cb; @@ -339,6 +347,7 @@ typedef enum { SCR_ST_CLOSE, /* 'close' (when emitClose) */ SCR_ST_END_W, /* half-open: end the writable side */ SCR_ST_WCB_ERR, /* a failed write's user cb (called plain) */ + SCR_ST_WRITE_ERR, /* onwriteError after a synchronous failed write */ SCR_ST_NEXT_EOF, /* a parked for-await's EOF sentinel, AFTER 'end' */ SCR_ST_FIN, /* notify finished()/pipeline watchers (already- * terminal registration) */ @@ -348,7 +357,7 @@ typedef struct ScrStreamTick { ScrStream *s; /* owned */ ScrStreamTickOp op; ScrError *err; /* owned or NULL (SCR_ST_ERROR) */ - ScrClosure *cb; /* owned or NULL (SCR_ST_AFTER_WRITE / SCR_ST_WCB_ERR) */ + ScrClosure *cb; /* owned or NULL (SCR_ST_AFTER_WRITE / SCR_ST_WCB_ERR / SCR_ST_WRITE_ERR) */ struct ScrStreamTick *next; } ScrStreamTick; @@ -413,6 +422,7 @@ static ScrError *scr_stream_mkerr(const char *code, const char *msg) { /* ── forward decls ────────────────────────────────────────────────────── */ static void scr_stream_flow(ScrStream *s); +static bool scr_stream_add_entry(ScrStream *s, void *entry /*moves*/); static void scr_stream_emit_readable_now(ScrStream *s); static void scr_stream_maybe_read_more(ScrStream *s); static void scr_stream_finish_maybe(ScrStream *s); @@ -913,8 +923,23 @@ bool scr_stream_write_dyn(ScrStream *s, const ScrDyn *d, ScrClosure *cb) { /* Resolve the parked waiter with a taken chunk (moves), the EOF * sentinel, or dyn-boxed content per next_dyn. */ +/* Node's iterator is an async generator: `yield chunk` awaits the chunk, + * so next() settles one microtask after the chunk is read. The end (a + * bare return) and errors (a throw) settle without that await. caps[0] + * the waiter, caps[1] the payload (dyn or bytes, by caps[2]). */ +static void scr_stream_next_yield_job(ScrClosure *job) { + ScrPromise *w = scr_box_get_ref(job->caps[0]); + bool dyn = scr_box_get_bool(job->caps[2]); + void *payload = scr_box_get_ref(job->caps[1]); + if (dyn) scr_promise_fulfill_ref(w, payload, scr_dyn_retain_v, scr_dyn_release_v, NULL); + else scr_promise_fulfill_ref(w, payload, scr_bytes_retain_v, scr_bytes_release_v, NULL); + scr_promise_release(w); +} + static void scr_stream_next_fulfill(ScrStreamState *st, ScrPromise *w /*moves*/, void *chunk /*moves or NULL=EOF*/) { - if (st->r.next_dyn) { + void *payload; + bool dyn = st->r.next_dyn; + if (dyn) { ScrDyn *d; if (chunk == NULL) { d = scr_dyn_retain(scr_dyn_undefined()); /* the EOF sentinel */ @@ -925,12 +950,25 @@ static void scr_stream_next_fulfill(ScrStreamState *st, ScrPromise *w /*moves*/, d = scr_dyn_new_buffer((ScrBytes *)chunk); scr_bytes_release((ScrBytes *)chunk); } - scr_promise_fulfill_ref(w, d, scr_dyn_retain_v, scr_dyn_release_v, NULL); + payload = d; } else { - ScrBytes *b = chunk ? (ScrBytes *)chunk : scr_bytes_new(SCR_BYTES_U8, 0); - scr_promise_fulfill_ref(w, b, scr_bytes_retain_v, scr_bytes_release_v, NULL); + payload = chunk ? (ScrBytes *)chunk : scr_bytes_new(SCR_BYTES_U8, 0); } - scr_promise_release(w); + if (chunk == NULL) { + if (dyn) scr_promise_fulfill_ref(w, payload, scr_dyn_retain_v, scr_dyn_release_v, NULL); + else scr_promise_fulfill_ref(w, payload, scr_bytes_retain_v, scr_bytes_release_v, NULL); + scr_promise_release(w); + return; + } + ScrClosure *job = scr_closure_new((void *)&scr_stream_next_yield_job, 3); + job->caps[0] = scr_box_new_obj(scr_promise_retain_v, scr_promise_release_v, scr_promise_trace_v); + scr_box_set_ref(job->caps[0], w); /* moves */ + job->caps[1] = dyn ? scr_box_new_obj(scr_dyn_retain_v, scr_dyn_release_v, NULL) + : scr_box_new_obj(scr_bytes_retain_v, scr_bytes_release_v, NULL); + scr_box_set_ref(job->caps[1], payload); /* moves */ + job->caps[2] = scr_box_new(SCR_BOX_BOOL); + scr_box_set_bool(job->caps[2], dyn); + scr_queue_microtask(job); } /* Settle the parked waiter if the stream's state can answer now: an @@ -939,7 +977,15 @@ static void scr_stream_next_fulfill(ScrStreamState *st, ScrPromise *w /*moves*/, * fulfills the sentinel. Called from push/eof/destroy transitions. */ static void scr_stream_settle_next(ScrStream *s) { ScrStreamState *st = s->st; - if (!st->r.next_waiter) return; + if (!st->r.next_waiter || st->r.next_reading) return; + if (st->errored && !st->error_emitted) { + /* Node's iterator learns of the error from its eos 'error' listener, + * after the listeners registered before it: the waiter rejects once + * the 'error' event is dispatched (SCR_ST_ERROR), and its listener + * keeps the error from being unhandled. */ + st->next_err_consumed = true; + return; + } if (st->errored) { ScrPromise *w = st->r.next_waiter; st->r.next_waiter = NULL; @@ -994,7 +1040,9 @@ static ScrPromise *scr_stream_next_chunk_impl(ScrStream *s, bool dyn) { scr_stream_settle_next(s); if (st->r.next_waiter && !st->r.reading && !st->r.ended && !st->destroyed) { st->r.need_readable = true; + st->r.next_reading = true; scr_stream_call_read(s); + st->r.next_reading = false; if (scr_exc_pending()) { ScrPromise *w = st->r.next_waiter; st->r.next_waiter = NULL; @@ -1027,25 +1075,366 @@ ScrPromise *scr_stream_next_chunk_dyn(ScrStream *s) { return scr_stream_next_chunk_impl(s, true); } -/* ── Readable.from (array-seeded object-entry streams) ────────────────── */ +/* ── Readable.from (array, string and Buffer sources) ─────────────────── */ -/* +1 stream fully seeded from the array's elements (strings or Buffers — - * one WHOLE entry per element, Node's objectMode delivery; hwm 1) and - * already EOF'd. Borrows arr. */ -ScrStream *scr_stream_from_arr(ScrArr *arr, bool strings) { - ScrStream *s = scr_stream_alloc(&scr_readable_vt, "Readable", true, false, - 1, -1, true, true, true); +/* from.js over an array source: the iterator position and the `reading` + * guard. The typed lane holds a ScrArr of ScrStr / ScrBytes elements; the + * JS lane holds the checked-dynamic array. `single` is from.js's string / + * Buffer special case: each _read pushes the value, then EOF. */ +enum { SCR_FROM_HWM_FROM, SCR_FROM_HWM_DEFAULT, SCR_FROM_HWM_SET }; + +typedef struct { + int rc; + ScrArr *arr; /* +1 or NULL */ + ScrDyn *dyn; /* +1 or NULL */ + size_t i; + bool strings, single, reading; +} ScrFromSrc; + +static ScrFromSrc *scr_from_src_retain(ScrFromSrc *f) { + f->rc++; + return f; +} + +static void scr_from_src_release(ScrFromSrc *f) { + if (--f->rc > 0) return; + if (f->arr) scr_arr_release(f->arr); + if (f->dyn) scr_dyn_release(f->dyn); + free(f); +} + +static void *scr_from_src_retain_v(void *f) { return scr_from_src_retain(f); } +static void scr_from_src_release_v(void *f) { scr_from_src_release(f); } + +/* One typed element into the stream: a whole entry in object mode, a + * chunk in byte mode. Moves e. Returns push's answer. */ +static bool scr_from_push_typed(ScrStream *s, void *e, bool strings) { + if (s->st->r.object_entries) return scr_stream_add_entry(s, e); + bool more = strings ? scr_stream_push_str(s, e) : scr_stream_push(s, e); + if (strings) scr_str_release(e); + else scr_bytes_release(e); + return more; +} + +static void scr_from_read_inv(ScrClosure *cb, ScrStream *s, double size) { + (void)size; + ScrFromSrc *f = scr_box_get_ref(cb->caps[0]); + if (f->single) { + scr_from_push_typed(s, scr_arr_get_ref(f->arr, 0), f->strings); + if (!scr_exc_pending()) scr_stream_push_null(s); + } else if (!f->reading) { + /* nextSyncWithSyncValues: push while push() answers true */ + f->reading = true; + for (;;) { + if (f->i >= f->arr->len) { + scr_stream_push_null(s); + break; + } + void *e = scr_arr_get_ref(f->arr, (double)f->i++); + bool more = scr_from_push_typed(s, e, f->strings); + if (scr_exc_pending()) break; + if (!more) { + f->reading = false; + break; + } + } + } + scr_from_src_release(f); +} + +/* readable._destroy: close() is an async function with nothing to await + * for an array iterator (no return/throw methods), so the callback runs + * after its promise reaction and then a nextTick. */ +typedef struct { + int rc; + ScrStream *s; /* +1 */ + ScrError *error; /* +1 or NULL */ +} ScrFromDestroy; + +static void *scr_from_destroy_retain_v(void *p) { + ((ScrFromDestroy *)p)->rc++; + return p; +} + +static void scr_from_destroy_release_v(void *p) { + ScrFromDestroy *d = p; + if (--d->rc > 0) return; + scr_stream_release(d->s); + if (d->error) scr_error_release(d->error); + free(d); +} + +static ScrClosure *scr_from_destroy_closure(void *fn, ScrFromDestroy *d) { + ScrClosure *job = scr_closure_new(fn, 1); + job->caps[0] = scr_box_new_obj(&scr_from_destroy_retain_v, &scr_from_destroy_release_v, NULL); + scr_box_set_ref(job->caps[0], scr_from_destroy_retain_v(d)); + return job; +} + +static void scr_from_destroy_tick(ScrClosure *job) { + ScrFromDestroy *d = scr_box_get_ref(job->caps[0]); + scr_stream_destroy_done(d->s, d->error ? scr_error_retain(d->error) : NULL); + scr_from_destroy_release_v(d); +} + +static void scr_from_destroy_job(ScrClosure *job) { + ScrFromDestroy *d = scr_box_get_ref(job->caps[0]); + scr_next_tick(scr_from_destroy_closure((void *)&scr_from_destroy_tick, d)); + scr_from_destroy_release_v(d); +} + +static void scr_from_destroy_inv(ScrClosure *cb, ScrStream *s, ScrError *err) { + (void)cb; + ScrFromDestroy *d = calloc(1, sizeof *d); + if (!d) scr_stream_oom(); + d->rc = 1; + d->s = scr_stream_retain(s); + d->error = err ? scr_error_retain(err) : NULL; + scr_queue_microtask(scr_from_destroy_closure((void *)&scr_from_destroy_job, d)); + scr_from_destroy_release_v(d); +} + +/* getHighWaterMark: a non-negative integer, or ERR_INVALID_ARG_VALUE + * (thrown; false on return). */ +static bool scr_from_check_hwm(double hwm) { + if (hwm >= 0 && hwm <= 9007199254740991.0 && hwm == (double)(int64_t)hwm) return true; + char num[40]; + char msg[96]; + num[scr_f64_to_str(hwm, num)] = 0; + int n = snprintf(msg, sizeof msg, "The property 'options.highWaterMark' is invalid. Received %s", num); + scr_throw_error_msg_code(SCR_ERR_TYPE, msg, (size_t)n, "ERR_INVALID_ARG_VALUE"); + return false; +} + +/* The stream from.js builds: object mode unless the options turn it off; + * highWaterMark 1 for iterables, the mode's default for the string / + * Buffer case or a nullish option, or the option's value. +1, or NULL with the error + * pending. Moves f. */ +static ScrStream *scr_from_stream(ScrFromSrc *f, ScrStreamReadInv read_inv, bool object_mode, + int hwm_kind, double hwm, bool strings) { + if (hwm_kind == SCR_FROM_HWM_SET && !scr_from_check_hwm(hwm)) { + scr_from_src_release(f); + return NULL; + } + if (hwm_kind == SCR_FROM_HWM_FROM && !f->single) hwm = 1; + else if (hwm_kind != SCR_FROM_HWM_SET) hwm = object_mode ? 16 : -1; + ScrClosure *read = scr_closure_new((void *)read_inv, 1); + read->caps[0] = scr_box_new_obj(scr_from_src_retain_v, scr_from_src_release_v, NULL); + scr_box_set_ref(read->caps[0], f); /* moves */ + ScrClosure *destroy = NULL; + if (!f->single) destroy = scr_closure_new((void *)&scr_from_destroy_inv, 0); + ScrStream *s = scr_stream_new_readable(hwm, true, true, read, read_inv, destroy, + destroy ? &scr_from_destroy_inv : NULL); ScrStreamState *st = s->st; - st->r.object_entries = true; - st->r.encoded = strings; - if (strings) st->r.enc = scr_str_new("utf8", 4); - for (size_t i = 0; i < arr->len; i++) { - scr_stream_rbuf_push(st, scr_arr_get_ref(arr, (double)i), false); + if (object_mode) { + st->r.object_entries = true; + st->r.object_kind_set = strings || f->arr != NULL; + st->r.encoded = strings; + if (strings) st->r.enc = scr_str_new("utf8", 4); } - scr_stream_push_null(s); return s; } +ScrStream *scr_stream_from_arr(ScrArr *arr, double flags, double hwm) { + int bits = (int)flags; + ScrFromSrc *f = calloc(1, sizeof *f); + if (!f) scr_stream_oom(); + f->rc = 1; + f->arr = scr_arr_retain(arr); + f->strings = (bits & SCR_FROM_STRINGS) != 0; + f->single = (bits & SCR_FROM_SINGLE) != 0; + return scr_from_stream(f, &scr_from_read_inv, (bits & SCR_FROM_BYTE_MODE) == 0, + (bits & SCR_FROM_HAS_HWM) ? SCR_FROM_HWM_SET : SCR_FROM_HWM_FROM, hwm, + f->strings); +} + +/* The JS lane: one checked-dynamic value into the stream. null is + * ERR_STREAM_NULL_VALUES; in object mode, values other than strings and + * Buffers (or a mix of the two) are outside the stream model; byte mode + * rejects them with push()'s ERR_INVALID_ARG_TYPE. Each failure destroys + * the stream and answers -1. Borrows v. */ +static int scr_from_push_dyn(ScrStream *s, const ScrDyn *v) { + ScrError *e = NULL; + if (v->kind == SCR_DYN_NULL) { + static const char m[] = "May not write null values to stream"; + ScrStr *msg = scr_str_new(m, sizeof m - 1); + e = scr_error_new(SCR_ERR_TYPE, msg); + scr_str_release(msg); + scr_error_set_code(e, "ERR_STREAM_NULL_VALUES"); + } else if (s->st->r.object_entries) { + int pushed = scr_stream_push_object(s, v); + if (pushed >= 0) return pushed; + static const char mixed[] = "a Readable.from source that mixes strings and Buffers is not supported yet"; + static const char other[] = "Readable.from over values other than strings and Buffers is not supported yet"; + const char *m = pushed == -2 ? mixed : other; + ScrStr *msg = scr_str_new(m, strlen(m)); + e = scr_error_new(SCR_ERR_ERROR, msg); + scr_str_release(msg); + } else if (v->kind == SCR_DYN_STR || v->kind == SCR_DYN_BYTES) { + return scr_stream_push_dyn(s, v) ? 1 : 0; + } else { + char detail[256]; + char m[400]; + int n = snprintf(m, sizeof m, + "The \"chunk\" argument must be of type string or an instance of Buffer, " + "TypedArray, or DataView. Received %s", + scr_dyn_specific_type(v, detail, sizeof detail)); + ScrStr *msg = scr_str_new(m, (size_t)n); + e = scr_error_new(SCR_ERR_TYPE, msg); + scr_str_release(msg); + scr_error_set_code(e, "ERR_INVALID_ARG_TYPE"); + } + scr_stream_release(scr_stream_destroy(s, e)); + scr_error_release(e); + return -1; +} + +static void scr_from_read_dyn_inv(ScrClosure *cb, ScrStream *s, double size) { + (void)size; + ScrFromSrc *f = scr_box_get_ref(cb->caps[0]); + if (f->single) { + if (scr_from_push_dyn(s, f->dyn) >= 0 && !scr_exc_pending()) scr_stream_push_null(s); + } else if (!f->reading) { + f->reading = true; + for (;;) { + if (f->i >= f->dyn->v.arr.len) { + scr_stream_push_null(s); + break; + } + ScrDyn *v = scr_dyn_arr_at(f->dyn, (double)f->i++); + if (!v) break; + if (v->kind == SCR_DYN_NULL) f->reading = false; + int pushed = scr_from_push_dyn(s, v); + scr_dyn_release(v); + if (pushed < 0 || scr_exc_pending()) break; + if (!pushed) { + f->reading = false; + break; + } + } + } + scr_from_src_release(f); +} + +ScrStream *scr_stream_from_dyn(ScrDyn *src, ScrDyn *opts) { + bool single = src->kind == SCR_DYN_STR || src->kind == SCR_DYN_BYTES; + if (!single) { + int kind = scr_dyn_iterable_kind(src); + if (kind < 0) return NULL; + if (kind == 0) { + scr_dyn_arg_type_fail("iterable", "an instance of Iterable", src); + return NULL; + } + if (src->kind != SCR_DYN_ARR) { + static const char msg[] = + "Readable.from over an iterable other than an array, string, or Buffer is not supported yet"; + scr_throw_error_msg(SCR_ERR_ERROR, msg, sizeof msg - 1); + return NULL; + } + } + bool object_mode = true; + int hwm_kind = SCR_FROM_HWM_FROM; + double hwm = 0; + if (opts && opts->kind == SCR_DYN_OBJ) { + /* new Readable({ objectMode: true, highWaterMark: 1, ...opts }) */ + ScrDyn *keys = scr_dyn_own_keys(opts); + if (!keys) return NULL; + for (size_t i = 0; i < keys->v.arr.len; i++) { + ScrDyn *k = keys->v.arr.items[i]; + if (k->kind != SCR_DYN_STR) continue; + ScrStr *name = k->v.str; + if ((name->len == 10 && memcmp(name->data, "objectMode", 10) == 0) || + (name->len == 13 && memcmp(name->data, "highWaterMark", 13) == 0)) { + continue; + } + char m[200]; + int n = snprintf(m, sizeof m, "the Readable.from option '%.*s' is not supported yet", + (int)(name->len > 64 ? 64 : name->len), name->data); + scr_dyn_release(keys); + scr_throw_error_msg(SCR_ERR_ERROR, m, (size_t)n); + return NULL; + } + scr_dyn_release(keys); + ScrDyn *om = scr_dyn_obj_get(opts, "objectMode", 10); + if (om) object_mode = scr_dyn_truthy(om); + ScrDyn *h = scr_dyn_obj_get(opts, "highWaterMark", 13); + if (h && (h->kind == SCR_DYN_UNDEF || h->kind == SCR_DYN_NULL)) { + hwm_kind = SCR_FROM_HWM_DEFAULT; + } else if (h) { + double v = h->kind == SCR_DYN_NUM ? h->v.num : -1; + if (!(v >= 0 && v <= 9007199254740991.0 && v == (double)(int64_t)v)) { + scr_dyn_arg_value_fail("options.highWaterMark", NULL, h); + return NULL; + } + hwm_kind = SCR_FROM_HWM_SET; + hwm = v; + } + } + ScrFromSrc *f = calloc(1, sizeof *f); + if (!f) scr_stream_oom(); + f->rc = 1; + f->dyn = scr_dyn_retain(src); + f->single = single; + return scr_from_stream(f, &scr_from_read_dyn_inv, object_mode, hwm_kind, hwm, false); +} + +/* ── object-mode streams for the stream operators (scr_stream_ops.c) ──── */ + +/* Readable.from's stream (objectMode, highWaterMark 1) around native read + * and destroy callbacks. +1; the callbacks move. */ +ScrStream *scr_stream_new_object_readable(ScrClosure *read, ScrStreamReadInv read_inv, + ScrClosure *destroy, ScrStreamErrInv destroy_inv) { + ScrStream *s = scr_stream_new_readable(1, true, true, read, read_inv, destroy, destroy_inv); + s->st->r.object_entries = true; + return s; +} + +/* readableAddChunk in object mode: every value is one whole entry, empty + * strings and Buffers included. */ +static bool scr_stream_add_entry(ScrStream *s, void *entry /*moves*/) { + ScrStreamState *st = s->st; + if (st->r.ended) { + scr_stream_entry_release(st, entry); + ScrError *e = scr_stream_mkerr("ERR_STREAM_PUSH_AFTER_EOF", "stream.push() after EOF"); + scr_stream_error_or_destroy(s, e); + scr_error_release(e); + return false; + } + if (st->destroyed || st->errored) { + scr_stream_entry_release(st, entry); + return false; + } + st->r.reading = false; + if (st->r.flowing == 1 && st->r.length == 0 && !st->r.in_read_sync && + (scr_emitter_has((ScrEmitter *)s, "data") || st->pipes.n > 0)) { + if (st->r.emitted_readable) st->r.emitted_readable = false; + scr_stream_emit_data(s, entry); + scr_stream_entry_release(st, entry); + if (scr_exc_pending()) return false; + } else { + scr_stream_rbuf_push(st, entry, false); + if (st->r.need_readable) scr_stream_emit_readable_nt(s); + } + scr_stream_maybe_read_more(s); + scr_stream_settle_next(s); + return !st->r.ended && (st->r.length < st->r.hwm || st->r.length == 0); +} + +int scr_stream_push_object(ScrStream *s, const ScrDyn *v) { + ScrStreamState *st = s->st; + bool is_str = v->kind == SCR_DYN_STR; + if (!is_str && v->kind != SCR_DYN_BYTES) return -1; + if (!st->r.object_kind_set) { + st->r.object_kind_set = true; + st->r.encoded = is_str; + if (is_str && !st->r.enc) st->r.enc = scr_str_new("utf8", 4); + } else if (st->r.encoded != is_str) { + return -2; + } + void *entry = is_str ? (void *)scr_str_retain(v->v.str) : (void *)scr_bytes_raw_view(v->v.bytes); + return scr_stream_add_entry(s, entry) ? 1 : 0; +} + /* ── the registration hook ('data'/'readable' listeners kick flow) ────── */ static void scr_stream_on_listener(ScrEmitter *em, ScrStr *name) { @@ -1179,13 +1568,16 @@ static void scr_stream_clear_buffer(ScrStream *s) { static void scr_stream_finish_maybe(ScrStream *s) { ScrStreamState *st = s->st; if (!st->w.ending || st->w.writing || st->w.head < st->w.n || st->w.finished || - st->destroyed || st->w.finish_scheduled) { + st->destroyed || st->w.finish_scheduled || st->w.final_pending || st->flush_pending) { + /* a pending _final/_flush callback holds the finish (Node's + * pendingcb) */ return; } if (!st->w.prefinished) { if ((st->is_transform || st->passthrough) && !st->flush_called) { st->flush_called = true; if (st->flush_cb) { + st->flush_pending = true; st->flush_inv(st->flush_cb, s); return; /* flush_done resumes the finish path */ } @@ -1195,6 +1587,7 @@ static void scr_stream_finish_maybe(ScrStream *s) { } if (st->w.final_cb && !st->w.final_called) { st->w.final_called = true; + st->w.final_pending = true; st->w.final_inv(st->w.final_cb, s); return; /* final_done resumes the finish path */ } @@ -1991,11 +2384,9 @@ void scr_stream_init_passthrough_dyn(ScrStream *s, ScrDyn *opts, * ERR_STREAM_PREMATURE_CLOSE ("Premature close") when the stream * closed before end/finish. finished returns a CLEANUP function that * unhooks the callback. - * - pipeline(a, b, c, cb) on a mid-stream error: the erroring stream - * destroys itself and emits 'error'+'close'; the pipeline then - * destroys every OTHER stream with the SAME error (pipeline order), - * each emitting 'error' then 'close'; cb(err) runs after the last - * 'close'. Success: cb(null) after the destination's 'close'. + * - pipeline(a, b, c, cb) follows pipeline.js (see scr_stream_pipeline_ex): + * cb runs on a nextTick once every stream's eos watcher finished, and + * the first error destroys the streams that have not finished. * - both register 'error' handlers, so an otherwise-unhandled stream * error no longer crashes (the SCR_ST_ERROR suppression above). */ @@ -2094,18 +2485,46 @@ void scr_stream_finished_dyn_inv(ScrClosure *cb, ScrStream *s, ScrError *err) { if (r != NULL) scr_dyn_release(r); } +static ScrError *scr_stream_abort_error(ScrDyn *signal); +static ScrDyn *scr_stream_signal_prop(ScrDyn *signal, const char *key); +static void *scr_stream_dyn_retain_v(void *p); +static void scr_stream_dyn_release_v(void *p); +static ScrClosure *scr_stream_eos_impl(ScrStream *s, ScrClosure *cb, ScrStreamErrInv inv, + ScrPromise *p, bool auto_cleanup, int readable, + int writable, bool on_error, ScrDyn *signal, + ScrAbortAddFn add); + +/* stream.pipeline over streams (lib/internal/streams/pipeline.js). Each + * stream gets the destroyer (a 'close' listener and an eos watcher over + * the sides it plays), an 'error' listener, and each adjacent pair the + * pipe() helper: src.pipe(dst, { end: false }), dst.end() on the source's + * 'end', a premature-close check on the destination's 'close', and eos + * watchers on both. The callback runs on a nextTick once every watcher + * finished, or the first error destroys the remaining streams. + * + * The context's listeners on the streams hold it, and it holds the + * streams until the final callback (or exit), which breaks the cycle. */ + typedef struct { size_t rc; - size_t n, closed; - size_t watched; /* stages with a watcher: all, or all but the last (end: false) */ - ScrStream **streams; /* owned */ - ScrClosure *cb; /* owned; NULL once fired */ + size_t n; + ScrStream **streams; /* +1 each; NULL after the final callback */ + bool *destroyer; /* stream i has a destroyer (destroys queue) */ + bool *finished; /* the destroyer's finished flag */ + bool *ended; /* pipe(streams[i-1], streams[i])'s ended flag */ + size_t destroy_head; /* the next destroys entry to run */ + long finish_count; + ScrError *error; /* +1 or NULL */ + ScrClosure *cb; /* +1 until the final callback is queued */ ScrStreamErrInv inv; - ScrError *err; /* first error, owned */ - ScrDyn *dispose; /* the signal listener's disposable (+1), or NULL */ - /* contexts holding a disposable: it holds the abort listener, which - * holds the context, so exit releases what a pipeline that never - * finished still holds */ + bool final_done; + ScrDyn *signal; /* +1 or NULL */ + ScrDyn *dispose; /* the abort listener's disposable (+1) or NULL */ + /* lastStreamCleanup: the last stream's eos cleanups and its 'error' + * listener, when that stream is readable */ + ScrClosure *last_cleanup[2]; + size_t last_cleanups; + ScrClosure *last_error_l; /* +1 */ void *live_prev, *live_next; } ScrStreamPipeCtx; @@ -2119,70 +2538,388 @@ static void scr_pipe_unlink(ScrStreamPipeCtx *ctx) { ctx->live_prev = ctx->live_next = NULL; } -static void scr_pipe_atexit(void) { - while (scr_pipe_live) { - ScrStreamPipeCtx *ctx = scr_pipe_live; - scr_pipe_unlink(ctx); - if (ctx->dispose) { - ScrDyn *d = ctx->dispose; - ctx->dispose = NULL; - scr_dyn_release(d); - } - } -} - static void *scr_stream_pipectx_retain_v(void *p) { ((ScrStreamPipeCtx *)p)->rc++; return p; } +/* What the context holds besides its flags: the streams, the pending + * callback, the signal's disposable and the cleanups. */ +static void scr_stream_pipectx_drop(ScrStreamPipeCtx *ctx) { + scr_pipe_unlink(ctx); + if (ctx->streams) { + for (size_t i = 0; i < ctx->n; i++) scr_stream_release(ctx->streams[i]); + free(ctx->streams); + ctx->streams = NULL; + } + if (ctx->cb) { + scr_closure_release(ctx->cb); + ctx->cb = NULL; + } + if (ctx->dispose) { + ScrDyn *d = ctx->dispose; + ctx->dispose = NULL; + scr_dyn_release(d); + } + if (ctx->signal) { + scr_dyn_release(ctx->signal); + ctx->signal = NULL; + } + for (size_t i = 0; i < ctx->last_cleanups; i++) scr_closure_release(ctx->last_cleanup[i]); + ctx->last_cleanups = 0; + if (ctx->last_error_l) { + scr_closure_release(ctx->last_error_l); + ctx->last_error_l = NULL; + } +} + static void scr_stream_pipectx_release_v(void *p) { ScrStreamPipeCtx *ctx = p; if (--ctx->rc > 0) return; - for (size_t i = 0; i < ctx->n; i++) scr_stream_release(ctx->streams[i]); - free(ctx->streams); - if (ctx->cb) scr_closure_release(ctx->cb); - if (ctx->err) scr_error_release(ctx->err); - if (ctx->dispose) scr_dyn_release(ctx->dispose); - scr_pipe_unlink(ctx); + scr_stream_pipectx_drop(ctx); + if (ctx->error) scr_error_release(ctx->error); + free(ctx->destroyer); + free(ctx->finished); + free(ctx->ended); free(ctx); } -static void scr_stream_pipectx_dispose(ScrStreamPipeCtx *ctx) { - if (!ctx->dispose) return; - scr_pipe_unlink(ctx); - ScrDyn *d = ctx->dispose; - ctx->dispose = NULL; - ScrDyn *r = scr_dyn_call(d, NULL, 0, "dispose"); - if (r) scr_dyn_release(r); - scr_dyn_release(d); +static void scr_pipe_atexit(void) { + while (scr_pipe_live) { + ScrStreamPipeCtx *ctx = scr_stream_pipectx_retain_v(scr_pipe_live); + scr_stream_pipectx_drop(ctx); + scr_stream_pipectx_release_v(ctx); + } } -static void scr_stream_pipeline_watch_inv(ScrClosure *cb, ScrStream *s, ScrError *err) { - ScrStreamPipeCtx *ctx = scr_box_get_ref(cb->caps[0]); /* +1 */ - ctx->closed++; - if (err != NULL && ctx->err == NULL) { - ctx->err = scr_error_retain(err); - scr_stream_pipectx_dispose(ctx); - /* Node's pipeline destroyer: every other stream goes down with the - * SAME error, pipeline order (each emits 'error' then 'close'). */ - for (size_t i = 0; i < ctx->watched; i++) { - if (ctx->streams[i] != s) { - ScrStream *chain = scr_stream_destroy(ctx->streams[i], ctx->err); - scr_stream_release(chain); +/* caps[0] the context, caps[1] the stream index. */ +static ScrClosure *scr_pipe_closure(void *fn, ScrStreamPipeCtx *ctx, size_t i) { + ScrClosure *c = scr_closure_new(fn, 2); + c->caps[0] = scr_box_new_obj(&scr_stream_pipectx_retain_v, &scr_stream_pipectx_release_v, NULL); + scr_box_set_ref(c->caps[0], scr_stream_pipectx_retain_v(ctx)); + c->caps[1] = scr_box_new(SCR_BOX_F64); + scr_box_set_f64(c->caps[1], (double)i); + return c; +} + +static ScrStreamPipeCtx *scr_pipe_of(ScrClosure *c, size_t *i) { + if (i) *i = (size_t)scr_box_get_f64(c->caps[1]); + return scr_box_get_ref(c->caps[0]); /* +1 */ +} + +static void scr_pipe_on(ScrStream *s, const char *name, ScrClosure *cb /*moves*/, ScrEeInvoke inv, + bool once) { + ScrStr *n = scr_str_new(name, strlen(name)); + scr_emitter_release(scr_emitter_on((ScrEmitter *)s, n, cb, inv, once, false)); + scr_str_release(n); +} + +static bool scr_pipe_err_is(const ScrError *e, const char *code, const char *name) { + if (code && e->code && e->code->len == strlen(code) && memcmp(e->code->data, code, e->code->len) == 0) + return true; + return name && e->name && e->name->len == strlen(name) && + memcmp(e->name->data, name, e->name->len) == 0; +} + +/* process.nextTick(callback, error): caps[0] the context, caps[1] the + * callback, caps[2] the last stream, caps[3] the error as it was when + * the tick was queued (an Error or NULL). */ +static void scr_pipe_callback_tick(ScrClosure *job) { + ScrStreamPipeCtx *ctx = scr_box_get_ref(job->caps[0]); + ScrClosure *cb = scr_box_get_ref(job->caps[1]); + ScrStream *last = scr_box_get_ref(job->caps[2]); + ScrError *err = job->ncaps > 3 ? scr_box_get_ref(job->caps[3]) : NULL; + ctx->inv(cb, last, err); + if (err) scr_error_release(err); + scr_closure_release(cb); + scr_stream_release(last); + scr_stream_pipectx_release_v(ctx); +} + +/* destroyImpl.destroyer(stream, err || new ERR_STREAM_DESTROYED('pipe')) */ +static void scr_pipe_destroy(ScrStream *s, ScrError *err) { + if (s->st->destroyed) return; + ScrError *e = err ? scr_error_retain(err) + : scr_stream_mkerr("ERR_STREAM_DESTROYED", "Cannot call pipe after a stream was destroyed"); + scr_stream_release(scr_stream_destroy(s, e)); + scr_error_release(e); +} + +static void scr_pipe_finish_impl(ScrStreamPipeCtx *ctx, ScrError *err, bool final) { + if (ctx->final_done) final = false; + if (err && (!ctx->error || scr_pipe_err_is(ctx->error, "ERR_STREAM_PREMATURE_CLOSE", "AbortError"))) { + if (ctx->error) scr_error_release(ctx->error); + ctx->error = scr_error_retain(err); + } + if (!ctx->error && !final) return; + if (!ctx->streams) return; + scr_stream_pipectx_retain_v(ctx); + while (ctx->destroy_head < ctx->n) { + size_t i = ctx->destroy_head++; + if (!ctx->destroyer[i] || ctx->finished[i]) continue; + ctx->finished[i] = true; + scr_pipe_destroy(ctx->streams[i], ctx->error); + if (!ctx->streams) break; + } + if (ctx->dispose) { + ScrDyn *d = ctx->dispose; + ctx->dispose = NULL; + ScrDyn *r = scr_dyn_call(d, NULL, 0, "dispose"); + if (r) scr_dyn_release(r); + scr_dyn_release(d); + } + if (final && ctx->streams) { + ctx->final_done = true; + if (!ctx->error) { + for (size_t i = 0; i < ctx->last_cleanups; i++) { + ScrClosure *c = ctx->last_cleanup[i]; + ((void (*)(ScrClosure *))c->fn)(c); + } + if (ctx->last_error_l) { + ScrStr *n = scr_str_new("error", 5); + scr_emitter_release(scr_emitter_off((ScrEmitter *)ctx->streams[ctx->n - 1], n, ctx->last_error_l)); + scr_str_release(n); } } - } - if (ctx->closed == ctx->watched && ctx->cb != NULL) { - scr_stream_pipectx_dispose(ctx); - ScrClosure *ucb = ctx->cb; + /* process.nextTick(callback, error, value) */ + ScrClosure *job = scr_closure_new((void *)&scr_pipe_callback_tick, ctx->error ? 4 : 3); + job->caps[0] = scr_box_new_obj(&scr_stream_pipectx_retain_v, &scr_stream_pipectx_release_v, NULL); + scr_box_set_ref(job->caps[0], scr_stream_pipectx_retain_v(ctx)); + job->caps[1] = scr_box_new_obj(&scr_closure_retain_v, &scr_closure_release_v, &scr_closure_trace_v); + scr_box_set_ref(job->caps[1], ctx->cb); /* moves */ ctx->cb = NULL; - ctx->inv(ucb, ctx->streams[ctx->n - 1], ctx->err); - scr_closure_release(ucb); + job->caps[2] = scr_box_new_obj(&scr_stream_retain_v, &scr_stream_release_v, &scr_stream_trace); + scr_box_set_ref(job->caps[2], scr_stream_retain(ctx->streams[ctx->n - 1])); + if (ctx->error) { + job->caps[3] = scr_box_new_obj(&scr_error_retain_v, &scr_error_release_v, NULL); + scr_box_set_ref(job->caps[3], scr_error_retain(ctx->error)); + } + scr_next_tick(job); + scr_stream_pipectx_drop(ctx); } scr_stream_pipectx_release_v(ctx); } +/* finish(err): finishImpl(err, --finishCount === 0) */ +static void scr_pipe_finish_inv(ScrClosure *cb, ScrStream *s, ScrError *err) { + (void)s; + ScrStreamPipeCtx *ctx = scr_pipe_of(cb, NULL); + scr_pipe_finish_impl(ctx, err, --ctx->finish_count == 0); + scr_stream_pipectx_release_v(ctx); +} + +static void scr_pipe_finish_l(ScrClosure *cb) { scr_pipe_finish_inv(cb, NULL, NULL); } + +static void scr_pipe_finish_err_l(ScrClosure *cb, void *err) { scr_pipe_finish_inv(cb, NULL, err); } + +/* destroyer: stream.on('close') marks finished; eos marks it !err. */ +static void scr_pipe_destroyer_close_l(ScrClosure *cb) { + size_t i; + ScrStreamPipeCtx *ctx = scr_pipe_of(cb, &i); + ctx->finished[i] = true; + scr_stream_pipectx_release_v(ctx); +} + +static void scr_pipe_destroyer_eos(ScrClosure *cb, ScrStream *s, ScrError *err) { + (void)s; + size_t i; + ScrStreamPipeCtx *ctx = scr_pipe_of(cb, &i); + ctx->finished[i] = err == NULL; + scr_stream_pipectx_release_v(ctx); +} + +/* onError: errors after pipe completed (not AbortError / premature close) */ +static void scr_pipe_error_l(ScrClosure *cb, void *errp) { + ScrError *err = errp; + ScrStreamPipeCtx *ctx = scr_pipe_of(cb, NULL); + if (err && !scr_pipe_err_is(err, "ERR_STREAM_PREMATURE_CLOSE", "AbortError")) { + scr_pipe_finish_impl(ctx, err, false); + } + scr_stream_pipectx_release_v(ctx); +} + +/* pipe(): the destination closed before the source ended */ +static void scr_pipe_dst_close_l(ScrClosure *cb) { + size_t i; + ScrStreamPipeCtx *ctx = scr_pipe_of(cb, &i); + if (!ctx->ended[i]) { + ScrError *e = scr_stream_mkerr("ERR_STREAM_PREMATURE_CLOSE", "Premature close"); + scr_pipe_finish_impl(ctx, e, false); + scr_error_release(e); + } + scr_stream_pipectx_release_v(ctx); +} + +/* pipe()'s endFn: ended = true; dst.end(). caps[1] the destination's + * index. */ +static void scr_pipe_end_fn(ScrClosure *cb) { + size_t i; + ScrStreamPipeCtx *ctx = scr_pipe_of(cb, &i); + ctx->ended[i] = true; + if (ctx->streams) { + ScrStream *r = scr_stream_end(ctx->streams[i], NULL, NULL, NULL); + if (r) scr_stream_release(r); + } + scr_stream_pipectx_release_v(ctx); +} + +/* pipe()'s eos(src, { readable: true, writable: false }): a premature + * close after the source already ended (no error) waits for 'end' or + * 'error' instead. caps[1] the source's index. */ +static void scr_pipe_src_eos(ScrClosure *cb, ScrStream *s, ScrError *err) { + size_t i; + ScrStreamPipeCtx *ctx = scr_pipe_of(cb, &i); + ScrStreamState *st = s->st; + if (err && scr_pipe_err_is(err, "ERR_STREAM_PREMATURE_CLOSE", NULL) && st->r.ended && + !st->errored && !st->error_emitted) { + scr_pipe_on(s, "end", scr_pipe_closure((void *)&scr_pipe_finish_l, ctx, i), scr_ee_inv_fixed0, true); + scr_pipe_on(s, "error", scr_pipe_closure((void *)&scr_pipe_finish_err_l, ctx, i), scr_ee_inv_fixed1, + true); + } else { + scr_pipe_finish_impl(ctx, err, --ctx->finish_count == 0); + } + scr_stream_pipectx_release_v(ctx); +} + +/* eos(stream, { readable, writable }, cb) with Node's listeners. */ +static ScrClosure *scr_pipe_eos(ScrStream *s, ScrClosure *cb, ScrStreamErrInv inv, bool readable, + bool writable) { + return scr_stream_eos_impl(s, cb, inv, NULL, false, readable, writable, true, NULL, NULL); +} + +static void scr_pipe_abort(ScrStreamPipeCtx *ctx) { + ScrError *e = scr_stream_abort_error(ctx->signal); + scr_pipe_finish_impl(ctx, e, false); + scr_error_release(e); +} + +static ScrDyn *scr_pipe_abort_thunk(ScrClosure *clo, ScrDyn *const *args, size_t argc) { + (void)args; + (void)argc; + ScrStreamPipeCtx *ctx = scr_pipe_of(clo, NULL); + if (ctx->signal) scr_pipe_abort(ctx); + scr_stream_pipectx_release_v(ctx); + return scr_dyn_retain(scr_dyn_undefined()); +} + +static void scr_pipe_abort_tick(ScrClosure *job) { + ScrStreamPipeCtx *ctx = scr_pipe_of(job, NULL); + if (ctx->signal) scr_pipe_abort(ctx); + scr_stream_pipectx_release_v(ctx); +} + +/* The context-free throws of pipelineImpl's loop. */ +static void scr_pipe_throw(const char *code, const char *msg, int kind) { + ScrStr *m = scr_str_new(msg, strlen(msg)); + ScrError *e = scr_error_new(kind, m); + scr_str_release(m); + scr_error_set_code(e, code); + scr_throw_obj(e, &scr_error_retain_v, &scr_error_release_v, scr_error_trace_arg()); +} + +static ScrStream *scr_stream_pipeline_ex(size_t n, ScrStream **streams, ScrClosure *cb, + ScrStreamErrInv inv, bool end_opt, ScrDyn *signal, + ScrAbortAddFn add) { + ScrStreamPipeCtx *ctx = calloc(1, sizeof *ctx); + if (!ctx) scr_stream_oom(); + ctx->rc = 1; + ctx->n = n; + ctx->streams = calloc(n, sizeof *ctx->streams); + ctx->destroyer = calloc(n, sizeof *ctx->destroyer); + ctx->finished = calloc(n, sizeof *ctx->finished); + ctx->ended = calloc(n, sizeof *ctx->ended); + if (!ctx->streams || !ctx->destroyer || !ctx->finished || !ctx->ended) scr_stream_oom(); + for (size_t i = 0; i < n; i++) ctx->streams[i] = scr_stream_retain(streams[i]); + ctx->cb = cb; + ctx->inv = inv; + static SCR_TL bool registered = false; + if (!registered) { + registered = true; + scr_atexit(scr_pipe_atexit); + } + ctx->live_next = scr_pipe_live; + if (scr_pipe_live) scr_pipe_live->live_prev = ctx; + scr_pipe_live = ctx; + bool aborted = false; + if (signal) { + ctx->signal = scr_dyn_retain(signal); + ScrDyn *a = scr_stream_signal_prop(signal, "aborted"); + aborted = a && scr_dyn_truthy(a); + if (a) scr_dyn_release(a); + ScrClosure *clo = scr_pipe_closure((void *)&scr_pipe_abort_thunk, ctx, 0); + ScrDyn *listener = scr_dyn_new_func(clo, &scr_pipe_abort_thunk, 0, "()", "abort"); + ctx->dispose = add(signal, listener); + scr_dyn_release(listener); + } + ScrStream *ret = NULL; + for (size_t i = 0; i < n; i++) { + ScrStream *s = streams[i]; + bool reading = i + 1 < n; + bool writing = i > 0; + bool end = reading || end_opt; + bool last = i + 1 == n; + if (reading && (streams[i + 1]->st->close_emitted || streams[i + 1]->st->destroyed)) { + scr_pipe_throw("ERR_STREAM_UNABLE_TO_PIPE", "Cannot pipe to a closed or destroyed stream", + SCR_ERR_ERROR); + scr_stream_pipectx_drop(ctx); + scr_stream_pipectx_release_v(ctx); + return NULL; + } + bool last_readable = last && scr_stream_prop(s, "readable"); + if (end) { + ctx->destroyer[i] = true; + scr_pipe_on(s, "close", scr_pipe_closure((void *)&scr_pipe_destroyer_close_l, ctx, i), + scr_ee_inv_fixed0, false); + ScrClosure *cleanup = scr_pipe_eos(s, scr_pipe_closure((void *)&scr_pipe_destroyer_eos, ctx, i), + &scr_pipe_destroyer_eos, reading, writing); + if (last_readable) ctx->last_cleanup[ctx->last_cleanups++] = cleanup; + else scr_closure_release(cleanup); + } + ScrClosure *on_error = scr_pipe_closure((void *)&scr_pipe_error_l, ctx, i); + scr_pipe_on(s, "error", scr_closure_retain(on_error), scr_ee_inv_fixed1, false); + if (last_readable) ctx->last_error_l = on_error; + else scr_closure_release(on_error); + if (i > 0) { + if (!ret->st->has_r) { + static const char msg[] = + "The \"val\" argument must be an instance of Readable, Iterable, AsyncIterable, " + "ReadableStream, or TransformStream. Received an instance of Writable"; + scr_pipe_throw("ERR_INVALID_ARG_TYPE", msg, SCR_ERR_TYPE); + scr_stream_pipectx_drop(ctx); + scr_stream_pipectx_release_v(ctx); + return NULL; + } + /* pipe(ret, s, finish, finishOnlyHandleError, { end }) */ + ctx->finish_count += 2; + scr_pipe_on(s, "close", scr_pipe_closure((void *)&scr_pipe_dst_close_l, ctx, i), scr_ee_inv_fixed0, + false); + scr_stream_release(scr_stream_pipe(ret, s, false)); + if (end) { + ScrClosure *end_fn = scr_pipe_closure((void *)&scr_pipe_end_fn, ctx, i); + if (ret->st->r.end_emitted) scr_next_tick(end_fn); + else scr_pipe_on(ret, "end", end_fn, scr_ee_inv_fixed0, true); + } else { + scr_pipe_finish_impl(ctx, NULL, --ctx->finish_count == 0); + } + scr_closure_release(scr_pipe_eos(ret, scr_pipe_closure((void *)&scr_pipe_src_eos, ctx, i - 1), + &scr_pipe_src_eos, true, false)); + ScrClosure *cleanup = scr_pipe_eos(s, scr_pipe_closure((void *)&scr_pipe_finish_inv, ctx, i), + &scr_pipe_finish_inv, false, true); + if (scr_stream_prop(s, "readable") && last) ctx->last_cleanup[ctx->last_cleanups++] = cleanup; + else scr_closure_release(cleanup); + } + ret = s; + } + if (aborted) scr_next_tick(scr_pipe_closure((void *)&scr_pipe_abort_tick, ctx, 0)); + scr_stream_pipectx_release_v(ctx); /* the listeners and the live list hold it */ + return scr_stream_retain(ret); +} + +ScrStream *scr_stream_pipeline(double n_d, ScrStream **streams /*borrowed*/, + ScrClosure *cb /*moves*/, ScrStreamErrInv inv) { + return scr_stream_pipeline_ex((size_t)n_d, streams, cb, inv, true, NULL, NULL); +} + /* The dyn-valued twins: the callback rides as a checked-dynamic value * (borrowed; the closure's cap box retains it). */ ScrClosure *scr_stream_finished_dyn(ScrStream *s, ScrDyn *cb) { @@ -2196,92 +2933,291 @@ ScrStream *scr_stream_pipeline_dyn(double n, ScrStream **streams, ScrDyn *cb) { return scr_stream_pipeline(n, streams, c, &scr_stream_finished_dyn_inv); } -static ScrError *scr_stream_abort_error(ScrDyn *signal); -static ScrDyn *scr_stream_signal_prop(ScrDyn *signal, const char *key); -static void *scr_stream_dyn_retain_v(void *p); -static void scr_stream_dyn_release_v(void *p); +/* ── stream.compose (compose.js over stream instances) ──────────────── + * The streams run as a pipeline; the answer is a Duplex whose writes go + * to the head and whose reads come from the tail. The context is shared + * by the Duplex's callbacks, the head/tail listeners, the tail's eos + * watcher and the pipeline callback. It holds the Duplex until the + * Duplex's destroy completes (or exit), which breaks the cycle through + * the Duplex's own callbacks. */ -/* pipeline's signal: an abort destroys every watched stage with - * new AbortError(undefined, { cause: signal.reason }) (finishImpl's - * destroyers); the callback follows the last close. caps[0] the context, - * caps[1] the signal. */ -static ScrDyn *scr_stream_pipeline_abort_thunk(ScrClosure *clo, ScrDyn *const *args, size_t argc) { - (void)args; - (void)argc; - ScrStreamPipeCtx *ctx = scr_box_get_ref(clo->caps[0]); /* +1 */ - ScrDyn *signal = scr_box_get_ref(clo->caps[1]); /* +1 */ - if (ctx->err == NULL && ctx->cb != NULL) { - ctx->err = scr_stream_abort_error(signal); - scr_stream_pipectx_dispose(ctx); - for (size_t i = 0; i < ctx->watched; i++) { - ScrStream *chain = scr_stream_destroy(ctx->streams[i], ctx->err); - scr_stream_release(chain); - } +enum { SCR_COMPOSE_RUNNING, SCR_COMPOSE_DONE, SCR_COMPOSE_WAIT }; + +typedef struct ScrCompose { + size_t rc; + ScrStream *head, *tail; /* +1 */ + ScrStream *d; /* +1 until teardown */ + ScrClosure *drain_l, *data_l, *end_l; /* +1, the listeners, or NULL */ + bool readable, writable; + bool ondrain, onfinish; + int onclose; /* compose.js's onclose: undefined, null, or d's destroy callback */ + struct ScrCompose *live_prev, *live_next; +} ScrCompose; + +static SCR_TL ScrCompose *scr_compose_live = NULL; + +static void *scr_compose_retain_v(void *p) { + ((ScrCompose *)p)->rc++; + return p; +} + +static void scr_compose_release_v(void *p) { + ScrCompose *c = p; + if (--c->rc > 0) return; + scr_stream_release(c->head); + scr_stream_release(c->tail); + free(c); +} + +static ScrClosure *scr_compose_closure(void *fn, ScrCompose *c) { + ScrClosure *cl = scr_closure_new(fn, 1); + cl->caps[0] = scr_box_new_obj(&scr_compose_retain_v, &scr_compose_release_v, NULL); + scr_box_set_ref(cl->caps[0], scr_compose_retain_v(c)); + return cl; +} + +static void scr_compose_off(ScrStream *s, const char *name, ScrClosure **l) { + if (!*l) return; + ScrStr *n = scr_str_new(name, strlen(name)); + scr_emitter_release(scr_emitter_off((ScrEmitter *)s, n, *l)); + scr_str_release(n); + scr_closure_release(*l); + *l = NULL; +} + +/* Drop the Duplex and the head/tail listeners. */ +static void scr_compose_teardown(ScrCompose *c) { + if (!c->d) return; + scr_compose_off(c->head, "drain", &c->drain_l); + scr_compose_off(c->tail, "data", &c->data_l); + scr_compose_off(c->tail, "end", &c->end_l); + if (c->live_prev) c->live_prev->live_next = c->live_next; + else scr_compose_live = c->live_next; + if (c->live_next) c->live_next->live_prev = c->live_prev; + c->live_prev = c->live_next = NULL; + ScrStream *d = c->d; + c->d = NULL; + scr_stream_release(d); + scr_compose_release_v(c); /* the live list's reference */ +} + +static void scr_compose_atexit(void) { + while (scr_compose_live) scr_compose_teardown(scr_compose_live); +} + +static void scr_compose_on(ScrStream *s, const char *name, ScrClosure **slot, void *fn, ScrCompose *c, + ScrEeInvoke inv) { + ScrStr *n = scr_str_new(name, strlen(name)); + *slot = scr_compose_closure(fn, c); + scr_emitter_release(scr_emitter_on((ScrEmitter *)s, n, scr_closure_retain(*slot), inv, false, false)); + scr_str_release(n); +} + +/* d's destroy callback: callback(err), then the context lets go. */ +static void scr_compose_destroyed(ScrCompose *c, ScrError *err) { + ScrStream *d = scr_stream_retain(c->d); + scr_compose_teardown(c); + scr_stream_destroy_done(d, err ? scr_error_retain(err) : NULL); + scr_stream_release(d); +} + +/* d._write: head.write(chunk) ? callback() : (ondrain = callback) */ +static void scr_compose_write_inv(ScrClosure *cb, ScrStream *s, ScrBytes *chunk) { + ScrCompose *c = scr_box_get_ref(cb->caps[0]); + if (scr_stream_write(c->head, chunk, NULL)) { + if (!scr_exc_pending()) scr_stream_write_done(s, NULL); + } else { + c->ondrain = true; } - scr_dyn_release(signal); - scr_stream_pipectx_release_v(ctx); - return scr_dyn_retain(scr_dyn_undefined()); + scr_compose_release_v(c); } -static ScrStream *scr_stream_pipeline_ex(size_t n, ScrStream **streams, ScrClosure *cb, - ScrStreamErrInv inv, bool end, ScrDyn *signal, - ScrAbortAddFn add); - -ScrStream *scr_stream_pipeline(double n_d, ScrStream **streams /*borrowed*/, - ScrClosure *cb /*moves*/, ScrStreamErrInv inv) { - return scr_stream_pipeline_ex((size_t)n_d, streams, cb, inv, true, NULL, NULL); +/* d._final: head.end(); onfinish = callback */ +static void scr_compose_final_inv(ScrClosure *cb, ScrStream *s) { + (void)s; + ScrCompose *c = scr_box_get_ref(cb->caps[0]); + ScrStream *r = scr_stream_end(c->head, NULL, NULL, NULL); + if (r) scr_stream_release(r); + c->onfinish = true; + scr_compose_release_v(c); } -static ScrStream *scr_stream_pipeline_ex(size_t n, ScrStream **streams, ScrClosure *cb, - ScrStreamErrInv inv, bool end, ScrDyn *signal, - ScrAbortAddFn add) { - ScrStreamPipeCtx *ctx = calloc(1, sizeof *ctx); - if (!ctx) scr_stream_oom(); - ctx->rc = 1; - ctx->n = n; - ctx->watched = end ? n : n - 1; - ctx->streams = calloc(n, sizeof *ctx->streams); - if (!ctx->streams) scr_stream_oom(); - for (size_t i = 0; i < n; i++) ctx->streams[i] = scr_stream_retain(streams[i]); - ctx->cb = cb; - ctx->inv = inv; - /* Chain the pipes first (Node's pipeline forwards EOF; end: false - * leaves the destination open). */ - for (size_t i = 0; i + 1 < n; i++) { - ScrStream *d = scr_stream_pipe(streams[i], streams[i + 1], end || i + 2 < n); +static void scr_compose_drain_l(ScrClosure *cb) { + ScrCompose *c = scr_box_get_ref(cb->caps[0]); + if (c->ondrain && c->d) { + c->ondrain = false; + scr_stream_write_done(c->d, NULL); + } + scr_compose_release_v(c); +} + +/* eos(tail): the pending final callback runs */ +static void scr_compose_tail_eos(ScrClosure *cb, ScrStream *s, ScrError *err) { + (void)s; + (void)err; + ScrCompose *c = scr_box_get_ref(cb->caps[0]); + if (c->onfinish && c->d) { + c->onfinish = false; + scr_stream_final_done(c->d, NULL); + } + scr_compose_release_v(c); +} + +/* d._read: tail.resume() */ +static void scr_compose_read_inv(ScrClosure *cb, ScrStream *s, double size) { + (void)s; + (void)size; + ScrCompose *c = scr_box_get_ref(cb->caps[0]); + if (c->d) scr_stream_release(scr_stream_resume(c->tail)); + scr_compose_release_v(c); +} + +/* tail.on('data'): d.push(chunk), pausing the tail when d is full. The + * chunk is the tail's Buffer, or its string when the tail is encoded. */ +static void scr_compose_data_l(ScrClosure *cb, ScrBytes *bytes, ScrStr *str) { + ScrCompose *c = scr_box_get_ref(cb->caps[0]); + ScrStream *d = c->d; + if (d) { + ScrStreamState *dst = d->st; + bool more; + if (dst->r.object_entries) { + if (!dst->r.object_kind_set) { + dst->r.object_kind_set = true; + dst->r.encoded = str != NULL; + if (str && !dst->r.enc) dst->r.enc = scr_str_new("utf8", 4); + } + more = scr_stream_add_entry(d, str ? (void *)scr_str_retain(str) : (void *)scr_bytes_retain(bytes)); + } else if (str) { + more = scr_stream_push_str(d, str); + } else { + more = scr_stream_push(d, bytes); + } + if (!more && !scr_exc_pending() && c->d) scr_stream_release(scr_stream_pause(c->tail)); + } + scr_compose_release_v(c); +} + +static void scr_compose_end_l(ScrClosure *cb) { + ScrCompose *c = scr_box_get_ref(cb->caps[0]); + if (c->d) scr_stream_push_null(c->d); + scr_compose_release_v(c); +} + +static ScrError *scr_compose_abort_error(void) { + ScrStr *message = scr_str_new("The operation was aborted", 25); + ScrError *e = scr_error_new(SCR_ERR_ERROR, message); + scr_str_release(message); + scr_str_release(e->name); + e->name = scr_str_new("AbortError", 10); + scr_error_set_code(e, "ABORT_ERR"); + e->name_present = true; + e->name_enumerable = true; + return e; +} + +/* d._destroy */ +static void scr_compose_destroy_inv(ScrClosure *cb, ScrStream *s, ScrError *err) { + (void)s; + ScrCompose *c = scr_box_get_ref(cb->caps[0]); + ScrError *e = err ? scr_error_retain(err) : NULL; + if (!e && c->onclose != SCR_COMPOSE_DONE) e = scr_compose_abort_error(); + c->ondrain = c->onfinish = false; + if (!c->tail->st->destroyed) scr_stream_release(scr_stream_destroy(c->tail, e)); + if (c->onclose == SCR_COMPOSE_DONE) { + if (c->d) scr_compose_destroyed(c, e); + } else { + c->onclose = SCR_COMPOSE_WAIT; + } + if (e) scr_error_release(e); + scr_compose_release_v(c); +} + +/* The pipeline callback (compose.js's onfinished). */ +static void scr_compose_finished(ScrClosure *cb, ScrStream *s, ScrError *err) { + (void)s; + ScrCompose *c = scr_box_get_ref(cb->caps[0]); + int onclose = c->onclose; + c->onclose = SCR_COMPOSE_DONE; + if (onclose == SCR_COMPOSE_WAIT) { + if (c->d) scr_compose_destroyed(c, err); + } else if (c->d && (err || (!c->readable && !c->writable))) { + ScrStream *d = scr_stream_retain(c->d); + scr_stream_release(scr_stream_destroy(d, err)); scr_stream_release(d); } - /* One watcher per stream (the destination is not watched under end: - * false): the terminal statuses drive error propagation and the final - * callback (after the LAST 'close'). */ - for (size_t i = 0; i < ctx->watched; i++) { - ScrClosure *w = scr_closure_new((void *)&scr_stream_pipeline_watch_inv, 1); - w->caps[0] = scr_box_new_obj(&scr_stream_pipectx_retain_v, &scr_stream_pipectx_release_v, NULL); - scr_box_set_ref(w->caps[0], scr_stream_pipectx_retain_v(ctx)); - scr_stream_fin_add(streams[i]->st, w, &scr_stream_pipeline_watch_inv); + scr_compose_release_v(c); +} + +static void scr_compose_arg_value(size_t i, const char *what, ScrStream *s) { + char m[160]; + int n = snprintf(m, sizeof m, "The argument 'streams[%zu]' must be %s. Received an instance of %s", + i, what, s->cls ? s->cls : "Stream"); + scr_throw_error_msg_code(SCR_ERR_TYPE, m, (size_t)n, "ERR_INVALID_ARG_VALUE"); +} + +ScrStream *scr_stream_compose(double n_d, ScrStream **streams) { + size_t n = (size_t)n_d; + if (n == 0) { + static const char msg[] = "The \"streams\" argument must be specified"; + scr_throw_error_msg_code(SCR_ERR_TYPE, msg, sizeof msg - 1, "ERR_MISSING_ARGS"); + return NULL; } - if (signal) { - ScrClosure *clo = scr_closure_new((void *)&scr_stream_pipeline_abort_thunk, 2); - clo->caps[0] = scr_box_new_obj(&scr_stream_pipectx_retain_v, &scr_stream_pipectx_release_v, NULL); - scr_box_set_ref(clo->caps[0], scr_stream_pipectx_retain_v(ctx)); - clo->caps[1] = scr_box_new_obj(&scr_stream_dyn_retain_v, &scr_stream_dyn_release_v, NULL); - scr_box_set_ref(clo->caps[1], scr_dyn_retain(signal)); - ScrDyn *listener = scr_dyn_new_func(clo, &scr_stream_pipeline_abort_thunk, 0, "()", "abort"); - ctx->dispose = add(signal, listener); - scr_dyn_release(listener); - if (ctx->dispose) { - static SCR_TL bool registered = false; - if (!registered) { - registered = true; - scr_atexit(scr_pipe_atexit); - } - ctx->live_next = scr_pipe_live; - if (scr_pipe_live) scr_pipe_live->live_prev = ctx; - scr_pipe_live = ctx; + if (n == 1) return scr_stream_retain(streams[0]); /* Duplex.from(duplex) */ + for (size_t i = 0; i < n; i++) { + if (i + 1 < n && !scr_stream_prop(streams[i], "readable")) { + scr_compose_arg_value(i, "readable", streams[i]); + return NULL; + } + if (i > 0 && !scr_stream_prop(streams[i], "writable")) { + scr_compose_arg_value(i, "writable", streams[i]); + return NULL; } } - scr_stream_pipectx_release_v(ctx); /* the boxes own it now */ - return scr_stream_retain(streams[n - 1]); + ScrCompose *c = calloc(1, sizeof *c); + if (!c) scr_stream_oom(); + c->rc = 1; + c->head = scr_stream_retain(streams[0]); + c->tail = scr_stream_retain(streams[n - 1]); + c->onclose = SCR_COMPOSE_RUNNING; + ScrStream *tail = scr_stream_pipeline_ex(n, streams, scr_compose_closure((void *)&scr_compose_finished, c), + &scr_compose_finished, true, NULL, NULL); + if (!tail) { + scr_compose_release_v(c); + return NULL; + } + scr_stream_release(tail); + c->writable = scr_stream_prop(c->head, "writable") != 0; + c->readable = scr_stream_prop(c->tail, "readable") != 0; + bool object_mode = c->tail->st->has_r && c->tail->st->r.object_entries; + ScrStream *d = scr_stream_new_duplex( + object_mode ? 16 : -1, -1, true, true, true, c->readable, c->writable, + c->readable ? scr_compose_closure((void *)&scr_compose_read_inv, c) : NULL, + c->readable ? &scr_compose_read_inv : NULL, + c->writable ? scr_compose_closure((void *)&scr_compose_write_inv, c) : NULL, + c->writable ? &scr_compose_write_inv : NULL, + c->writable ? scr_compose_closure((void *)&scr_compose_final_inv, c) : NULL, + c->writable ? &scr_compose_final_inv : NULL, + scr_compose_closure((void *)&scr_compose_destroy_inv, c), &scr_compose_destroy_inv); + if (object_mode) d->st->r.object_entries = true; + c->d = scr_stream_retain(d); + if (c->writable) { + scr_compose_on(c->head, "drain", &c->drain_l, (void *)&scr_compose_drain_l, c, scr_ee_inv_fixed0); + scr_closure_release(scr_stream_finished( + c->tail, scr_compose_closure((void *)&scr_compose_tail_eos, c), &scr_compose_tail_eos)); + } + if (c->readable) { + scr_compose_on(c->tail, "data", &c->data_l, (void *)&scr_compose_data_l, c, scr_ee_inv_fixed2); + scr_compose_on(c->tail, "end", &c->end_l, (void *)&scr_compose_end_l, c, scr_ee_inv_fixed0); + } + static SCR_TL bool registered = false; + if (!registered) { + registered = true; + scr_atexit(scr_compose_atexit); + } + c->live_next = scr_compose_live; + if (scr_compose_live) scr_compose_live->live_prev = c; + scr_compose_live = c; /* the reference c was created with */ + return d; } /* ── node:stream/promises ───────────────────────────────────────────── @@ -2323,7 +3259,8 @@ ScrPromise *scr_sp_finished(ScrStream *s) { ScrPromise *scr_sp_pipeline(double n, ScrStream **streams) { ScrPromise *p = scr_promise_new(); ScrStream *dst = scr_stream_pipeline(n, streams, scr_sp_watcher(p), &scr_sp_settle_inv); - scr_stream_release(dst); + if (dst) scr_stream_release(dst); + else scr_promise_reject_pending(p); return p; } @@ -2349,7 +3286,8 @@ ScrPromise *scr_sp_pipeline_opts(double n, ScrStream **streams, bool end, ScrDyn } ScrStream *dst = scr_stream_pipeline_ex((size_t)n, streams, scr_sp_watcher(p), &scr_sp_settle_inv, end, signal, add); - scr_stream_release(dst); + if (dst) scr_stream_release(dst); + else scr_promise_reject_pending(p); return p; } @@ -3637,8 +4575,18 @@ void scr_stream_write_done(ScrStream *s, ScrError *err /*moves*/) { ScrClosure *ucb = st->w.inflight_cb; st->w.inflight_cb = NULL; if (err) { - if (ucb) scr_st_tick(s, SCR_ST_WCB_ERR, NULL, ucb); - scr_stream_error_or_destroy(s, err); + /* Node's onwrite: errored is set now; onwriteError (the callback, + * then errorOrDestroy) runs on a tick after a synchronous write. */ + if (!st->errored) st->errored = scr_error_retain(err); + if (st->w.wsync) { + scr_st_tick(s, SCR_ST_WRITE_ERR, err, ucb); + return; + } + if (ucb) { + ((void (*)(ScrClosure *))ucb->fn)(ucb); + scr_closure_release(ucb); + } + if (!scr_exc_pending()) scr_stream_error_or_destroy(s, err); scr_error_release(err); return; } @@ -3659,6 +4607,7 @@ void scr_stream_write_done(ScrStream *s, ScrError *err /*moves*/) { void scr_stream_final_done(ScrStream *s, ScrError *err /*moves*/) { ScrStreamState *st = s->st; + st->w.final_pending = false; if (err) { scr_stream_error_or_destroy(s, err); scr_error_release(err); @@ -3716,6 +4665,7 @@ void scr_stream_transform_done(ScrStream *s, ScrError *err /*moves*/, ScrBytes * void scr_stream_flush_done(ScrStream *s, ScrError *err /*moves*/, ScrBytes *data /*moves*/, ScrStr *data_str /*moves*/) { ScrStreamState *st = s->st; + st->flush_pending = false; if (err) { if (data) scr_bytes_release(data); if (data_str) scr_str_release(data_str); @@ -3984,6 +4934,7 @@ static void scr_stream_run_tick(ScrStreamTick *t) { if (st->next_err_consumed && !scr_emitter_has((ScrEmitter *)s, "error")) { /* the for-await's rejection consumed it (Node's iterator * registers its own 'error' handler — no unhandled crash) */ + scr_stream_settle_next(s); break; } if (st->fin.n > 0 && !scr_emitter_has((ScrEmitter *)s, "error")) { @@ -3993,6 +4944,7 @@ static void scr_stream_run_tick(ScrStreamTick *t) { break; } scr_stream_emit_error(s, t->err); + if (!scr_exc_pending()) scr_stream_settle_next(s); } break; } @@ -4031,6 +4983,16 @@ static void scr_stream_run_tick(ScrStreamTick *t) { } break; } + case SCR_ST_WRITE_ERR: { + ScrClosure *cb = t->cb; + t->cb = NULL; + if (cb) { + ((void (*)(ScrClosure *))cb->fn)(cb); + scr_closure_release(cb); + } + if (!scr_exc_pending()) scr_stream_error_or_destroy(s, t->err); + break; + } case SCR_ST_FIN: { scr_stream_notify_finished(s); break; diff --git a/packages/runtime/src/scr_stream_ops.c b/packages/runtime/src/scr_stream_ops.c new file mode 100644 index 000000000..1f64d5fa7 --- /dev/null +++ b/packages/runtime/src/scr_stream_ops.c @@ -0,0 +1,2057 @@ +/* node:stream's Readable operators (lib/internal/streams/operators.js): + * map, filter, forEach, toArray, some, find, every, flatMap, drop, take + * and reduce, over streams whose chunks are strings or Buffers. Linked + * only when a program calls one (the streamOps feature). + * + * Node writes the operators as async functions and async generators, and + * wraps the stream-returning ones in Readable.from. Their observable + * ordering (when callbacks run relative to the consumer, when promises + * settle) is the order of those functions' awaits, so this unit follows + * the JavaScript source step for step: every `await x` is one op_await + * (a promise job after x settles, the same hop a compiled await takes), + * the generators follow the async generator protocol (request queue, + * yield awaiting its operand, return and throw requests), and the + * Readable.from(generator) wrapper is from.js's nextAsync/close. Each + * step runs on its own fiber through scr_async_spawn_after, so the unit + * works on every target, wasm32 included. + * + * The user's callbacks are checked-dynamic function values called with + * one argument (the chunk); callbacks that declare the options argument + * ({ signal }) are refused, because the AbortSignal Node passes there has + * no representation here. options.signal itself is polled where Node + * polls it (signal.aborted), so no listener is needed. Stream-returning + * operators produce object-mode Readables whose entries are strings or + * Buffers; any other value is refused when it would be pushed. */ +#include "scr_runtime.h" + +#include +#include +#include +#include + +static void op_oom(void) { + fputs("scriptc: out of memory\n", stderr); + abort(); +} + +static void *op_alloc(size_t n) { + void *p = calloc(1, n); + if (!p) op_oom(); + return p; +} + +/* ── refcounted step owners ────────────────────────────────────────── */ + +typedef struct OpObj OpObj; +struct OpObj { + size_t rc; + void (*drop)(OpObj *self); /* frees the object's own fields and itself */ +}; + +static void *op_ref(void *o) { + ((OpObj *)o)->rc++; + return o; +} + +static void op_unref(void *o) { + OpObj *b = o; + if (--b->rc == 0) b->drop(b); +} + +/* ── sentinels ─────────────────────────────────────────────────────── */ + +/* kEmpty (a callback result that yields nothing), kEof (the pump's end + * marker) and the generators' done marker: process-lived objects + * compared by identity, never visible to the program. */ +static SCR_TL ScrDyn *op_k_empty = NULL; +static SCR_TL ScrDyn *op_k_eof = NULL; +static SCR_TL ScrDyn *op_k_done = NULL; + +static void op_sentinels_free(void) { + scr_dyn_release(op_k_empty); + scr_dyn_release(op_k_eof); + scr_dyn_release(op_k_done); + op_k_empty = op_k_eof = op_k_done = NULL; +} + +static void op_sentinels_init(void) { + if (op_k_empty) return; + op_k_empty = scr_dyn_new_obj(); + op_k_eof = scr_dyn_new_obj(); + op_k_done = scr_dyn_new_obj(); + scr_atexit(op_sentinels_free); +} + +/* ── awaits ────────────────────────────────────────────────────────── */ + +/* A continuation receives the awaited value (+1), or NULL with the + * rejection pending in the exception cell. It must leave no exception + * pending when it returns. */ +typedef void (*OpCont)(void *self, ScrDyn *value); + +typedef struct { + void *self; + OpCont k; + ScrPromise *p; +} OpAwait; + +static void op_await_entry(ScrFiber *fiber, void *arg) { + (void)fiber; + OpAwait a = *(OpAwait *)arg; + free(arg); + ScrDyn *v = scr_await_dyn(a.p); + scr_promise_release(a.p); + a.k(a.self, v); + if (scr_exc_pending()) scr_exc_clear(); + op_unref(a.self); +} + +/* `await p`: k runs in the promise job after p settles. Borrows p. */ +static void op_await(ScrPromise *p, OpCont k, void *self) { + OpAwait *a = op_alloc(sizeof *a); + a->self = op_ref(self); + a->k = k; + a->p = scr_promise_retain(p); + ScrPromise *fiber = scr_async_spawn_after(p, &op_await_entry, a); + scr_promise_release(fiber); +} + +static ScrPromise *op_resolved(ScrDyn *v /*borrowed*/) { + ScrPromise *p = scr_promise_new(); + scr_promise_fulfill_ref(p, scr_dyn_retain(v), scr_dyn_retain_v, scr_dyn_release_v, NULL); + return p; +} + +/* `await v` over a value: a promise is awaited, anything else is awaited + * as an already resolved promise (one job). Borrows v. */ +static void op_await_value(ScrDyn *v, OpCont k, void *self) { + if (v->kind == SCR_DYN_PROMISE) { + op_await(v->v.promise, k, self); + return; + } + ScrPromise *p = op_resolved(v); + op_await(p, k, self); + scr_promise_release(p); +} + +static void op_fulfill(ScrPromise *p, ScrDyn *v /*borrowed*/) { + scr_promise_fulfill_ref(p, scr_dyn_retain(v), scr_dyn_retain_v, scr_dyn_release_v, NULL); +} + +/* Rejects p with a saved exception (borrowed). */ +static void op_reject(ScrPromise *p, const ScrCaught *c) { + scr_rethrow(c); + scr_promise_reject_pending(p); +} + +/* PromiseReject(err) for a saved exception: +1 dyn promise. */ +static ScrDyn *op_rejected_dyn(const ScrCaught *c) { + ScrPromise *p = scr_promise_new(); + op_reject(p, c); + ScrDyn *d = scr_dyn_new_promise(p); + scr_promise_release(p); + return d; +} + +/* ── errors ────────────────────────────────────────────────────────── */ + +static ScrCaught *op_take_error(ScrError *e /*moves*/) { + scr_throw_obj(e, &scr_error_retain_v, &scr_error_release_v, scr_error_trace_arg()); + return scr_exc_take(); +} + +/* new AbortError() or new AbortError(undefined, { cause }) */ +static ScrError *op_abort_error(const ScrDyn *cause /*NULL: no cause*/) { + ScrStr *message = scr_str_new("The operation was aborted", 25); + ScrError *e = scr_error_new(SCR_ERR_ERROR, message); + scr_str_release(message); + scr_str_release(e->name); + e->name = scr_str_new("AbortError", 10); + /* AbortError assigns code, then name: own enumerable properties */ + scr_error_set_code(e, "ABORT_ERR"); + e->name_present = true; + e->name_enumerable = true; + if (cause) scr_error_define_cause(e, (ScrDyn *)cause); + return e; +} + +static ScrError *op_coded_error(int kind, const char *code, const char *message) { + ScrStr *m = scr_str_new(message, strlen(message)); + ScrError *e = scr_error_new(kind, m); + scr_str_release(m); + scr_error_set_code(e, code); + return e; +} + +/* ERR_OUT_OF_RANGE: `The value of "" is out of range. It must be + * . Received ` (thrown; pending on return). */ +static void op_throw_out_of_range(const char *name, const char *range, double received) { + char num[48]; + scr_num_received(received, num); + char msg[256]; + int n = snprintf(msg, sizeof msg, "The value of \"%s\" is out of range. It must be %s. Received %s", + name, range, num); + scr_throw_error_msg_code(SCR_ERR_RANGE, msg, (size_t)n, "ERR_OUT_OF_RANGE"); +} + +/* The runtime refusal for a value outside the string/Buffer model. */ +static ScrError *op_unsupported(const char *what) { + char msg[256]; + int n = snprintf(msg, sizeof msg, "%s is not supported yet", what); + ScrStr *m = scr_str_new(msg, (size_t)n); + ScrError *e = scr_error_new(SCR_ERR_ERROR, m); + scr_str_release(m); + return e; +} + +/* ── options ───────────────────────────────────────────────────────── */ + +static ScrDyn *op_get(const ScrDyn *obj, const char *key) { + if (!obj) return NULL; + size_t n = strlen(key); + if (obj->kind == SCR_DYN_OBJ) { + ScrDyn *v = scr_dyn_obj_get(obj, key, n); + return v ? scr_dyn_retain(v) : NULL; + } + if (obj->kind == SCR_DYN_HANDLE) { + ScrStr *k = scr_str_new(key, n); + ScrDyn *v = scr_dyn_handle_key_get(obj, k); + scr_str_release(k); + return v; + } + return NULL; +} + +static bool op_nullish(const ScrDyn *v) { + return v == NULL || v->kind == SCR_DYN_UNDEF || v->kind == SCR_DYN_NULL; +} + +/* validateObject(options, 'options') when options != null: false with + * ERR_INVALID_ARG_TYPE pending. */ +static bool op_check_options(const ScrDyn *options) { + if (op_nullish(options)) return true; + if ((options->kind == SCR_DYN_OBJ || options->kind == SCR_DYN_HANDLE) && + !scr_dyn_is_callable(options)) { + return true; + } + scr_dyn_arg_type_fail("options", "of type object", options); + return false; +} + +/* options.signal when != null, validated (validateAbortSignal): +1, or + * NULL when absent. *ok false with ERR_INVALID_ARG_TYPE pending. */ +static ScrDyn *op_signal(const ScrDyn *options, bool *ok) { + *ok = true; + ScrDyn *s = op_get(options, "signal"); + if (op_nullish(s)) { + if (s) scr_dyn_release(s); + return NULL; + } + ScrDyn *aborted = NULL; + if (s->kind == SCR_DYN_OBJ || s->kind == SCR_DYN_HANDLE) aborted = op_get(s, "aborted"); + if (!aborted) { + scr_dyn_prop_type_fail("options.signal", "an instance of AbortSignal", s); + scr_dyn_release(s); + *ok = false; + return NULL; + } + scr_dyn_release(aborted); + return s; +} + +static bool op_aborted(const ScrDyn *signal) { + if (!signal) return false; + ScrDyn *a = op_get(signal, "aborted"); + bool r = a && scr_dyn_truthy(a); + if (a) scr_dyn_release(a); + return r; +} + +/* The callback argument: a function that does not read the options + * argument. `max` is how many parameters Node's call passes before the + * options. False with the error pending. */ +static bool op_check_fn(const ScrDyn *fn, const char *name, uint32_t max) { + if (!fn || !scr_dyn_is_callable(fn)) { + scr_dyn_arg_type_fail(name, "of type function", fn ? fn : scr_dyn_undefined()); + return false; + } + if (fn->kind == SCR_DYN_FUNC && fn->v.fn.arity > max) { + static const char m[] = + "stream operator callbacks that read the options argument ({ signal }) are not supported yet"; + scr_throw_error_msg(SCR_ERR_ERROR, m, sizeof m - 1); + return false; + } + return true; +} + +/* ── iteration sources ─────────────────────────────────────────────── */ + +typedef struct OpGen OpGen; + +/* `for await (const v of src)` over a Readable (its async iterator) or an + * operator generator. */ +typedef struct { + ScrStream *s; /* +1, or NULL */ + OpGen *g; /* +1, or NULL */ +} OpSrc; + +static ScrPromise *gen_next(OpGen *g); +static ScrPromise *gen_return(OpGen *g, ScrDyn *v); +static ScrPromise *gen_throw(OpGen *g, const ScrCaught *c); + +/* iterator.next(): +1 promise of the next value or the end. */ +static ScrPromise *src_next(OpSrc *src) { + return src->s ? scr_stream_next_chunk_dyn(src->s) : gen_next(src->g); +} + +static bool src_done(const OpSrc *src, const ScrDyn *v) { + return src->s ? v->kind == SCR_DYN_UNDEF : v == op_k_done; +} + +/* The Readable iterator's return(): the iterator generator resumes at its + * yield with a return completion (one await), its finally destroys the + * stream (destroyer: an AbortError when not finished), and the request + * settles. */ +typedef struct { + OpObj base; + ScrStream *s; + ScrPromise *p; +} OpStreamReturn; + +static void op_stream_return_drop(OpObj *o) { + OpStreamReturn *r = (OpStreamReturn *)o; + scr_stream_release(r->s); + scr_promise_release(r->p); + free(r); +} + +static void op_stream_return_k(void *self, ScrDyn *v) { + OpStreamReturn *r = self; + if (v) scr_dyn_release(v); + scr_stream_release(scr_stream_iterator_close(r->s)); + op_fulfill(r->p, op_k_done); +} + +static ScrPromise *src_return(OpSrc *src) { + if (src->g) return gen_return(src->g, scr_dyn_undefined()); + OpStreamReturn *r = op_alloc(sizeof *r); + r->base.rc = 1; + r->base.drop = &op_stream_return_drop; + r->s = scr_stream_retain(src->s); + r->p = scr_promise_new(); + ScrPromise *result = scr_promise_retain(r->p); + op_await_value(scr_dyn_undefined(), &op_stream_return_k, r); + op_unref(r); + return result; +} + +static void src_release(OpSrc *src) { + if (src->s) scr_stream_release(src->s); + if (src->g) op_unref(src->g); + src->s = NULL; + src->g = NULL; +} + +/* ── the async generator protocol ──────────────────────────────────── */ + +enum { GS_START, GS_SUSPENDED, GS_EXECUTING, GS_AWAIT_RETURN, GS_DONE }; +enum { GR_NEXT, GR_RETURN, GR_THROW }; + +typedef struct { + ScrPromise *p; + int kind; + ScrDyn *value; /* return value (+1) */ + ScrCaught *error; /* throw value (+1) */ +} OpGenReq; + +/* An operator generator: the protocol state plus the body's entry points. + * resume runs the body from its start or from the yield it is suspended + * at (normal completion); ret delivers a return completion at that yield + * (after the unwrap await) and thr a throw completion. */ +struct OpGen { + OpObj base; + int state; + OpGenReq *q; + size_t qh, qn, qcap; + void (*resume)(OpGen *g); + void (*ret)(OpGen *g, ScrDyn *v); + void (*thr)(OpGen *g, ScrCaught *c); + void (*body_drop)(OpGen *g); +}; + +static void gen_drop(OpObj *o) { + OpGen *g = (OpGen *)o; + for (size_t i = g->qh; i < g->qn; i++) { + scr_promise_release(g->q[i].p); + if (g->q[i].value) scr_dyn_release(g->q[i].value); + if (g->q[i].error) scr_caught_release(g->q[i].error); + } + free(g->q); + g->body_drop(g); +} + +static ScrPromise *gen_enqueue(OpGen *g, int kind, ScrDyn *value, const ScrCaught *error) { + if (g->qn == g->qcap) { + if (g->qh > 0) { + memmove(g->q, g->q + g->qh, (g->qn - g->qh) * sizeof *g->q); + g->qn -= g->qh; + g->qh = 0; + } + if (g->qn == g->qcap) { + g->qcap = g->qcap ? g->qcap * 2 : 4; + g->q = realloc(g->q, g->qcap * sizeof *g->q); + if (!g->q) op_oom(); + } + } + OpGenReq *r = &g->q[g->qn++]; + r->p = scr_promise_new(); + r->kind = kind; + r->value = value ? scr_dyn_retain(value) : NULL; + r->error = error ? scr_caught_retain((ScrCaught *)error) : NULL; + return scr_promise_retain(r->p); +} + +static bool gen_has_req(const OpGen *g) { return g->qh < g->qn; } + +static OpGenReq gen_shift(OpGen *g) { + OpGenReq r = g->q[g->qh++]; + if (g->qh == g->qn) g->qh = g->qn = 0; + return r; +} + +static void gen_req_release(OpGenReq *r) { + scr_promise_release(r->p); + if (r->value) scr_dyn_release(r->value); + if (r->error) scr_caught_release(r->error); +} + +static void gen_drain(OpGen *g); + +/* AsyncGeneratorAwaitReturn: await the value, then the request settles + * done (a rejection rejects it). */ +typedef struct { + OpObj base; + OpGen *g; + ScrPromise *p; +} OpAwaitReturn; + +static void op_await_return_drop(OpObj *o) { + OpAwaitReturn *a = (OpAwaitReturn *)o; + op_unref(a->g); + scr_promise_release(a->p); + free(a); +} + +static void op_await_return_k(void *self, ScrDyn *v) { + OpAwaitReturn *a = self; + OpGen *g = a->g; + if (v) { + scr_dyn_release(v); + op_fulfill(a->p, op_k_done); + } else { + scr_promise_reject_pending(a->p); + } + g->state = GS_DONE; + gen_drain(g); +} + +static void gen_await_return(OpGen *g, OpGenReq *r) { + g->state = GS_AWAIT_RETURN; + OpAwaitReturn *a = op_alloc(sizeof *a); + a->base.rc = 1; + a->base.drop = &op_await_return_drop; + a->g = op_ref(g); + a->p = scr_promise_retain(r->p); + op_await_value(r->value ? r->value : scr_dyn_undefined(), &op_await_return_k, a); + op_unref(a); +} + +/* The completed generator answers its queued requests: next settles done + * at once, throw rejects at once, return awaits its value first. */ +static void gen_drain(OpGen *g) { + while (g->state == GS_DONE && gen_has_req(g)) { + OpGenReq r = gen_shift(g); + if (r.kind == GR_RETURN) { + gen_await_return(g, &r); + gen_req_release(&r); + return; + } + if (r.kind == GR_THROW) { + op_reject(r.p, r.error); + } else { + op_fulfill(r.p, op_k_done); + } + gen_req_release(&r); + } +} + +/* The body finished: the first request settles (done, or rejected with + * the body's exception), then the queue drains. Borrows c. */ +static void gen_complete(OpGen *g, const ScrCaught *c) { + g->state = GS_DONE; + if (gen_has_req(g)) { + OpGenReq r = gen_shift(g); + if (c) op_reject(r.p, c); + else op_fulfill(r.p, op_k_done); + gen_req_release(&r); + } + gen_drain(g); +} + +/* Delivers the first queued request to a body suspended at a yield (or + * already running its continuation): next resumes it, return awaits the + * value and then delivers the return completion, throw delivers the + * throw completion. */ +typedef struct { + OpObj base; + OpGen *g; +} OpUnwrapReturn; + +static void op_unwrap_return_drop(OpObj *o) { + OpUnwrapReturn *u = (OpUnwrapReturn *)o; + op_unref(u->g); + free(u); +} + +static void op_unwrap_return_k(void *self, ScrDyn *v) { + OpGen *g = ((OpUnwrapReturn *)self)->g; + if (!v) { + ScrCaught *c = scr_exc_take(); + g->thr(g, c); + scr_caught_release(c); + return; + } + g->ret(g, v); + scr_dyn_release(v); +} + +static void gen_deliver(OpGen *g) { + OpGenReq *r = &g->q[g->qh]; + g->state = GS_EXECUTING; + if (r->kind == GR_NEXT) { + g->resume(g); + } else if (r->kind == GR_THROW) { + g->thr(g, r->error); + } else { + OpUnwrapReturn *u = op_alloc(sizeof *u); + u->base.rc = 1; + u->base.drop = &op_unwrap_return_drop; + u->g = op_ref(g); + op_await_value(r->value ? r->value : scr_dyn_undefined(), &op_unwrap_return_k, u); + op_unref(u); + } +} + +static ScrPromise *gen_request(OpGen *g, int kind, ScrDyn *value, const ScrCaught *error) { + ScrPromise *p = gen_enqueue(g, kind, value, error); + if (g->state == GS_START) { + if (kind == GR_NEXT) { + gen_deliver(g); + } else { + /* A generator that never started completes without running. */ + g->state = GS_DONE; + gen_drain(g); + } + } else if (g->state == GS_SUSPENDED) { + gen_deliver(g); + } else if (g->state == GS_DONE) { + gen_drain(g); + } + return p; +} + +static ScrPromise *gen_next(OpGen *g) { return gen_request(g, GR_NEXT, NULL, NULL); } +static ScrPromise *gen_return(OpGen *g, ScrDyn *v) { return gen_request(g, GR_RETURN, v, NULL); } +static ScrPromise *gen_throw(OpGen *g, const ScrCaught *c) { return gen_request(g, GR_THROW, NULL, c); } + +/* The body yields an awaited value: the request settles with it, and the + * body continues at once when another request is queued (the generator + * never suspends), or suspends until the next one. Borrows v. */ +static void gen_yield_now(OpGen *g, ScrDyn *v) { + OpGenReq r = gen_shift(g); + op_fulfill(r.p, v); + gen_req_release(&r); + if (gen_has_req(g)) { + gen_deliver(g); + } else { + g->state = GS_SUSPENDED; + } +} + +/* `yield v` in an async generator awaits v first. */ +static void op_yield_k(void *self, ScrDyn *v) { + OpGen *g = self; + if (!v) { + ScrCaught *c = scr_exc_take(); + g->thr(g, c); + scr_caught_release(c); + return; + } + gen_yield_now(g, v); + scr_dyn_release(v); +} + +static void gen_yield(OpGen *g, ScrDyn *v /*borrowed*/) { op_await_value(v, &op_yield_k, g); } + +static void gen_init(OpGen *g, void (*body_drop)(OpGen *)) { + g->base.rc = 1; + g->base.drop = &gen_drop; + g->state = GS_START; + g->body_drop = body_drop; +} + +/* ── map (and filter, forEach, every through it) ───────────────────── */ + +/* How each source value turns into the queued result: map calls fn + * directly; filter, forEach and every go through Node's async wrappers + * (filterFn, forEachFn, and every's negating predicate inside filterFn). */ +enum { MODE_MAP, MODE_FILTER, MODE_FOREACH, MODE_EVERY }; + +typedef struct { + OpGen g; + OpSrc src; + ScrDyn *fn; /* +1 */ + ScrDyn *signal; /* +1 or NULL */ + int mode; + double concurrency, hwm; + ScrDyn **queue; /* +1 entries: dyn promises or kEof */ + size_t qh, qn, qcap; + double cnt; + bool done; + ScrPromise *next_p; /* the consumer's `await new Promise(r => next = r)` */ + ScrPromise *resume_p; /* the pump's `await new Promise(r => resume = r)` */ + bool started; +} OpMap; + +static void op_map_body_drop(OpGen *g) { + OpMap *m = (OpMap *)g; + src_release(&m->src); + scr_dyn_release(m->fn); + if (m->signal) scr_dyn_release(m->signal); + for (size_t i = m->qh; i < m->qn; i++) scr_dyn_release(m->queue[i]); + free(m->queue); + if (m->next_p) scr_promise_release(m->next_p); + if (m->resume_p) scr_promise_release(m->resume_p); + free(m); +} + +static void map_push(OpMap *m, ScrDyn *v /*moves*/) { + if (m->qn == m->qcap) { + if (m->qh > 0) { + memmove(m->queue, m->queue + m->qh, (m->qn - m->qh) * sizeof *m->queue); + m->qn -= m->qh; + m->qh = 0; + } + if (m->qn == m->qcap) { + m->qcap = m->qcap ? m->qcap * 2 : 4; + m->queue = realloc(m->queue, m->qcap * sizeof *m->queue); + if (!m->queue) op_oom(); + } + } + m->queue[m->qn++] = v; +} + +static size_t map_qlen(const OpMap *m) { return m->qn - m->qh; } + +/* resolve() of the pending resume/next promise */ +static void map_wake(ScrPromise **slot) { + if (!*slot) return; + ScrPromise *p = *slot; + *slot = NULL; + op_fulfill(p, scr_dyn_undefined()); + scr_promise_release(p); +} + +static void map_maybe_resume(OpMap *m) { + if (m->resume_p && !m->done && m->cnt < m->concurrency && (double)map_qlen(m) < m->hwm) { + map_wake(&m->resume_p); + } +} + +static void map_after_item(OpMap *m) { + m->cnt -= 1; + map_maybe_resume(m); +} + +/* PromisePrototypeThen(val, afterItemProcessed, onCatch) */ +static void map_item_settled_k(void *self, ScrDyn *v) { + OpMap *m = self; + if (v) { + scr_dyn_release(v); + } else { + scr_exc_clear(); + m->done = true; + } + map_after_item(m); +} + +/* ── the async wrappers around the user's callback ── */ + +/* An async function's promise settled from an awaited inner value. */ +typedef struct { + OpObj base; + ScrPromise *p; /* the wrapper's promise */ + ScrDyn *chunk; /* the source value (filter returns it) */ + int mode; + int stage; /* MODE_EVERY: 0 the negating arrow, 1 filterFn */ +} OpWrap; + +static void op_wrap_drop(OpObj *o) { + OpWrap *w = (OpWrap *)o; + scr_promise_release(w->p); + if (w->chunk) scr_dyn_release(w->chunk); + free(w); +} + +static OpWrap *op_wrap_new(int mode, ScrDyn *chunk, int stage) { + OpWrap *w = op_alloc(sizeof *w); + w->base.rc = 1; + w->base.drop = &op_wrap_drop; + w->p = scr_promise_new(); + w->chunk = chunk ? scr_dyn_retain(chunk) : NULL; + w->mode = mode; + w->stage = stage; + return w; +} + +static void op_wrap_k(void *self, ScrDyn *v) { + OpWrap *w = self; + if (!v) { + scr_promise_reject_pending(w->p); + return; + } + ScrDyn *out; + if (w->mode == MODE_FOREACH) { + out = scr_dyn_retain(op_k_empty); + } else if (w->mode == MODE_EVERY && w->stage == 0) { + out = scr_dyn_new_bool(!scr_dyn_truthy(v)); + } else { + out = scr_dyn_retain(scr_dyn_truthy(v) ? w->chunk : op_k_empty); + } + scr_dyn_release(v); + op_fulfill(w->p, out); + scr_dyn_release(out); +} + +/* Calls fn(chunk): +1 result, or NULL with the exception pending. */ +static ScrDyn *op_call(ScrDyn *fn, ScrDyn *chunk) { + ScrDyn *args[1] = {chunk}; + return scr_dyn_call(fn, args, 1, "fn"); +} + +/* An async wrapper body: `await fn(chunk)` then its transform. The + * wrapper's promise rejects at once when fn throws synchronously. +1. */ +static ScrPromise *op_wrap_call(OpMap *m, ScrDyn *chunk, int mode, int stage) { + OpWrap *w = op_wrap_new(mode, chunk, stage); + ScrPromise *result = scr_promise_retain(w->p); + ScrDyn *r = op_call(m->fn, chunk); + if (!r) { + scr_promise_reject_pending(w->p); + } else { + op_await_value(r, &op_wrap_k, w); + scr_dyn_release(r); + } + op_unref(w); + return result; +} + +/* every's filterFn: `await predicate(value)` where predicate is the + * negating async arrow, then filterFn's own test. */ +static ScrPromise *op_every_call(OpMap *m, ScrDyn *chunk) { + ScrPromise *inner = op_wrap_call(m, chunk, MODE_EVERY, 0); + OpWrap *w = op_wrap_new(MODE_EVERY, chunk, 1); + ScrPromise *result = scr_promise_retain(w->p); + op_await(inner, &op_wrap_k, w); + scr_promise_release(inner); + op_unref(w); + return result; +} + +/* val = fn(val, signalOpt), then PromiseResolve(val) — or + * PromiseReject(err) when the call throws. +1 dyn promise. */ +static ScrDyn *map_item(OpMap *m, ScrDyn *chunk) { + ScrPromise *p; + if (m->mode == MODE_MAP) { + ScrDyn *r = op_call(m->fn, chunk); + if (!r) { + ScrCaught *c = scr_exc_take(); + ScrDyn *d = op_rejected_dyn(c); + scr_caught_release(c); + return d; + } + if (r->kind == SCR_DYN_PROMISE) return r; + p = op_resolved(r); + scr_dyn_release(r); + } else if (m->mode == MODE_EVERY) { + p = op_every_call(m, chunk); + } else { + p = op_wrap_call(m, chunk, m->mode, 0); + } + ScrDyn *d = scr_dyn_new_promise(p); + scr_promise_release(p); + return d; +} + +/* ── the pump: `for await (let val of stream)` filling the queue ── */ + +static void map_pump_next(OpMap *m); + +static void map_pump_finally(OpMap *m) { + m->done = true; + map_wake(&m->next_p); +} + +/* catch (err): a rejected entry the consumer will throw. Borrows c. */ +static void map_pump_catch(OpMap *m, const ScrCaught *c) { + ScrDyn *val = op_rejected_dyn(c); + op_await_value(val, &map_item_settled_k, m); + map_push(m, val); + map_pump_finally(m); +} + +/* After the loop's AsyncIteratorClose: a return (`if (done) return`) or + * the AbortError thrown in the body. */ +typedef struct { + OpObj base; + OpMap *m; + ScrCaught *thrown; /* +1, or NULL for the return */ +} OpPumpClose; + +static void op_pump_close_drop(OpObj *o) { + OpPumpClose *c = (OpPumpClose *)o; + op_unref(c->m); + if (c->thrown) scr_caught_release(c->thrown); + free(c); +} + +static void op_pump_close_k(void *self, ScrDyn *v) { + OpPumpClose *c = self; + if (v) scr_dyn_release(v); + else scr_exc_clear(); /* a throw completion keeps the original error */ + if (c->thrown) map_pump_catch(c->m, c->thrown); + else map_pump_finally(c->m); +} + +static void map_pump_close(OpMap *m, ScrCaught *thrown /*moves or NULL*/) { + OpPumpClose *c = op_alloc(sizeof *c); + c->base.rc = 1; + c->base.drop = &op_pump_close_drop; + c->m = op_ref(m); + c->thrown = thrown; + ScrPromise *p = src_return(&m->src); + op_await(p, &op_pump_close_k, c); + scr_promise_release(p); + op_unref(c); +} + +static void map_pump_resumed_k(void *self, ScrDyn *v) { + if (v) scr_dyn_release(v); + map_pump_next(self); +} + +static void map_pump_val_k(void *self, ScrDyn *v) { + OpMap *m = self; + if (!v) { + ScrCaught *c = scr_exc_take(); + map_pump_catch(m, c); + scr_caught_release(c); + return; + } + if (src_done(&m->src, v)) { + scr_dyn_release(v); + map_push(m, scr_dyn_retain(op_k_eof)); + map_pump_finally(m); + return; + } + if (m->done) { + scr_dyn_release(v); + map_pump_close(m, NULL); + return; + } + if (op_aborted(m->signal)) { + scr_dyn_release(v); + map_pump_close(m, op_take_error(op_abort_error(NULL))); + return; + } + ScrDyn *val = map_item(m, v); + scr_dyn_release(v); + m->cnt += 1; + op_await_value(val, &map_item_settled_k, m); + map_push(m, val); + map_wake(&m->next_p); + if (!m->done && ((double)map_qlen(m) >= m->hwm || m->cnt >= m->concurrency)) { + m->resume_p = scr_promise_new(); + op_await(m->resume_p, &map_pump_resumed_k, m); + return; + } + map_pump_next(m); +} + +static void map_pump_next(OpMap *m) { + ScrPromise *p = src_next(&m->src); + op_await(p, &map_pump_val_k, m); + scr_promise_release(p); +} + +/* ── the generator body: awaiting queue[0] and yielding ── */ + +static void map_consume(OpMap *m); + +/* finally { done = true; if (resume) { resume(); resume = null; } } */ +static void map_finally(OpMap *m) { + m->done = true; + map_wake(&m->resume_p); +} + +static void map_cons_next_k(void *self, ScrDyn *v) { + if (v) scr_dyn_release(v); + map_consume(self); +} + +static void map_cons_val_k(void *self, ScrDyn *v) { + OpMap *m = self; + OpGen *g = &m->g; + if (!v) { + ScrCaught *c = scr_exc_take(); + map_finally(m); + gen_complete(g, c); + scr_caught_release(c); + return; + } + if (v == op_k_eof) { + scr_dyn_release(v); + map_finally(m); + gen_complete(g, NULL); + return; + } + if (op_aborted(m->signal)) { + scr_dyn_release(v); + ScrCaught *c = op_take_error(op_abort_error(NULL)); + map_finally(m); + gen_complete(g, c); + scr_caught_release(c); + return; + } + if (v != op_k_empty) { + gen_yield(g, v); + scr_dyn_release(v); + return; + } + scr_dyn_release(v); + scr_dyn_release(m->queue[m->qh++]); + map_maybe_resume(m); + map_consume(m); +} + +static void map_consume(OpMap *m) { + if (map_qlen(m) > 0) { + op_await_value(m->queue[m->qh], &map_cons_val_k, m); + return; + } + m->next_p = scr_promise_new(); + op_await(m->next_p, &map_cons_next_k, m); +} + +/* Resumed after a yield, or started: the first resumption starts the pump + * (pump() runs before the consumer loop), a later one shifts the entry + * that was yielded. */ +static void map_resume(OpGen *g) { + OpMap *m = (OpMap *)g; + if (!m->started) { + m->started = true; + map_pump_next(m); + } else { + scr_dyn_release(m->queue[m->qh++]); + map_maybe_resume(m); + } + map_consume(m); +} + +/* A return or throw delivered at the yield: the finally block, then the + * generator completes. */ +static void map_ret(OpGen *g, ScrDyn *v) { + (void)v; + map_finally((OpMap *)g); + gen_complete(g, NULL); +} + +static void map_thr(OpGen *g, ScrCaught *c) { + map_finally((OpMap *)g); + gen_complete(g, c); +} + +static OpMap *map_new(ScrStream *s, ScrDyn *fn, int mode, double concurrency, double hwm, + ScrDyn *signal) { + OpMap *m = op_alloc(sizeof *m); + gen_init(&m->g, &op_map_body_drop); + m->g.resume = &map_resume; + m->g.ret = &map_ret; + m->g.thr = &map_thr; + m->src.s = scr_stream_retain(s); + m->fn = scr_dyn_retain(fn); + m->signal = signal ? scr_dyn_retain(signal) : NULL; + m->mode = mode; + m->concurrency = concurrency; + m->hwm = hwm; + return m; +} + +/* ── drop and take ─────────────────────────────────────────────────── */ + +/* async function* drop() / take(): `for await (const val of this)` with + * the signal polled before the loop and at every value; take returns once + * its count is spent, closing the source (which destroys it). */ +typedef struct { + OpGen g; + OpSrc src; + ScrDyn *signal; /* +1 or NULL */ + double number; + bool take; + bool started; +} OpSlice; + +static void op_slice_body_drop(OpGen *g) { + OpSlice *t = (OpSlice *)g; + src_release(&t->src); + if (t->signal) scr_dyn_release(t->signal); + free(t); +} + +static void slice_next(OpSlice *t); + +/* AsyncIteratorClose(source), then the generator completes (with the + * thrown exception when one is given). */ +typedef struct { + OpObj base; + OpGen *g; + ScrCaught *thrown; /* +1 or NULL */ +} OpGenClose; + +static void op_gen_close_drop(OpObj *o) { + OpGenClose *c = (OpGenClose *)o; + op_unref(c->g); + if (c->thrown) scr_caught_release(c->thrown); + free(c); +} + +static void op_gen_close_k(void *self, ScrDyn *v) { + OpGenClose *c = self; + if (v) { + scr_dyn_release(v); + gen_complete(c->g, c->thrown); + return; + } + /* A rejected return() replaces a normal completion; a throw keeps its + * own exception. */ + ScrCaught *err = scr_exc_take(); + gen_complete(c->g, c->thrown ? c->thrown : err); + scr_caught_release(err); +} + +static void gen_close_source(OpGen *g, OpSrc *src, ScrCaught *thrown /*moves or NULL*/) { + OpGenClose *c = op_alloc(sizeof *c); + c->base.rc = 1; + c->base.drop = &op_gen_close_drop; + c->g = op_ref(g); + c->thrown = thrown; + ScrPromise *p = src_return(src); + op_await(p, &op_gen_close_k, c); + scr_promise_release(p); + op_unref(c); +} + +static void slice_val_k(void *self, ScrDyn *v) { + OpSlice *t = self; + if (!v) { + ScrCaught *c = scr_exc_take(); + gen_complete(&t->g, c); + scr_caught_release(c); + return; + } + if (src_done(&t->src, v)) { + scr_dyn_release(v); + gen_complete(&t->g, NULL); + return; + } + if (op_aborted(t->signal)) { + scr_dyn_release(v); + gen_close_source(&t->g, &t->src, op_take_error(op_abort_error(NULL))); + return; + } + bool yield; + if (t->take) { + yield = t->number-- > 0; + } else { + yield = t->number-- <= 0; + } + if (yield) { + gen_yield(&t->g, v); + scr_dyn_release(v); + return; + } + scr_dyn_release(v); + if (t->take && t->number <= 0) { + gen_close_source(&t->g, &t->src, NULL); + return; + } + slice_next(t); +} + +static void slice_next(OpSlice *t) { + ScrPromise *p = src_next(&t->src); + op_await(p, &slice_val_k, t); + scr_promise_release(p); +} + +static void slice_resume(OpGen *g) { + OpSlice *t = (OpSlice *)g; + if (!t->started) { + t->started = true; + if (op_aborted(t->signal)) { + ScrCaught *c = op_take_error(op_abort_error(NULL)); + gen_complete(g, c); + scr_caught_release(c); + return; + } + } else if (t->take && t->number <= 0) { + /* `if (number <= 0) return;` after the yield */ + gen_close_source(g, &t->src, NULL); + return; + } + slice_next(t); +} + +static void slice_ret(OpGen *g, ScrDyn *v) { + (void)v; + gen_close_source(g, &((OpSlice *)g)->src, NULL); +} + +static void slice_thr(OpGen *g, ScrCaught *c) { + gen_close_source(g, &((OpSlice *)g)->src, scr_caught_retain(c)); +} + +/* ── flatMap ───────────────────────────────────────────────────────── */ + +/* async function* flatMap() { for await (const val of values) yield* val; } + * over map's generator. yield* over an array (or a string's code points) + * goes through the async-from-sync iterator: each step resolves after the + * element's PromiseResolve job, and yield* awaits that result before + * yielding without a further await. */ +typedef struct { + OpGen g; + OpSrc values; /* the map generator */ + ScrDyn *inner; /* +1: the array or string being delegated to, or NULL */ + size_t idx; /* element index, or the string's byte offset */ + ScrPromise *step; /* the async-from-sync iterator result being awaited */ + bool started; +} OpFlat; + +static void op_flat_body_drop(OpGen *g) { + OpFlat *f = (OpFlat *)g; + src_release(&f->values); + if (f->inner) scr_dyn_release(f->inner); + if (f->step) scr_promise_release(f->step); + free(f); +} + +static void flat_next_value(OpFlat *f); + +/* The next element (+1), or NULL when the delegate is exhausted. */ +static ScrDyn *flat_take_element(OpFlat *f) { + ScrDyn *in = f->inner; + if (in->kind == SCR_DYN_ARR) { + if (f->idx >= in->v.arr.len) return NULL; + ScrDyn *e = in->v.arr.items[f->idx++]; + return e ? scr_dyn_retain(e) : scr_dyn_retain(scr_dyn_undefined()); + } + /* a string iterates by code point */ + const ScrStr *s = in->v.str; + if (f->idx >= s->len) return NULL; + size_t i = f->idx; + unsigned char c = (unsigned char)s->data[i]; + size_t n = c < 0x80 ? 1 : c < 0xE0 ? 2 : c < 0xF0 ? 3 : 4; + if (i + n > s->len) n = s->len - i; + f->idx += n; + ScrStr *piece = scr_str_new(s->data + i, n); + ScrDyn *d = scr_dyn_new_str(piece); + scr_str_release(piece); + return d; +} + +/* Await(innerResult): the iterator result is ready one job later. */ +static void flat_step_k(void *self, ScrDyn *v) { + OpFlat *f = self; + if (f->step) { + scr_promise_release(f->step); + f->step = NULL; + } + if (v == op_k_done || (v && v == op_k_eof)) { + scr_dyn_release(v); + scr_dyn_release(f->inner); + f->inner = NULL; + flat_next_value(f); + return; + } + gen_yield_now(&f->g, v); + scr_dyn_release(v); +} + +/* The async-from-sync continuation: PromiseResolve(value) then the result + * promise settles; yield* awaits that promise. */ +typedef struct { + OpObj base; + ScrPromise *p; + bool done; +} OpFlatWrap; + +static void op_flat_wrap_drop(OpObj *o) { + OpFlatWrap *w = (OpFlatWrap *)o; + scr_promise_release(w->p); + free(w); +} + +static void op_flat_wrap_k(void *self, ScrDyn *v) { + OpFlatWrap *w = self; + if (!v) { + scr_promise_reject_pending(w->p); + return; + } + op_fulfill(w->p, w->done ? op_k_done : v); + scr_dyn_release(v); +} + +static void flat_delegate_step(OpFlat *f) { + ScrDyn *e = flat_take_element(f); + OpFlatWrap *w = op_alloc(sizeof *w); + w->base.rc = 1; + w->base.drop = &op_flat_wrap_drop; + w->p = scr_promise_new(); + w->done = e == NULL; + f->step = scr_promise_retain(w->p); + op_await_value(e ? e : scr_dyn_undefined(), &op_flat_wrap_k, w); + if (e) scr_dyn_release(e); + op_await(w->p, &flat_step_k, f); + op_unref(w); +} + +static void flat_val_k(void *self, ScrDyn *v) { + OpFlat *f = self; + if (!v) { + ScrCaught *c = scr_exc_take(); + gen_complete(&f->g, c); + scr_caught_release(c); + return; + } + if (src_done(&f->values, v)) { + scr_dyn_release(v); + gen_complete(&f->g, NULL); + return; + } + if (v->kind != SCR_DYN_ARR && v->kind != SCR_DYN_STR) { + scr_dyn_release(v); + ScrCaught *c = op_take_error( + op_unsupported("flatMap callback results other than arrays or strings")); + gen_close_source(&f->g, &f->values, c); + return; + } + f->inner = v; + f->idx = 0; + flat_delegate_step(f); +} + +static void flat_next_value(OpFlat *f) { + ScrPromise *p = src_next(&f->values); + op_await(p, &flat_val_k, f); + scr_promise_release(p); +} + +static void flat_resume(OpGen *g) { + OpFlat *f = (OpFlat *)g; + if (!f->started) { + f->started = true; + flat_next_value(f); + return; + } + flat_delegate_step(f); +} + +/* A return during delegation: the async-from-sync iterator's return() + * resolves at once (arrays have no return method), yield* awaits it and + * then the return value, and the return completion leaves the for await, + * closing map's generator. */ +typedef struct { + OpObj base; + OpFlat *f; + int stage; +} OpFlatRet; + +static void op_flat_ret_drop(OpObj *o) { + OpFlatRet *r = (OpFlatRet *)o; + op_unref(r->f); + free(r); +} + +static void op_flat_ret_k(void *self, ScrDyn *v) { + OpFlatRet *r = self; + if (v) scr_dyn_release(v); + else scr_exc_clear(); + if (r->stage == 0) { + r->stage = 1; + op_await_value(scr_dyn_undefined(), &op_flat_ret_k, r); + return; + } + if (r->f->inner) { + scr_dyn_release(r->f->inner); + r->f->inner = NULL; + } + gen_close_source(&r->f->g, &r->f->values, NULL); +} + +static void flat_ret(OpGen *g, ScrDyn *v) { + (void)v; + OpFlatRet *r = op_alloc(sizeof *r); + r->base.rc = 1; + r->base.drop = &op_flat_ret_drop; + r->f = op_ref((OpFlat *)g); + op_await_value(scr_dyn_undefined(), &op_flat_ret_k, r); + op_unref(r); +} + +/* A throw during delegation: the array iterator has no throw method, so + * yield* rejects with Node's TypeError, which leaves the for await and + * closes map's generator. */ +static void flat_thr(OpGen *g, ScrCaught *c) { + (void)c; + OpFlat *f = (OpFlat *)g; + if (f->inner) { + scr_dyn_release(f->inner); + f->inner = NULL; + } + ScrError *e = op_coded_error(SCR_ERR_TYPE, NULL, "The iterator does not provide a 'throw' method."); + gen_close_source(g, &f->values, op_take_error(e)); +} + +/* ── Readable.from(generator) ──────────────────────────────────────── */ + +/* The operator's output stream: from.js's nextAsync pulls the generator + * while push() accepts, and _destroy runs close(error) (the generator's + * throw, then return) before the destroy callback, on a nextTick. */ +typedef struct { + OpObj base; + OpGen *g; /* +1 */ + ScrStream *out; /* borrowed: the stream owns this object */ + bool reading; +} OpFrom; + +static void op_from_drop(OpObj *o) { + OpFrom *f = (OpFrom *)o; + op_unref(f->g); + free(f); +} + +/* One pull: the stream is held for the await. */ +typedef struct { + OpObj base; + OpFrom *from; + ScrStream *out; /* +1 */ +} OpFromStep; + +static void op_from_step_drop(OpObj *o) { + OpFromStep *st = (OpFromStep *)o; + op_unref(st->from); + scr_stream_release(st->out); + free(st); +} + +static void from_next(OpFrom *f, ScrStream *out); + +/* An operator error as a stream error: Error values as themselves, any + * other thrown value refused. +1. */ +static ScrError *op_caught_error(const ScrCaught *c) { + ScrDyn *d = scr_caught_to_dyn(c); + ScrError *e = d ? scr_errdyn_err_of(d) : NULL; + if (d) scr_dyn_release(d); + if (e) return e; + return op_unsupported("a stream operator rejection that is not an Error object"); +} + +static void op_from_destroy_with(ScrStream *out, ScrError *e /*moves*/) { + scr_stream_release(scr_stream_destroy(out, e)); + scr_error_release(e); +} + +static void op_from_step_k(void *self, ScrDyn *v) { + OpFromStep *st = self; + OpFrom *f = st->from; + ScrStream *out = st->out; + if (!v) { + ScrCaught *c = scr_exc_take(); + op_from_destroy_with(out, op_caught_error(c)); + scr_caught_release(c); + return; + } + if (v == op_k_done) { + scr_dyn_release(v); + scr_stream_push_null(out); + return; + } + if (v->kind == SCR_DYN_NULL) { + scr_dyn_release(v); + f->reading = false; + op_from_destroy_with(out, op_coded_error(SCR_ERR_TYPE, "ERR_STREAM_NULL_VALUES", + "May not write null values to stream")); + return; + } + int pushed = scr_stream_push_object(out, v); + scr_dyn_release(v); + if (pushed == -2) { + op_from_destroy_with(out, op_unsupported( + "a stream operator output that mixes strings and Buffers")); + return; + } + if (pushed < 0) { + op_from_destroy_with(out, op_unsupported( + "a stream operator value that is not a string or Buffer (object-mode streams of other values)")); + return; + } + if (pushed) { + from_next(f, out); + return; + } + f->reading = false; +} + +static void from_next(OpFrom *f, ScrStream *out) { + OpFromStep *st = op_alloc(sizeof *st); + st->base.rc = 1; + st->base.drop = &op_from_step_drop; + st->from = op_ref(f); + st->out = scr_stream_retain(out); + ScrPromise *p = gen_next(f->g); + op_await(p, &op_from_step_k, st); + scr_promise_release(p); + op_unref(st); +} + +static OpFrom *op_from_of(ScrClosure *cb) { return scr_box_get_ref(cb->caps[0]); /* +1 */ } + +/* readable._read */ +static void op_from_read_inv(ScrClosure *cb, ScrStream *s, double size) { + (void)size; + OpFrom *f = op_from_of(cb); + if (!f->reading) { + f->reading = true; + from_next(f, s); + } + op_unref(f); +} + +/* close(error): `await iterator.throw(error)` (when there is an error), + * `await value`, then `await iterator.return()` and `await value`; the + * destroy callback follows on a nextTick from the promise's reaction. */ +typedef struct { + OpObj base; + OpFrom *from; + ScrStream *out; /* +1 */ + ScrError *error; /* +1, the destroy error, or NULL */ + ScrError *fail; /* +1, close()'s rejection */ + int stage; +} OpFromClose; + +static void op_from_close_drop(OpObj *o) { + OpFromClose *c = (OpFromClose *)o; + op_unref(c->from); + scr_stream_release(c->out); + if (c->error) scr_error_release(c->error); + if (c->fail) scr_error_release(c->fail); + free(c); +} + +static void op_from_cb_tick(ScrClosure *job) { + OpFromClose *c = scr_box_get_ref(job->caps[0]); + ScrError *e = c->fail ? c->fail : c->error; + scr_stream_destroy_done(c->out, e ? scr_error_retain(e) : NULL); + op_unref(c); +} + +static void op_from_close_settled(OpFromClose *c) { + /* PromisePrototypeThen(close(error), ...): the reaction job, then + * process.nextTick(cb, ...) */ + ScrClosure *job = scr_closure_new((void *)&op_from_cb_tick, 1); + job->caps[0] = scr_box_new_obj(&op_ref, &op_unref, NULL); + scr_box_set_ref(job->caps[0], op_ref(c)); + scr_next_tick(job); +} + +static void op_from_close_k(void *self, ScrDyn *v); + +static void op_from_close_step(OpFromClose *c, ScrPromise *p) { + op_await(p, &op_from_close_k, c); + scr_promise_release(p); +} + +static void op_from_close_k(void *self, ScrDyn *v) { + OpFromClose *c = self; + if (!v) { + ScrCaught *err = scr_exc_take(); + c->fail = op_caught_error(err); + scr_caught_release(err); + c->stage = 9; + op_await_value(scr_dyn_undefined(), &op_from_close_k, c); + return; + } + bool done = v == op_k_done; + scr_dyn_release(v); + switch (c->stage) { + case 0: /* await iterator.throw(error) */ + c->stage = done ? 2 : 1; + op_await_value(scr_dyn_undefined(), &op_from_close_k, c); /* await value */ + return; + case 1: /* not done: iterator.return() */ + case 3: + c->stage = 4; + op_from_close_step(c, gen_return(c->from->g, scr_dyn_undefined())); + return; + case 2: /* done after the throw: close() returns */ + case 5: + case 9: + op_from_close_settled(c); + return; + case 4: /* await iterator.return() */ + c->stage = 5; + op_await_value(scr_dyn_undefined(), &op_from_close_k, c); /* await value */ + return; + default: + op_from_close_settled(c); + } +} + +static void op_from_destroy_inv(ScrClosure *cb, ScrStream *s, ScrError *err) { + OpFrom *f = op_from_of(cb); + OpFromClose *c = op_alloc(sizeof *c); + c->base.rc = 1; + c->base.drop = &op_from_close_drop; + c->from = f; /* the +1 from op_from_of moves here */ + c->out = scr_stream_retain(s); + c->error = err ? scr_error_retain(err) : NULL; + if (err) { + ScrCaught *thrown = op_take_error(scr_error_retain(err)); + c->stage = 0; + op_from_close_step(c, gen_throw(f->g, thrown)); + scr_caught_release(thrown); + } else { + c->stage = 4; + op_from_close_step(c, gen_return(f->g, scr_dyn_undefined())); + } + op_unref(c); +} + +static ScrClosure *op_from_closure(void *fn, OpFrom *f) { + ScrClosure *c = scr_closure_new(fn, 1); + c->caps[0] = scr_box_new_obj(&op_ref, &op_unref, NULL); + scr_box_set_ref(c->caps[0], op_ref(f)); + return c; +} + +/* Readable.from(generator): +1 object-mode stream (hwm 1). Moves g. */ +static ScrStream *op_from_gen(OpGen *g) { + OpFrom *f = op_alloc(sizeof *f); + f->base.rc = 1; + f->base.drop = &op_from_drop; + f->g = g; + ScrStream *out = scr_stream_new_object_readable(op_from_closure((void *)&op_from_read_inv, f), + &op_from_read_inv, + op_from_closure((void *)&op_from_destroy_inv, f), + &op_from_destroy_inv); + f->out = out; + op_unref(f); + return out; +} + +/* ── the promise operators ─────────────────────────────────────────── */ + +enum { PK_DYN, PK_F64, PK_BOOL, PK_STR, PK_VOID }; + +/* An async operator function: its promise and the loop state. */ +typedef struct { + OpObj base; + ScrPromise *p; + OpSrc src; + ScrDyn *fn; /* +1 or NULL */ + ScrDyn *signal; /* +1 or NULL */ + ScrDyn *acc; /* toArray's array / reduce's accumulator (+1) */ + int op; + int kind; /* the promise payload (PK_*) */ + bool has_init, got_any, negate; +} OpFn; + +enum { FN_TO_ARRAY, FN_REDUCE, FN_SOME, FN_FIND, FN_EVERY, FN_FOR_EACH }; + +static void op_fn_drop(OpObj *o) { + OpFn *a = (OpFn *)o; + scr_promise_release(a->p); + src_release(&a->src); + if (a->fn) scr_dyn_release(a->fn); + if (a->signal) scr_dyn_release(a->signal); + if (a->acc) scr_dyn_release(a->acc); + free(a); +} + +static OpFn *op_fn_new(int op, int kind) { + OpFn *a = op_alloc(sizeof *a); + a->base.rc = 1; + a->base.drop = &op_fn_drop; + a->p = scr_promise_new(); + a->op = op; + a->kind = kind; + return a; +} + +/* Settles the operator's promise with v converted to its payload kind (a + * value of the wrong kind rejects with the checked conversion's error). */ +static void op_fn_resolve(OpFn *a, ScrDyn *v /*borrowed*/) { + switch (a->kind) { + case PK_VOID: + scr_promise_fulfill_void(a->p); + return; + case PK_BOOL: + scr_promise_fulfill_bool(a->p, scr_dyn_truthy(v)); + return; + case PK_F64: + if (v->kind != SCR_DYN_NUM) break; + scr_promise_fulfill_f64(a->p, v->v.num); + return; + case PK_STR: + if (v->kind != SCR_DYN_STR) break; + scr_promise_fulfill_str(a->p, scr_str_retain(v->v.str)); + return; + default: + op_fulfill(a->p, v); + return; + } + static const char m[] = "reduce result does not match its declared type"; + scr_throw_error_msg(SCR_ERR_TYPE, m, sizeof m - 1); + scr_promise_reject_pending(a->p); +} + +static void op_fn_reject(OpFn *a, const ScrCaught *c) { op_reject(a->p, c); } + +static void op_fn_next(OpFn *a); + +/* After the loop's AsyncIteratorClose: the throw that left the loop, or + * the early return's result. */ +typedef struct { + OpObj base; + OpFn *a; + ScrCaught *thrown; /* +1 or NULL */ + ScrDyn *result; /* +1 or NULL */ +} OpFnClose; + +static void op_fn_close_drop(OpObj *o) { + OpFnClose *c = (OpFnClose *)o; + op_unref(c->a); + if (c->thrown) scr_caught_release(c->thrown); + if (c->result) scr_dyn_release(c->result); + free(c); +} + +static void op_fn_finish(OpFn *a, ScrDyn *result); + +static void op_fn_close_k(void *self, ScrDyn *v) { + OpFnClose *c = self; + if (v) { + scr_dyn_release(v); + } else if (!c->thrown) { + ScrCaught *err = scr_exc_take(); + op_fn_reject(c->a, err); + scr_caught_release(err); + return; + } else { + scr_exc_clear(); + } + if (c->thrown) op_fn_reject(c->a, c->thrown); + else op_fn_finish(c->a, c->result); +} + +static void op_fn_close(OpFn *a, ScrCaught *thrown /*moves*/, ScrDyn *result /*moves*/) { + OpFnClose *c = op_alloc(sizeof *c); + c->base.rc = 1; + c->base.drop = &op_fn_close_drop; + c->a = op_ref(a); + c->thrown = thrown; + c->result = result; + ScrPromise *p = src_return(&a->src); + op_await(p, &op_fn_close_k, c); + scr_promise_release(p); + op_unref(c); +} + +/* every: `!(await some(...))` — one more job after some's result. */ +static void op_every_k(void *self, ScrDyn *v) { + OpFn *a = self; + if (!v) { + scr_promise_reject_pending(a->p); + return; + } + scr_promise_fulfill_bool(a->p, !scr_dyn_truthy(v)); + scr_dyn_release(v); +} + +static void op_fn_finish(OpFn *a, ScrDyn *result) { + op_fn_resolve(a, result ? result : scr_dyn_undefined()); +} + +static void op_reduce_value_k(void *self, ScrDyn *v) { + OpFn *a = self; + if (!v) { + /* the rejection leaves the for await: AsyncIteratorClose */ + op_fn_close(a, scr_exc_take(), NULL); + return; + } + scr_dyn_release(a->acc); + a->acc = v; + op_fn_next(a); +} + +static void op_fn_val_k(void *self, ScrDyn *v) { + OpFn *a = self; + if (!v) { + ScrCaught *c = scr_exc_take(); + op_fn_reject(a, c); + scr_caught_release(c); + return; + } + if (src_done(&a->src, v)) { + scr_dyn_release(v); + switch (a->op) { + case FN_TO_ARRAY: + case FN_REDUCE: + if (a->op == FN_REDUCE && !a->got_any && !a->has_init) { + ScrError *e = op_coded_error(SCR_ERR_TYPE, "ERR_MISSING_ARGS", + "Reduce of an empty stream requires an initial value"); + ScrCaught *c = op_take_error(e); + op_fn_reject(a, c); + scr_caught_release(c); + return; + } + op_fn_finish(a, a->acc); + return; + case FN_SOME: + case FN_EVERY: { + ScrDyn *f = scr_dyn_new_bool(false); + op_fn_finish(a, f); + scr_dyn_release(f); + return; + } + default: + op_fn_finish(a, NULL); + return; + } + } + switch (a->op) { + case FN_TO_ARRAY: + if (op_aborted(a->signal)) { + ScrDyn *reason = op_get(a->signal, "reason"); + ScrError *e = op_abort_error(reason ? reason : scr_dyn_undefined()); + if (reason) scr_dyn_release(reason); + scr_dyn_release(v); + op_fn_close(a, op_take_error(e), NULL); + return; + } + scr_dyn_arr_push(a->acc, v); /* moves v */ + op_fn_next(a); + return; + case FN_REDUCE: { + a->got_any = true; + if (op_aborted(a->signal)) { + scr_dyn_release(v); + op_fn_close(a, op_take_error(op_abort_error(NULL)), NULL); + return; + } + if (!a->has_init) { + scr_dyn_release(a->acc); + a->acc = v; + a->has_init = true; + op_fn_next(a); + return; + } + ScrDyn *args[2] = {a->acc, v}; + ScrDyn *r = scr_dyn_call(a->fn, args, 2, "reducer"); + scr_dyn_release(v); + if (!r) { + op_fn_close(a, scr_exc_take(), NULL); + return; + } + op_await_value(r, &op_reduce_value_k, a); + scr_dyn_release(r); + return; + } + case FN_SOME: + case FN_EVERY: { + scr_dyn_release(v); + op_fn_close(a, NULL, scr_dyn_new_bool(true)); + return; + } + case FN_FIND: + op_fn_close(a, NULL, v); + return; + default: /* forEach never receives a value */ + scr_dyn_release(v); + op_fn_next(a); + return; + } +} + +static void op_fn_next(OpFn *a) { + ScrPromise *p = src_next(&a->src); + op_await(p, &op_fn_val_k, a); + scr_promise_release(p); +} + +/* ── validation and entry points ───────────────────────────────────── */ + +/* map()'s option reading: concurrency, highWaterMark (MathFloor, then + * validateInteger). False with the error pending. */ +static bool op_map_options(const ScrDyn *options, double *concurrency, double *hwm) { + *concurrency = 1; + ScrDyn *c = op_get(options, "concurrency"); + if (!op_nullish(c)) { + double n = scr_dyn_number_coerce(c); + if (scr_exc_pending()) { + scr_dyn_release(c); + return false; + } + *concurrency = floor(n); + } + if (c) scr_dyn_release(c); + *hwm = *concurrency - 1; + ScrDyn *h = op_get(options, "highWaterMark"); + if (!op_nullish(h)) { + double n = scr_dyn_number_coerce(h); + if (scr_exc_pending()) { + scr_dyn_release(h); + return false; + } + *hwm = floor(n); + } + if (h) scr_dyn_release(h); + const double max = 9007199254740991.0; + if (!isfinite(*concurrency) || floor(*concurrency) != *concurrency) { + op_throw_out_of_range("options.concurrency", "an integer", *concurrency); + return false; + } + if (*concurrency < 1 || *concurrency > max) { + op_throw_out_of_range("options.concurrency", ">= 1 && <= 9007199254740991", *concurrency); + return false; + } + if (!isfinite(*hwm) || floor(*hwm) != *hwm) { + op_throw_out_of_range("options.highWaterMark", "an integer", *hwm); + return false; + } + if (*hwm < 0 || *hwm > max) { + op_throw_out_of_range("options.highWaterMark", ">= 0 && <= 9007199254740991", *hwm); + return false; + } + *hwm += *concurrency; + return true; +} + +/* map(fn, options) with fn already checked: +1 generator, or NULL with + * the error pending. */ +static OpMap *op_map_make(ScrStream *s, ScrDyn *fn, ScrDyn *options, int mode) { + op_sentinels_init(); + if (!op_check_options(options)) return NULL; + bool ok; + ScrDyn *signal = op_signal(options, &ok); + if (!ok) return NULL; + double concurrency, hwm; + if (!op_map_options(options, &concurrency, &hwm)) { + if (signal) scr_dyn_release(signal); + return NULL; + } + OpMap *m = map_new(s, fn, mode, concurrency, hwm, signal); + if (signal) scr_dyn_release(signal); + return m; +} + +ScrStream *scr_stream_op_map(ScrStream *s, ScrDyn *fn, ScrDyn *options) { + if (!op_check_fn(fn, "fn", 1)) return NULL; + OpMap *m = op_map_make(s, fn, options, MODE_MAP); + return m ? op_from_gen(&m->g) : NULL; +} + +ScrStream *scr_stream_op_filter(ScrStream *s, ScrDyn *fn, ScrDyn *options) { + if (!op_check_fn(fn, "fn", 1)) return NULL; + OpMap *m = op_map_make(s, fn, options, MODE_FILTER); + return m ? op_from_gen(&m->g) : NULL; +} + +ScrStream *scr_stream_op_flat_map(ScrStream *s, ScrDyn *fn, ScrDyn *options) { + if (!op_check_fn(fn, "fn", 1)) return NULL; + OpMap *m = op_map_make(s, fn, options, MODE_MAP); + if (!m) return NULL; + OpFlat *f = op_alloc(sizeof *f); + gen_init(&f->g, &op_flat_body_drop); + f->g.resume = &flat_resume; + f->g.ret = &flat_ret; + f->g.thr = &flat_thr; + f->values.g = &m->g; /* moves */ + return op_from_gen(&f->g); +} + +/* drop(number, options) / take(number, options) */ +static ScrStream *op_slice(ScrStream *s, ScrDyn *number, ScrDyn *options, bool take) { + op_sentinels_init(); + if (!op_check_options(options)) return NULL; + bool ok; + ScrDyn *signal = op_signal(options, &ok); + if (!ok) return NULL; + /* toIntegerOrInfinity: Number(number), NaN → 0, negative → out of range */ + double n = scr_dyn_number_constructor(number ? number : scr_dyn_undefined()); + if (scr_exc_pending()) { + if (signal) scr_dyn_release(signal); + return NULL; + } + if (isnan(n)) n = 0; + if (n < 0) { + op_throw_out_of_range("number", ">= 0", n); + if (signal) scr_dyn_release(signal); + return NULL; + } + OpSlice *t = op_alloc(sizeof *t); + gen_init(&t->g, &op_slice_body_drop); + t->g.resume = &slice_resume; + t->g.ret = &slice_ret; + t->g.thr = &slice_thr; + t->src.s = scr_stream_retain(s); + t->signal = signal; + t->number = n; + t->take = take; + return op_from_gen(&t->g); +} + +ScrStream *scr_stream_op_drop(ScrStream *s, ScrDyn *number, ScrDyn *options) { + return op_slice(s, number, options, false); +} + +ScrStream *scr_stream_op_take(ScrStream *s, ScrDyn *number, ScrDyn *options) { + return op_slice(s, number, options, true); +} + +/* An async operator whose synchronous prefix threw: the promise rejects. */ +static ScrPromise *op_fn_reject_now(OpFn *a) { + scr_promise_reject_pending(a->p); + ScrPromise *p = scr_promise_retain(a->p); + op_unref(a); + return p; +} + +static ScrPromise *op_fn_start(OpFn *a) { + ScrPromise *p = scr_promise_retain(a->p); + op_fn_next(a); + op_unref(a); + return p; +} + +ScrPromise *scr_stream_op_to_array(ScrStream *s, ScrDyn *options) { + op_sentinels_init(); + OpFn *a = op_fn_new(FN_TO_ARRAY, PK_DYN); + if (!op_check_options(options)) return op_fn_reject_now(a); + bool ok; + a->signal = op_signal(options, &ok); + if (!ok) return op_fn_reject_now(a); + a->acc = scr_dyn_new_arr(); + a->src.s = scr_stream_retain(s); + return op_fn_start(a); +} + +/* reduce's pre-aborted path: destroy with the AbortError, wait for the + * stream to finish, then reject with that error. */ +static void op_reduce_aborted_k(void *self, ScrDyn *v) { + OpFn *a = self; + if (v) scr_dyn_release(v); + else scr_exc_clear(); + ScrError *e = scr_stream_errored(a->src.s); + if (!e) e = op_abort_error(NULL); + ScrCaught *c = op_take_error(e); + op_fn_reject(a, c); + scr_caught_release(c); +} + +static void op_noop_listener(ScrClosure *cb, void *err) { + (void)cb; + (void)err; +} + +ScrPromise *scr_stream_op_reduce(ScrStream *s, ScrDyn *fn, ScrDyn *init, bool has_init, + ScrDyn *options, double kind) { + op_sentinels_init(); + OpFn *a = op_fn_new(FN_REDUCE, (int)kind); + if (!op_check_fn(fn, "reducer", 2)) return op_fn_reject_now(a); + if (!op_check_options(options)) return op_fn_reject_now(a); + bool ok; + a->signal = op_signal(options, &ok); + if (!ok) return op_fn_reject_now(a); + a->fn = scr_dyn_retain(fn); + a->has_init = has_init; + a->acc = scr_dyn_retain(init && has_init ? init : scr_dyn_undefined()); + a->src.s = scr_stream_retain(s); + if (op_aborted(a->signal)) { + ScrDyn *reason = op_get(a->signal, "reason"); + ScrError *e = op_abort_error(reason ? reason : scr_dyn_undefined()); + if (reason) scr_dyn_release(reason); + /* this.once('error', () => {}) */ + ScrStr *name = scr_str_new("error", 5); + scr_emitter_release(scr_emitter_on((ScrEmitter *)s, name, + scr_closure_new((void *)&op_noop_listener, 0), + scr_ee_inv_fixed1, true, false)); + scr_str_release(name); + scr_stream_release(scr_stream_destroy(s, e)); + scr_error_release(e); + ScrPromise *fin = scr_sp_finished(s); + ScrPromise *p = scr_promise_retain(a->p); + op_await(fin, &op_reduce_aborted_k, a); + scr_promise_release(fin); + op_unref(a); + return p; + } + return op_fn_start(a); +} + +/* some/find/every/forEach: the loop over map's generator (filter's for + * some, find and every). */ +static ScrPromise *op_gen_fn(ScrStream *s, ScrDyn *fn, ScrDyn *options, int op, int mode, int kind) { + op_sentinels_init(); + OpFn *a = op_fn_new(op, kind); + if (!op_check_fn(fn, "fn", 1)) return op_fn_reject_now(a); + OpMap *m = op_map_make(s, fn, options, mode); + if (!m) return op_fn_reject_now(a); + a->src.g = &m->g; /* moves */ + return op_fn_start(a); +} + +ScrPromise *scr_stream_op_some(ScrStream *s, ScrDyn *fn, ScrDyn *options) { + return op_gen_fn(s, fn, options, FN_SOME, MODE_FILTER, PK_BOOL); +} + +ScrPromise *scr_stream_op_find(ScrStream *s, ScrDyn *fn, ScrDyn *options) { + return op_gen_fn(s, fn, options, FN_FIND, MODE_FILTER, PK_DYN); +} + +ScrPromise *scr_stream_op_for_each(ScrStream *s, ScrDyn *fn, ScrDyn *options) { + return op_gen_fn(s, fn, options, FN_FOR_EACH, MODE_FOREACH, PK_VOID); +} + +/* every(fn, options) = !(await some.call(this, async (...args) => + * !(await fn(...args)), options)) */ +ScrPromise *scr_stream_op_every(ScrStream *s, ScrDyn *fn, ScrDyn *options) { + op_sentinels_init(); + OpFn *outer = op_fn_new(FN_EVERY, PK_BOOL); + if (!op_check_fn(fn, "fn", 1)) return op_fn_reject_now(outer); + ScrPromise *some = op_gen_fn(s, fn, options, FN_EVERY, MODE_EVERY, PK_DYN); + ScrPromise *p = scr_promise_retain(outer->p); + op_await(some, &op_every_k, outer); + scr_promise_release(some); + op_unref(outer); + return p; +} diff --git a/tests/corpus/5335-stream-operators-values.ts b/tests/corpus/5335-stream-operators-values.ts new file mode 100644 index 000000000..be56f32c6 --- /dev/null +++ b/tests/corpus/5335-stream-operators-values.ts @@ -0,0 +1,21 @@ +import { Readable } from "node:stream"; + +const bytes = (): Readable => new Readable({ read() { this.push(Buffer.from("ab")); this.push(Buffer.from("cd")); this.push(null); } }); +console.log(await bytes().toArray()); +console.log(await bytes().map((b: Buffer) => b.toString("hex")).toArray()); +console.log(await bytes().filter((b: Buffer) => b[0] === 0x63).toArray()); +console.log(await bytes().drop(1).toArray(), await bytes().take(1).toArray()); +console.log(await bytes().reduce((n: number, b: Buffer) => n + b.length, 0)); +console.log(await bytes().find((b: Buffer) => b.length === 2), await bytes().every((b: Buffer) => b.length === 2)); +console.log(await Readable.from(["a", "b"]).flatMap((c: string) => [c, c + c]).toArray()); +console.log(await Readable.from(["a", "b"]).map(async (c: string) => c + "!", { concurrency: 2 }).toArray()); +try { await Readable.from(["x"]).reduce((a: string, b: string) => a + b); } catch (e) { console.log((e as Error & { code?: string }).code, (e as Error).message); } +try { Readable.from(["a"]).drop(-1); } catch (e) { console.log((e as Error & { code?: string }).code, (e as Error).message); } +try { Readable.from(["a"]).map((x: string) => x, { concurrency: 0 }); } catch (e) { console.log((e as Error & { code?: string }).code, (e as Error).message); } +const ac = new AbortController(); +ac.abort(); +try { await Readable.from(["a"]).toArray({ signal: ac.signal }); } catch (e) { console.log("toArray aborted", (e as Error).name); } +try { await Readable.from(["a"]).map((x: string) => x, { signal: ac.signal }).toArray(); } catch (e) { console.log("map aborted", (e as Error).name); } +try { await Readable.from(["a"]).reduce((a: string, b: string) => a + b, "", { signal: ac.signal }); } catch (e) { console.log("reduce aborted", (e as Error).name); } +const m = Readable.from(["a"]).map((x: string) => x); +console.log(m.readableObjectMode, m.readableHighWaterMark); diff --git a/tests/corpus/5336-stream-operators-promise-timing.ts b/tests/corpus/5336-stream-operators-promise-timing.ts new file mode 100644 index 000000000..016583ba8 --- /dev/null +++ b/tests/corpus/5336-stream-operators-promise-timing.ts @@ -0,0 +1,14 @@ +import { Readable } from "node:stream"; +const log: string[] = []; +const p = Readable.from(["a", "b"]).toArray().then((a: unknown) => log.push(`toArray ${String(a)}`)); +const q = Readable.from(["x", "y"]).some((c: string) => { log.push(`some ${c}`); return c === "x"; }).then((v: boolean) => log.push(`some -> ${v}`)); +const r = Readable.from(["1", "2"]).every((c: string) => { log.push(`every ${c}`); return c !== "2"; }).then((v: boolean) => log.push(`every -> ${v}`)); +const s = Readable.from(["k", "l"]).find((c: string) => c === "l").then((v: unknown) => log.push(`find -> ${String(v)}`)); +const t = Readable.from(["m", "n"]).forEach((c: string) => { log.push(`each ${c}`); }).then(() => log.push("forEach done")); +const u = Readable.from(["p", "q"]).reduce((acc: string, c: string) => acc + c, ">").then((v: string) => log.push(`reduce -> ${v}`)); +let i = 0; +const hop = (): void => { if (i++ < 40) { log.push(`hop ${i}`); Promise.resolve().then(hop); } }; +Promise.resolve().then(hop); +await Promise.all([p, q, r, s, t, u]); +await new Promise((res) => setTimeout(res, 5)); +console.log(log.join("\n")); diff --git a/tests/corpus/5337-stream-operators-concurrency.ts b/tests/corpus/5337-stream-operators-concurrency.ts new file mode 100644 index 000000000..98856194c --- /dev/null +++ b/tests/corpus/5337-stream-operators-concurrency.ts @@ -0,0 +1,40 @@ +import { Readable } from "node:stream"; + +// map output interleaved with microtasks, ticks and immediates +await new Promise((done) => { + const log: string[] = []; + const src = Readable.from(["a", "b", "c"]); + const out = src.map((c: string) => { + log.push(`map ${c}`); + return c.toUpperCase(); + }); + out.on("data", (c: string) => log.push(`data ${c}`)); + out.on("end", () => { + log.push("end"); + console.log(log.join("\n")); + done(); + }); + Promise.resolve() + .then(() => log.push("micro1")) + .then(() => log.push("micro2")) + .then(() => log.push("micro3")); + process.nextTick(() => log.push("tick")); + setImmediate(() => log.push("immediate")); +}); + +// concurrency 2 keeps output order; take() destroys its source +const log: string[] = []; +const wait = (ms: number): Promise => new Promise((r) => setTimeout(r, ms)); +const out = Readable.from(["a", "b", "c", "d"]).map(async (c: string) => { + log.push(`start ${c}`); + await wait(c === "a" ? 20 : 5); + log.push(`end ${c}`); + return c + c; +}, { concurrency: 2 }); +for await (const v of out) log.push(`got ${String(v)}`); +console.log(log.join("\n")); +const taken = Readable.from(["1", "2", "3", "4"]); +const firstTwo = await taken.map((c: string) => { log.length = 0; return c; }).take(2).toArray(); +console.log(firstTwo, taken.destroyed); +const dropped = await Readable.from(["1", "2", "3"]).drop(1.5).toArray(); +console.log(dropped); diff --git a/tests/corpus/5338-stream-operators-checks.cjs b/tests/corpus/5338-stream-operators-checks.cjs new file mode 100644 index 000000000..2ef58c72a --- /dev/null +++ b/tests/corpus/5338-stream-operators-checks.cjs @@ -0,0 +1,38 @@ +'use strict'; +const { Readable } = require('node:stream'); +const r = () => Readable.from(['a', 'b', 'c']); +function syncCheck(label, f) { + try { f(); console.log(label, 'ok'); } catch (e) { console.log(label, e.name, e.code, e.message); } +} +async function asyncCheck(label, p) { + console.log(label); + try { await p; } catch (e) { console.log(' rejected', e.name, e.code, e.message); } +} +async function main() { + syncCheck('map(5)', () => r().map(5)); + syncCheck('filter(null)', () => r().filter(null)); + syncCheck('map options 7', () => r().map((x) => x, 7)); + syncCheck('map options []', () => r().map((x) => x, [])); + syncCheck('map signal {}', () => r().map((x) => x, { signal: {} })); + syncCheck('map concurrency NaN', () => r().map((x) => x, { concurrency: 'two' })); + syncCheck('map concurrency 1.5', () => r().map((x) => x, { concurrency: 1.5 })); + syncCheck('map highWaterMark -1', () => r().map((x) => x, { highWaterMark: -1 })); + syncCheck('take(-0.5)', () => r().take(-0.5)); + syncCheck('drop("x")', () => r().drop('x')); + syncCheck('take options null', () => r().take(1, null)); + await asyncCheck('forEach(1)', r().forEach(1)); + await asyncCheck('toArray(3)', r().toArray(3)); + await asyncCheck('reduce(null)', r().reduce(null)); + await asyncCheck('some({})', r().some({})); + await asyncCheck('every signal 1', r().every((x) => x, { signal: 1 })); + try { await Readable.from([]).reduce((a, b) => a + b); } catch (e) { console.log(e.name, e.code, e.message); } + console.log(await Readable.from(['x']).reduce((a, b) => a + b)); + console.log(await Readable.from(['1', '2']).reduce((a, b) => a + Number(b), 0)); + console.log(await r().map((c) => c + c).filter((c) => c !== 'bb').toArray()); + console.log(await r().drop(0).take(Infinity).toArray(), await r().take(0).toArray()); + const thrower = r().map((c) => { if (c === 'b') throw new Error('bad ' + c); return c; }); + thrower.on('error', (e) => console.log('stream error', e.message)); + try { for await (const v of thrower) console.log('v', v); } catch (e) { console.log('loop threw', e.message); } + try { await r().forEach(async (c) => { if (c === 'c') throw new Error('async bad'); }); } catch (e) { console.log('forEach rejected', e.message); } +} +main(); diff --git a/tests/corpus/5339-stream-for-await-hops.ts b/tests/corpus/5339-stream-for-await-hops.ts new file mode 100644 index 000000000..0a9c99544 --- /dev/null +++ b/tests/corpus/5339-stream-for-await-hops.ts @@ -0,0 +1,17 @@ +import { Readable } from "node:stream"; +const log: string[] = []; +let i = 0; +const hop = (): void => { if (i++ < 12) { log.push(`hop ${i}`); Promise.resolve().then(hop); } }; +async function run(): Promise { + for await (const c of Readable.from(["a", "b"])) log.push(`got ${String(c)}`); + log.push("loop done"); +} +Promise.resolve().then(hop); +const p = run(); +await p; +await new Promise((r) => setTimeout(r, 5)); +console.log(log.join(" | ")); + +// pushes made inside the iterator's own _read arrive as one chunk +const r = new Readable({ read() { this.push(Buffer.from("ab")); this.push(Buffer.from("cd")); this.push(null); } }); +for await (const c of r) console.log("chunk", c); diff --git a/tests/corpus/5340-readable-from-options.ts b/tests/corpus/5340-readable-from-options.ts new file mode 100644 index 000000000..2a9df7ec8 --- /dev/null +++ b/tests/corpus/5340-readable-from-options.ts @@ -0,0 +1,48 @@ +import { Readable } from "node:stream"; + +const a = Readable.from(["x", "y"]); +console.log("array", a.readableLength, a.readableHighWaterMark, a.readableObjectMode); +const s = Readable.from("hello"); +console.log("string", s.readableLength, s.readableHighWaterMark, s.readableObjectMode); +const b = Readable.from(Buffer.from("hi")); +console.log("buffer", b.readableHighWaterMark, b.readableObjectMode); +const bytes = Readable.from(["ab", "cd"], { objectMode: false }); +console.log("objectMode false", bytes.readableObjectMode, bytes.readableHighWaterMark); +bytes.on("data", (c: Buffer) => console.log("bytes data", c)); +const hw = Readable.from(["1", "2", "3"], { highWaterMark: 2 }); +console.log("hwm 2", hw.readableHighWaterMark); +hw.on("data", (c: string) => console.log("hw data", c, hw.readableLength)); +hw.on("close", () => console.log("hw close")); +for (const h of [-1, 1.5, NaN, Infinity]) { + try { + Readable.from(["a"], { highWaterMark: h }); + } catch (err) { + console.log((err as NodeJS.ErrnoException).code, (err as Error).message); + } +} +const lazy = Readable.from([Buffer.from("p"), Buffer.from("q")]); +lazy.on("readable", () => { + let c: Buffer | null; + while ((c = lazy.read()) !== null) console.log("read", String(c), lazy.readableLength); +}); +lazy.on("end", () => console.log("lazy end")); +lazy.on("close", () => console.log("lazy close")); +const sb = Readable.from(Buffer.from("xyz"), { objectMode: false }); +sb.on("data", (c: Buffer) => console.log("single bytes", c, sb.readableHighWaterMark)); +const d = Readable.from(["d1", "d2"]); +d.on("close", () => console.log("destroyed close")); +d.destroy(); +process.nextTick(() => console.log("tick 1")); +queueMicrotask(() => console.log("micro 1")); +const ord = Readable.from(["o1", "o2", "o3"]); +ord.on("data", (c: string) => { + console.log("ord", c); + process.nextTick(() => console.log("ord tick", c)); +}); +ord.on("end", () => console.log("ord end")); +ord.on("close", () => console.log("ord close")); +// an `any` source takes the checked-dynamic form, options included +const anySrc: any = JSON.parse('["a","b"]'); +const fromAny = Readable.from(anySrc, { objectMode: false }); +console.log("any source", fromAny.readableObjectMode, fromAny.readableHighWaterMark); +fromAny.on("data", (c: Buffer) => console.log("any data", c)); diff --git a/tests/corpus/5341-readable-from-dynamic.cjs b/tests/corpus/5341-readable-from-dynamic.cjs new file mode 100644 index 000000000..70b5919b5 --- /dev/null +++ b/tests/corpus/5341-readable-from-dynamic.cjs @@ -0,0 +1,34 @@ +// @no-node-shims +'use strict'; +const { Readable } = require('node:stream'); +const e = Readable.from([]); +e.on('end', () => console.log('empty end')); +e.resume(); +for (const bad of [42, null, undefined, { a: 1 }, true, Promise.resolve(1)]) { + try { Readable.from(bad); console.log('accepted', bad); } catch (err) { console.log(err.code, err.message); } +} +const n = Readable.from(['ok', null, 'later']); +n.on('data', (c) => console.log('null-stream data', c)); +n.on('error', (err) => console.log('null-stream error', err.name, err.code, err.message)); +n.on('close', () => console.log('null-stream close')); +const bm = Readable.from(['ok', 7], { objectMode: false }); +bm.on('data', (c) => console.log('bm data', c)); +bm.on('error', (err) => console.log('bm error', err.name, err.code, err.message)); +const dynSrc = JSON.parse('["j1","j2"]'); +const j = Readable.from(dynSrc, { highWaterMark: 4 }); +console.log('dyn', j.readableHighWaterMark, j.readableObjectMode, j.readableLength); +j.on('data', (c) => console.log('dyn data', typeof c, c)); +const und = Readable.from(['u'], { highWaterMark: undefined }); +console.log('undef hwm', und.readableHighWaterMark); +const und2 = Readable.from(['u'], { objectMode: false, highWaterMark: null }); +console.log('null hwm bytes', und2.readableHighWaterMark, und2.readableObjectMode); +try { Readable.from(['x'], { highWaterMark: '2' }); } catch (err) { console.log(err.code, err.message); } +const one = Readable.from('single'); +one.on('data', (c) => console.log('single', c)); +const opt = Readable.from(Buffer.from('bb'), JSON.parse('{"objectMode":false}')); +opt.on('data', (c) => console.log('opt', c, opt.readableObjectMode)); +(async () => { + for await (const c of Readable.from(['f1', 'f2'])) console.log('for await', c); + const arr = await Readable.from(['t1', 't2']).toArray(); + console.log('toArray', arr); +})(); diff --git a/tests/corpus/5342-stream-compose.ts b/tests/corpus/5342-stream-compose.ts new file mode 100644 index 000000000..25959634b --- /dev/null +++ b/tests/corpus/5342-stream-compose.ts @@ -0,0 +1,72 @@ +import { compose, PassThrough, Readable, Transform, Writable } from "node:stream"; + +const upper = new Transform({ + transform(chunk: Buffer, _enc, cb) { + cb(null, chunk.toString().toUpperCase()); + }, +}); +const exclaim = new Transform({ + transform(chunk: Buffer, _enc, cb) { + cb(null, chunk.toString() + "!"); + }, +}); +const d = compose(upper, exclaim); +console.log("duplex", d.readable, d.writable, d.readableObjectMode); +d.on("data", (c: Buffer) => console.log("data", c.toString())); +d.on("finish", () => console.log("finish")); +d.on("end", () => console.log("end")); +d.on("close", () => console.log("close")); +d.write("a"); +d.write("b"); +d.end("c"); + +const p = new PassThrough(); +console.log("single", compose(p) === p); + +try { + compose(); +} catch (err) { + console.log((err as Error).name, (err as NodeJS.ErrnoException).code, (err as Error).message); +} + +// A readable head: the composed stream is read-only. +const src = Readable.from(["x", "y"]); +const pt = new PassThrough(); +const r = compose(src, pt); +console.log("read-only", r.readable, r.writable); +r.on("data", (c: Buffer) => console.log("r data", c.toString())); +r.on("end", () => console.log("r end")); +r.on("close", () => console.log("r close")); + +// A writable tail: the composed stream is write-only. +const seen: string[] = []; +const sink = new Writable({ + write(chunk: Buffer, _enc, cb) { + seen.push(chunk.toString()); + cb(); + }, +}); +const w = compose(new PassThrough(), sink); +console.log("write-only", w.readable, w.writable); +w.on("finish", () => console.log("w finish", seen.join(","))); +w.on("close", () => console.log("w close")); +w.end("z"); + +// Errors: a failing stage destroys the composed stream. +const bad = new Transform({ + transform(_chunk: Buffer, _enc, cb) { + cb(new Error("boom")); + }, +}); +const e = compose(new PassThrough(), bad); +e.on("error", (err: Error) => console.log("e error", err.message)); +e.on("close", () => console.log("e close")); +e.write("q"); + +// destroy() before the pipeline ends: AbortError. +const k = compose(new PassThrough(), new PassThrough()); +k.on("error", (err: Error) => console.log("k error", err.name, (err as NodeJS.ErrnoException).code)); +k.on("close", () => console.log("k close")); +k.destroy(); + +process.nextTick(() => console.log("tick")); diff --git a/tests/corpus/5343-stream-compose-edges.ts b/tests/corpus/5343-stream-compose-edges.ts new file mode 100644 index 000000000..488012a02 --- /dev/null +++ b/tests/corpus/5343-stream-compose-edges.ts @@ -0,0 +1,51 @@ +import { compose, PassThrough, Readable, Transform, Writable } from "node:stream"; + +// object-mode tail: Readable.from strings through compose +const objs = compose(Readable.from(["a", "b", "c"]), new PassThrough({ objectMode: false })); +console.log("objs", objs.readableObjectMode, objs.readableHighWaterMark); +objs.on("data", (c: Buffer) => console.log("objs data", c.toString())); +objs.on("end", () => console.log("objs end")); + +// an operator stream as the tail: object mode +const mapped = compose(new PassThrough(), new PassThrough()).map((c: Buffer) => c.toString() + "?"); +mapped.on("data", (c: string) => console.log("mapped", c)); + +// backpressure: a slow sink makes the composed writable buffer +const slow = new Writable({ + highWaterMark: 2, + write(chunk: Buffer, _enc, cb) { + console.log("slow write", chunk.toString()); + setTimeout(cb, 1); + }, +}); +const bp = compose(new PassThrough({ highWaterMark: 2 }), slow); +console.log("bp write 1", bp.write("aa")); +console.log("bp write 2", bp.write("bb")); +console.log("bp write 3", bp.write("cc")); +bp.on("drain", () => console.log("bp drain")); +bp.on("finish", () => console.log("bp finish")); +bp.end(); + +// a destroyed stream in the chain +const gone = new PassThrough(); +gone.destroy(); +try { + compose(gone, new PassThrough()); +} catch (err) { + console.log((err as NodeJS.ErrnoException).code, (err as Error).message.slice(0, 44)); +} + +// upper-casing round trip with a for-await reader +const up = compose( + new Transform({ + transform(c: Buffer, _e, cb) { + cb(null, c.toString().toUpperCase()); + }, + }), + new PassThrough(), +); +(async () => { + up.end("hello"); + for await (const c of up) console.log("for await", String(c)); + console.log("done"); +})(); diff --git a/tests/corpus/5344-stream-pipeline-timing.ts b/tests/corpus/5344-stream-pipeline-timing.ts new file mode 100644 index 000000000..163130db3 --- /dev/null +++ b/tests/corpus/5344-stream-pipeline-timing.ts @@ -0,0 +1,71 @@ +import { PassThrough, pipeline, Transform, Writable } from "node:stream"; + +const a = new PassThrough(); +const b = new PassThrough(); +b.resume(); +b.on("close", () => { + console.log("b close"); + process.nextTick(() => console.log("tick after b close")); + queueMicrotask(() => console.log("micro after b close")); +}); +pipeline(a, b, (err?: Error | null) => console.log("cb", err ? err.message : "ok")); +a.end("x"); + +const c = new PassThrough(); +const d = new PassThrough(); +d.on("close", () => { + console.log("d close"); + process.nextTick(() => console.log("tick after d close")); +}); +c.on("close", () => { + console.log("c close"); + process.nextTick(() => console.log("tick after c close")); +}); +pipeline(c, d, (err?: Error | null) => console.log("cb2", err ? err.message : "ok")); +d.destroy(new Error("bad")); + +// a synchronous write error reaches 'error' one tick later + +let k = 0; +const note = (s: string) => console.log(k++, s); +const src = new PassThrough(); +const failing = new Transform({ + transform(chunk: Buffer, _enc, cb) { + note("transform " + chunk.toString()); + cb(new Error("boom")); + }, +}); +failing.on("error", (err: Error) => note("failing error " + err.message)); +src.on("resume", () => note("src resume")); +src.pipe(failing); +src.write("q"); +note("after write"); +let u = 0; +const step = () => { + if (u++ < 3) { + note("step " + u); + process.nextTick(step); + } +}; +process.nextTick(step); + +// 'finish' waits for a pending _final callback, even when end() runs again +const slowFinal = new Writable({ + write(_c, _e, cb) { + cb(); + }, + final(cb) { + console.log("final called"); + setTimeout(() => { + console.log("final done"); + cb(); + }, 5); + }, +}); +slowFinal.on("finish", () => console.log("slow final finish")); +slowFinal.write("a"); +slowFinal.end(); +setTimeout(() => { + slowFinal.end(); + console.log("second end"); +}, 1); From 7e8fee8457217e43496e7fe58b4dcb24e84b84e4 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Wed, 14 Oct 2026 21:01:27 -0500 Subject: [PATCH 2/7] Regenerate the Node compatibility artifacts --- .../generated/node-v24-backlog.json | 60 +++++++++---------- .../generated/node-v24-internal.json | 32 +++++----- 2 files changed, 46 insertions(+), 46 deletions(-) diff --git a/internal/compatibility/generated/node-v24-backlog.json b/internal/compatibility/generated/node-v24-backlog.json index e5179e4be..ca9c3be22 100644 --- a/internal/compatibility/generated/node-v24-backlog.json +++ b/internal/compatibility/generated/node-v24-backlog.json @@ -61927,15 +61927,12 @@ "priority": "high", "tiers": { "static": { - "status": "partial", - "action": "audit-partial", - "verification": "test-backed", - "confidence": "high", - "source": "compiler-feature:stream.stream.Readable.from", - "tests": [ - "tests/corpus/5340-readable-from-options.ts", - "tests/corpus/5341-readable-from-dynamic.cjs" - ] + "status": "not-implemented", + "action": "verify-gap", + "verification": "registry-gap", + "confidence": "low", + "source": "compiler-unmatched:stream", + "tests": [] } } }, @@ -61980,12 +61977,15 @@ "priority": "high", "tiers": { "static": { - "status": "not-implemented", - "action": "verify-gap", - "verification": "registry-gap", - "confidence": "low", - "source": "compiler-unmatched:stream", - "tests": [] + "status": "partial", + "action": "audit-partial", + "verification": "test-backed", + "confidence": "high", + "source": "compiler-feature:stream.stream.Readable.from", + "tests": [ + "tests/corpus/5340-readable-from-options.ts", + "tests/corpus/5341-readable-from-dynamic.cjs" + ] } } }, @@ -62522,15 +62522,12 @@ "priority": "high", "tiers": { "static": { - "status": "partial", - "action": "audit-partial", - "verification": "test-backed", - "confidence": "high", - "source": "compiler-feature:stream.stream.Readable.from", - "tests": [ - "tests/corpus/5340-readable-from-options.ts", - "tests/corpus/5341-readable-from-dynamic.cjs" - ] + "status": "not-implemented", + "action": "verify-gap", + "verification": "registry-gap", + "confidence": "low", + "source": "compiler-unmatched:stream", + "tests": [] } } }, @@ -62575,12 +62572,15 @@ "priority": "high", "tiers": { "static": { - "status": "not-implemented", - "action": "verify-gap", - "verification": "registry-gap", - "confidence": "low", - "source": "compiler-unmatched:stream", - "tests": [] + "status": "partial", + "action": "audit-partial", + "verification": "test-backed", + "confidence": "high", + "source": "compiler-feature:stream.stream.Readable.from", + "tests": [ + "tests/corpus/5340-readable-from-options.ts", + "tests/corpus/5341-readable-from-dynamic.cjs" + ] } } }, diff --git a/internal/compatibility/generated/node-v24-internal.json b/internal/compatibility/generated/node-v24-internal.json index cafca7ddb..7684b044a 100644 --- a/internal/compatibility/generated/node-v24-internal.json +++ b/internal/compatibility/generated/node-v24-internal.json @@ -85673,12 +85673,8 @@ "inherited": true }, "static": { - "status": "partial", - "evidence": "compiler-feature:stream.stream.Readable.from", - "tests": [ - "tests/corpus/5340-readable-from-options.ts", - "tests/corpus/5341-readable-from-dynamic.cjs" - ] + "status": "not-implemented", + "evidence": "compiler-unmatched:stream" }, "anchorSource": "exact" }, @@ -85723,8 +85719,12 @@ "inherited": true }, "static": { - "status": "not-implemented", - "evidence": "compiler-unmatched:stream" + "status": "partial", + "evidence": "compiler-feature:stream.stream.Readable.from", + "tests": [ + "tests/corpus/5340-readable-from-options.ts", + "tests/corpus/5341-readable-from-dynamic.cjs" + ] }, "anchorSource": "exact" }, @@ -86256,12 +86256,8 @@ "inherited": true }, "static": { - "status": "partial", - "evidence": "compiler-feature:stream.stream.Readable.from", - "tests": [ - "tests/corpus/5340-readable-from-options.ts", - "tests/corpus/5341-readable-from-dynamic.cjs" - ] + "status": "not-implemented", + "evidence": "compiler-unmatched:stream" }, "anchorSource": "ancestor" }, @@ -86306,8 +86302,12 @@ "inherited": true }, "static": { - "status": "not-implemented", - "evidence": "compiler-unmatched:stream" + "status": "partial", + "evidence": "compiler-feature:stream.stream.Readable.from", + "tests": [ + "tests/corpus/5340-readable-from-options.ts", + "tests/corpus/5341-readable-from-dynamic.cjs" + ] }, "anchorSource": "ancestor" }, From 24ecefc4acf31bb20f48b2180b30f043185080a9 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Wed, 14 Oct 2026 23:11:39 -0500 Subject: [PATCH 3/7] Keep stream buffer entries in the buffer's representation setEncoding() converted the buffered entries once, but Readable.from's lazy reader, the operators and compose added later object-mode entries in the kind they arrived in, so a Buffer entry could be read as a string. Every insertion now passes the entry's kind: Buffer entries decode through the stream's decoder, entries that would mix strings and Buffers are refused, and the buffer insert checks the kind. setEncoding() on an ended stream also flushes the decoder's partial tail, as in Node 24. --- .../stream-encoded-from-entries.cjs.json | 6 + .../stream-encoded-from-entries.ts.json | 6 + packages/runtime/src/scr_stream.c | 134 ++++++++++++++---- tests/corpus/stream-encoded-from-entries.cjs | 29 ++++ tests/corpus/stream-encoded-from-entries.ts | 71 ++++++++++ 5 files changed, 216 insertions(+), 30 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/stream-encoded-from-entries.cjs.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/stream-encoded-from-entries.ts.json create mode 100644 tests/corpus/stream-encoded-from-entries.cjs create mode 100644 tests/corpus/stream-encoded-from-entries.ts diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/stream-encoded-from-entries.cjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/stream-encoded-from-entries.cjs.json new file mode 100644 index 000000000..15f7272e4 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/stream-encoded-from-entries.cjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/stream-encoded-from-entries.cjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/stream-encoded-from-entries.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/stream-encoded-from-entries.ts.json new file mode 100644 index 000000000..daa9e6e5c --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/stream-encoded-from-entries.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/stream-encoded-from-entries.ts" + ], + "diags": [] +} diff --git a/packages/runtime/src/scr_stream.c b/packages/runtime/src/scr_stream.c index e96ada971..23d1fc0c1 100644 --- a/packages/runtime/src/scr_stream.c +++ b/packages/runtime/src/scr_stream.c @@ -101,7 +101,13 @@ struct ScrStreamState { bool reading, in_read_sync, ended, end_emitted, end_scheduled; bool need_readable, emitted_readable, readable_listening; bool resume_scheduled, maybe_more_scheduled; - bool encoded; /* string-chunk mode (decoder active) */ + /* The buffer's representation: encoded means every entry is a ScrStr, + * otherwise every entry is a ScrBytes (scr_stream_rbuf_push checks + * it). decoding: setEncoding() installed a decoder, so Buffer entries + * decode on the way in; a string object stream without one + * (Readable.from over strings) holds strings but decodes nothing. */ + bool encoded; + bool decoding; ScrStr *enc; /* owned canonical encoding name, or NULL */ double dec_pending; /* scr_strdec packed pending state */ /* The defaultEncoding option's push-side effect: how push(string) @@ -422,7 +428,7 @@ static ScrError *scr_stream_mkerr(const char *code, const char *msg) { /* ── forward decls ────────────────────────────────────────────────────── */ static void scr_stream_flow(ScrStream *s); -static bool scr_stream_add_entry(ScrStream *s, void *entry /*moves*/); +static int scr_stream_add_entry(ScrStream *s, void *entry /*moves*/, bool is_str); static void scr_stream_emit_readable_now(ScrStream *s); static void scr_stream_maybe_read_more(ScrStream *s); static void scr_stream_finish_maybe(ScrStream *s); @@ -454,7 +460,13 @@ static void scr_stream_error_or_destroy(ScrStream *s, ScrError *err) { /* ── readable internals ───────────────────────────────────────────────── */ -static void scr_stream_rbuf_push(ScrStreamState *st, void *chunk /*moves*/, bool front) { +static void scr_stream_rbuf_push(ScrStreamState *st, void *chunk /*moves*/, bool is_str, + bool front) { + if (is_str != st->r.encoded) { + /* every entry has the buffer's representation (encoded: ScrStr) */ + fputs("scriptc: internal error: stream buffer entry of the wrong kind\n", stderr); + abort(); + } if (st->r.n == st->r.cap) { st->r.cap = st->r.cap ? st->r.cap * 2 : 4; st->r.buf = realloc(st->r.buf, st->r.cap * sizeof *st->r.buf); @@ -675,7 +687,7 @@ static bool scr_stream_add_chunk(ScrStream *s, ScrBytes *chunk, bool front) { scr_stream_entry_release(st, entry); if (scr_exc_pending()) return false; } else { - scr_stream_rbuf_push(st, entry, front); + scr_stream_rbuf_push(st, entry, st->r.encoded, front); if (st->r.need_readable) scr_stream_emit_readable_nt(s); } if (!front) scr_stream_maybe_read_more(s); @@ -1106,10 +1118,25 @@ static void scr_from_src_release(ScrFromSrc *f) { static void *scr_from_src_retain_v(void *f) { return scr_from_src_retain(f); } static void scr_from_src_release_v(void *f) { scr_from_src_release(f); } +/* A Readable.from source whose values no longer fit the buffer's kind + * (strings beside Buffers): destroyed with the "not supported yet" Error. */ +static void scr_from_destroy_mixed(ScrStream *s) { + static const char m[] = "a Readable.from source that mixes strings and Buffers is not supported yet"; + ScrStr *msg = scr_str_new(m, sizeof m - 1); + ScrError *e = scr_error_new(SCR_ERR_ERROR, msg); + scr_str_release(msg); + scr_stream_release(scr_stream_destroy(s, e)); + scr_error_release(e); +} + /* One typed element into the stream: a whole entry in object mode, a * chunk in byte mode. Moves e. Returns push's answer. */ static bool scr_from_push_typed(ScrStream *s, void *e, bool strings) { - if (s->st->r.object_entries) return scr_stream_add_entry(s, e); + if (s->st->r.object_entries) { + int pushed = scr_stream_add_entry(s, e, strings); + if (pushed < 0) scr_from_destroy_mixed(s); + return pushed > 0; + } bool more = strings ? scr_stream_push_str(s, e) : scr_stream_push(s, e); if (strings) scr_str_release(e); else scr_bytes_release(e); @@ -1389,20 +1416,50 @@ ScrStream *scr_stream_new_object_readable(ScrClosure *read, ScrStreamReadInv rea return s; } +/* An object-mode entry in the buffer's representation: a Buffer on a + * stream with a decoder decodes through it (Node's decoder.write in + * readableAddChunkPushObjectMode); an entry of the other kind otherwise + * mixes strings and Buffers, which the buffer cannot hold. Moves entry; + * +1, or NULL (entry released). */ +static void *scr_stream_entry_convert(ScrStreamState *st, void *entry, bool is_str) { + if (is_str == st->r.encoded) return entry; + if (!is_str && st->r.decoding) { + ScrBytes *b = entry; + ScrStr *dec = scr_strdec_write(st->r.enc, st->r.dec_pending, b); + st->r.dec_pending = scr_strdec_next(st->r.enc, st->r.dec_pending, b); + scr_bytes_release(b); + return dec; + } + if (is_str) scr_str_release(entry); + else scr_bytes_release(entry); + return NULL; +} + /* readableAddChunk in object mode: every value is one whole entry, empty - * strings and Buffers included. */ -static bool scr_stream_add_entry(ScrStream *s, void *entry /*moves*/) { + * strings and Buffers included. is_str gives entry's kind. Answers push()'s + * result, or -2 (entry released) when the kinds cannot mix. */ +static int scr_stream_add_entry(ScrStream *s, void *entry /*moves*/, bool is_str) { ScrStreamState *st = s->st; + if (!st->r.object_kind_set && st->r.length == 0) { + /* the first entry picks the representation */ + st->r.object_kind_set = true; + if (is_str && !st->r.encoded) { + st->r.encoded = true; + if (!st->r.enc) st->r.enc = scr_str_new("utf8", 4); + } + } + entry = scr_stream_entry_convert(st, entry, is_str); + if (entry == NULL) return -2; if (st->r.ended) { scr_stream_entry_release(st, entry); ScrError *e = scr_stream_mkerr("ERR_STREAM_PUSH_AFTER_EOF", "stream.push() after EOF"); scr_stream_error_or_destroy(s, e); scr_error_release(e); - return false; + return 0; } if (st->destroyed || st->errored) { scr_stream_entry_release(st, entry); - return false; + return 0; } st->r.reading = false; if (st->r.flowing == 1 && st->r.length == 0 && !st->r.in_read_sync && @@ -1410,9 +1467,9 @@ static bool scr_stream_add_entry(ScrStream *s, void *entry /*moves*/) { if (st->r.emitted_readable) st->r.emitted_readable = false; scr_stream_emit_data(s, entry); scr_stream_entry_release(st, entry); - if (scr_exc_pending()) return false; + if (scr_exc_pending()) return 0; } else { - scr_stream_rbuf_push(st, entry, false); + scr_stream_rbuf_push(st, entry, st->r.encoded, false); if (st->r.need_readable) scr_stream_emit_readable_nt(s); } scr_stream_maybe_read_more(s); @@ -1421,18 +1478,11 @@ static bool scr_stream_add_entry(ScrStream *s, void *entry /*moves*/) { } int scr_stream_push_object(ScrStream *s, const ScrDyn *v) { - ScrStreamState *st = s->st; bool is_str = v->kind == SCR_DYN_STR; if (!is_str && v->kind != SCR_DYN_BYTES) return -1; - if (!st->r.object_kind_set) { - st->r.object_kind_set = true; - st->r.encoded = is_str; - if (is_str && !st->r.enc) st->r.enc = scr_str_new("utf8", 4); - } else if (st->r.encoded != is_str) { - return -2; - } void *entry = is_str ? (void *)scr_str_retain(v->v.str) : (void *)scr_bytes_raw_view(v->v.bytes); - return scr_stream_add_entry(s, entry) ? 1 : 0; + int pushed = scr_stream_add_entry(s, entry, is_str); + return pushed < 0 ? pushed : pushed ? 1 : 0; } /* ── the registration hook ('data'/'readable' listeners kick flow) ────── */ @@ -3081,12 +3131,17 @@ static void scr_compose_data_l(ScrClosure *cb, ScrBytes *bytes, ScrStr *str) { ScrStreamState *dst = d->st; bool more; if (dst->r.object_entries) { - if (!dst->r.object_kind_set) { - dst->r.object_kind_set = true; - dst->r.encoded = str != NULL; - if (str && !dst->r.enc) dst->r.enc = scr_str_new("utf8", 4); + int pushed = scr_stream_add_entry( + d, str ? (void *)scr_str_retain(str) : (void *)scr_bytes_retain(bytes), str != NULL); + if (pushed < 0) { + static const char m[] = "a composed stream whose tail mixes strings and Buffers is not supported yet"; + ScrStr *msg = scr_str_new(m, sizeof m - 1); + ScrError *e = scr_error_new(SCR_ERR_ERROR, msg); + scr_str_release(msg); + scr_stream_release(scr_stream_destroy(d, e)); + scr_error_release(e); } - more = scr_stream_add_entry(d, str ? (void *)scr_str_retain(str) : (void *)scr_bytes_retain(bytes)); + more = pushed > 0; } else if (str) { more = scr_stream_push_str(d, str); } else { @@ -4238,7 +4293,7 @@ bool scr_stream_push_null(ScrStream *s) { ScrStr *tail = scr_strdec_end(st->r.enc, st->r.dec_pending); st->r.dec_pending = 0; if (scr_str_utf16_len(tail) > 0) { - scr_stream_rbuf_push(st, tail, false); + scr_stream_rbuf_push(st, tail, true, false); if (st->r.need_readable) scr_stream_emit_readable_nt(s); } else { scr_str_release(tail); @@ -4266,7 +4321,7 @@ void scr_stream_unshift(ScrStream *s, ScrBytes *chunk) { if (st->r.head_off > 0) { /* materialize the part-read head so the front slot is whole */ void *rest = scr_stream_rbuf_take(st, scr_stream_entry_len(st, st->r.buf[0]) - st->r.head_off); - scr_stream_rbuf_push(st, rest, true); + scr_stream_rbuf_push(st, rest, st->r.encoded, true); } scr_stream_add_chunk(s, chunk, true); } @@ -4360,11 +4415,13 @@ ScrStream *scr_stream_set_encoding(ScrStream *s, ScrStr *enc) { /* switching encodings mid-stream: fold the old decoder's pending * tail into the buffer first (rare; Node re-creates the decoder) */ ScrStr *tail = scr_strdec_end(st->r.enc, st->r.dec_pending); - if (scr_str_utf16_len(tail) > 0) scr_stream_rbuf_push(st, tail, false); + if (scr_str_utf16_len(tail) > 0) scr_stream_rbuf_push(st, tail, true, false); else scr_str_release(tail); scr_str_release(st->r.enc); st->r.enc = scr_str_retain(enc); st->r.dec_pending = 0; + st->r.decoding = true; + st->r.object_kind_set = true; return scr_stream_retain(s); } st->r.enc = scr_str_retain(enc); @@ -4392,12 +4449,29 @@ ScrStream *scr_stream_set_encoding(ScrStream *s, ScrStr *enc) { content = joined; } } + if (st->r.ended) { + /* Node 24: an ended stream's decoder flushes its partial tail */ + ScrStr *tail = scr_strdec_end(st->r.enc, st->r.dec_pending); + st->r.dec_pending = 0; + if (content == NULL) { + content = tail; + } else { + ScrStr *joined = scr_str_concat(content, tail); + scr_str_release(content); + scr_str_release(tail); + content = joined; + } + } st->r.n = 0; st->r.head_off = 0; st->r.length = 0; - st->r.encoded = true; /* AFTER the byte-entry walk above */ + /* AFTER the byte-entry walk above: from here every entry is a string, + * and Buffer entries pushed later decode (scr_stream_entry_convert) */ + st->r.encoded = true; + st->r.decoding = true; + st->r.object_kind_set = true; if (content != NULL) { - if (scr_str_utf16_len(content) > 0) scr_stream_rbuf_push(st, content, false); + if (scr_str_utf16_len(content) > 0) scr_stream_rbuf_push(st, content, true, false); else scr_str_release(content); } return scr_stream_retain(s); diff --git a/tests/corpus/stream-encoded-from-entries.cjs b/tests/corpus/stream-encoded-from-entries.cjs new file mode 100644 index 000000000..64c763e4d --- /dev/null +++ b/tests/corpus/stream-encoded-from-entries.cjs @@ -0,0 +1,29 @@ +// @no-node-shims +// setEncoding on Readable.from streams from CommonJS: Buffer, string and +// Uint8Array entries, a decoder set after the first chunk, and pipe. +'use strict'; +const { Readable, Writable } = require('node:stream'); +async function main() { + const mixed = Readable.from([Buffer.from('b1'), 's2', new Uint8Array([0x75, 0x33])]); + mixed.setEncoding('utf8'); + for await (const c of mixed) console.log('mixed', typeof c, c); + const late = Readable.from([Buffer.from('q1'), Buffer.from('q2')], { objectMode: false }); + const rest = []; + await new Promise((resolve) => { + late.once('data', (first) => { + console.log('late first', Buffer.isBuffer(first), String(first)); + late.setEncoding('hex'); + late.on('data', (c) => rest.push(typeof c + ':' + c)); + late.on('end', () => resolve(undefined)); + }); + }); + console.log('late rest', rest.join(' ')); + const seen = []; + const w = new Writable({ write(chunk, enc, cb) { seen.push(chunk.toString('hex')); cb(); } }); + const src = Readable.from([Buffer.from('w1'), Buffer.from('w2')]); + src.setEncoding('utf8'); + src.pipe(w); + await new Promise((resolve) => w.on('finish', resolve)); + console.log('piped', seen.join(',')); +} +main(); diff --git a/tests/corpus/stream-encoded-from-entries.ts b/tests/corpus/stream-encoded-from-entries.ts new file mode 100644 index 000000000..d96c0a505 --- /dev/null +++ b/tests/corpus/stream-encoded-from-entries.ts @@ -0,0 +1,71 @@ +// setEncoding on Readable.from streams: Buffer and Uint8Array entries +// decode through the stream's decoder whether they were buffered before +// setEncoding or pulled after it, in object and byte mode, and reach +// for-await, pipe, pipeline, an operator's output and a composed stream +// as strings. +import { PassThrough, Readable, Writable, compose, pipeline } from "node:stream"; + +const log: string[] = []; +const sink = (label: string): Writable => + new Writable({ + write(chunk: Buffer, _enc, cb) { + log.push(`${label} wrote ${chunk.toString("hex")}`); + cb(); + }, + }); +const drain = async (label: string, r: Readable): Promise => { + for await (const c of r) log.push(`${label} got ${typeof c} ${String(c)}`); +}; + +// setEncoding before the first lazy read +const before = Readable.from([Buffer.from("ab"), Buffer.from("c\\u00e9")]); +before.setEncoding("utf8"); +await drain("before", before); + +// setEncoding after the first lazy read (byte mode: Node re-counts an +// object-mode buffer's length in characters) +const after = Readable.from([Buffer.from("xy"), Buffer.from("zw"), Buffer.from("!")], { + objectMode: false, +}); +await new Promise((resolve) => { + after.once("data", (c: Buffer) => { + log.push(`after first ${c.toString()}`); + after.pause(); + after.setEncoding("hex"); + after.on("data", (s: string) => log.push(`after rest ${s}`)); + after.on("end", () => resolve()); + after.resume(); + }); +}); + +// Uint8Array entries, a split multi-byte character across entries +const split = Readable.from([new Uint8Array([0xe2, 0x82]), new Uint8Array([0xac, 0x41])]); +split.setEncoding("utf8"); +await drain("split", split); + +// byte mode +const bytes = Readable.from([Buffer.from("by"), Buffer.from("te")], { objectMode: false }); +bytes.setEncoding("base64"); +await drain("bytes", bytes); + +// pipe and pipeline delivery of an encoded stream +const piped = Readable.from([Buffer.from("p1"), Buffer.from("p2")]); +piped.setEncoding("utf8"); +await new Promise((resolve) => piped.pipe(sink("pipe")).on("finish", () => resolve())); +const lined = Readable.from([Buffer.from("l1"), Buffer.from("l2")]); +lined.setEncoding("latin1"); +await new Promise((resolve) => + pipeline(lined, new PassThrough(), sink("pipeline"), () => resolve()), +); + +// an operator's output and a composed stream, encoded +const mapped = Readable.from([Buffer.from("m")]).map((b: Buffer) => Buffer.concat([b, b])); +mapped.setEncoding("utf8"); +await drain("mapped", mapped); +const composed = compose(Readable.from([Buffer.from("k1"), Buffer.from("k2")]), new PassThrough()); +composed.setEncoding("utf8"); +const parts: string[] = []; +composed.on("data", (c: string) => parts.push(typeof c + ":" + c)); +await new Promise((resolve) => composed.on("end", () => resolve())); +log.push(`composed ${parts.join("").replace(/string:/g, "")} ${parts.every((p) => p.startsWith("string:"))}`); +console.log(log.join("\n")); From 83c5b8f327e5071b0c6c4ff1a28348d4b4b717fd Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 15 Oct 2026 00:21:35 -0500 Subject: [PATCH 4/7] Make the stream operators concurrency program independent of timer precision 5337 ordered the concurrent map() callbacks with 20 ms and 5 ms timers, so a slow machine changed which callback ended first, in Node as well. The callbacks now resume on setImmediate, and "a" waits for a promise that "c" resolves, which keeps the same order and output. --- tests/corpus/5337-stream-operators-concurrency.ts | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/corpus/5337-stream-operators-concurrency.ts b/tests/corpus/5337-stream-operators-concurrency.ts index 98856194c..ffbefb1ae 100644 --- a/tests/corpus/5337-stream-operators-concurrency.ts +++ b/tests/corpus/5337-stream-operators-concurrency.ts @@ -22,13 +22,17 @@ await new Promise((done) => { setImmediate(() => log.push("immediate")); }); -// concurrency 2 keeps output order; take() destroys its source +// concurrency 2 keeps output order; take() destroys its source. "a" is +// released only after "c" ends and the others resume on setImmediate, so +// the order does not depend on timer precision. const log: string[] = []; -const wait = (ms: number): Promise => new Promise((r) => setTimeout(r, ms)); +const gate = Promise.withResolvers(); const out = Readable.from(["a", "b", "c", "d"]).map(async (c: string) => { log.push(`start ${c}`); - await wait(c === "a" ? 20 : 5); + if (c === "a") await gate.promise; + else await new Promise((r) => setImmediate(r)); log.push(`end ${c}`); + if (c === "c") gate.resolve(); return c + c; }, { concurrency: 2 }); for await (const v of out) log.push(`got ${String(v)}`); From 61d38f9dbe6cea1da42ade83827d2c8dff58cafc Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 15 Oct 2026 00:59:09 -0500 Subject: [PATCH 5/7] Check a stream chunk's kind after the read that produced it A typed read() checked the stream's representation before reading, but the read itself can change it: a lazy Readable.from pull over a dynamic array makes the stream hold strings with its first string, and _read can call setEncoding(). The string then came back typed as a Buffer. Reads now answer each chunk's kind as it was taken from the buffer: typed read() and the typed iterator throw the encoding fence for a string, the dynamic read boxes by that kind, and entries released after a 'data' emit are released by it. --- .../stream-dynamic-entry-kinds.cjs.json | 6 + .../corpus/stream-dynamic-entry-kinds.ts.json | 6 + packages/runtime/src/scr_stream.c | 107 +++++++++--- tests/corpus/stream-dynamic-entry-kinds.cjs | 28 ++++ tests/corpus/stream-dynamic-entry-kinds.ts | 18 ++ tests/harness/stream-entry-kinds.test.ts | 158 ++++++++++++++++++ 6 files changed, 297 insertions(+), 26 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/stream-dynamic-entry-kinds.cjs.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/stream-dynamic-entry-kinds.ts.json create mode 100644 tests/corpus/stream-dynamic-entry-kinds.cjs create mode 100644 tests/corpus/stream-dynamic-entry-kinds.ts create mode 100644 tests/harness/stream-entry-kinds.test.ts diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/stream-dynamic-entry-kinds.cjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/stream-dynamic-entry-kinds.cjs.json new file mode 100644 index 000000000..31e4202c6 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/stream-dynamic-entry-kinds.cjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/stream-dynamic-entry-kinds.cjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/stream-dynamic-entry-kinds.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/stream-dynamic-entry-kinds.ts.json new file mode 100644 index 000000000..3f64353a3 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/stream-dynamic-entry-kinds.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/stream-dynamic-entry-kinds.ts" + ], + "diags": [] +} diff --git a/packages/runtime/src/scr_stream.c b/packages/runtime/src/scr_stream.c index 19525aea8..37ef96577 100644 --- a/packages/runtime/src/scr_stream.c +++ b/packages/runtime/src/scr_stream.c @@ -246,6 +246,13 @@ static void scr_stream_entry_release(const ScrStreamState *st, void *e) { else scr_bytes_release((ScrBytes *)e); } +/* Releases an entry by the kind it had when it left the buffer: a 'data' + * listener may call setEncoding() while the entry is out. */ +static void scr_stream_entry_release_kind(void *e, bool is_str) { + if (is_str) scr_str_release((ScrStr *)e); + else scr_bytes_release((ScrBytes *)e); +} + static void scr_stream_state_drop(ScrStreamState *st, bool gc) { if (!st) return; if (st->errored) scr_error_release(st->errored); @@ -446,6 +453,7 @@ static void scr_stream_clear_buffer(ScrStream *s); static void scr_stream_do_destroy(ScrStream *s, ScrError *err /*borrowed*/); static void scr_stream_notify_finished(ScrStream *s); static void *scr_stream_read_n(ScrStream *s, double size); +static void *scr_stream_read_n_kind(ScrStream *s, double size, bool *is_str); static void scr_stream_settle_next(ScrStream *s); static ScrStream *scr_stream_alloc(const ScrVt *vt, const char *cls, bool has_r, bool has_w, double rhwm, double whwm, bool auto_destroy, @@ -693,8 +701,9 @@ static bool scr_stream_add_chunk(ScrStream *s, ScrBytes *chunk, bool front) { (scr_emitter_has((ScrEmitter *)s, "data") || st->pipes.n > 0)) { /* the direct-emit fast path: flowing, empty buffer, async push */ if (st->r.emitted_readable) st->r.emitted_readable = false; + bool is_str = st->r.encoded; scr_stream_emit_data(s, entry); - scr_stream_entry_release(st, entry); + scr_stream_entry_release_kind(entry, is_str); if (scr_exc_pending()) return false; } else { scr_stream_rbuf_push(st, entry, st->r.encoded, front); @@ -717,7 +726,18 @@ static void scr_stream_maybe_read_more(ScrStream *s) { * Answers a +1 Buffer — or a +1 string once encoded (lengths and the * size argument then count JS string units, Node's encoded accounting). */ static void *scr_stream_read_n(ScrStream *s, double size) { + bool is_str; + return scr_stream_read_n_kind(s, size, &is_str); +} + +/* read_n, also answering the chunk's kind as it was taken from the buffer + * (a string when the stream was encoded then). A _read or a lazy pull + * inside the read can change the representation, and a 'data' listener + * can call setEncoding() before this returns, so callers that need the + * kind read it here rather than from st->r.encoded afterwards. */ +static void *scr_stream_read_n_kind(ScrStream *s, double size, bool *is_str) { ScrStreamState *st = s->st; + *is_str = false; bool absent = size < 0; size_t n = 0; if (!absent) { @@ -774,6 +794,7 @@ static void *scr_stream_read_n(ScrStream *s, double size) { } } } + *is_str = st->r.encoded; void *ret = want > 0 && want <= st->r.length ? scr_stream_rbuf_take(st, want) : NULL; if (ret == NULL) { st->r.need_readable = st->r.length <= st->r.hwm; @@ -788,7 +809,7 @@ static void *scr_stream_read_n(ScrStream *s, double size) { if (ret != NULL && !st->error_emitted && !st->close_emitted) { scr_stream_emit_data(s, ret); if (scr_exc_pending()) { - scr_stream_entry_release(st, ret); + scr_stream_entry_release_kind(ret, *is_str); return NULL; } } @@ -798,9 +819,10 @@ static void *scr_stream_read_n(ScrStream *s, double size) { static void scr_stream_flow(ScrStream *s) { ScrStreamState *st = s->st; while (st->r.flowing == 1 && !scr_exc_pending()) { - void *b = scr_stream_read_n(s, -1); + bool is_str; + void *b = scr_stream_read_n_kind(s, -1, &is_str); if (b == NULL) break; - scr_stream_entry_release(st, b); + scr_stream_entry_release_kind(b, is_str); } } @@ -958,14 +980,32 @@ static void scr_stream_next_yield_job(ScrClosure *job) { scr_promise_release(w); } -static void scr_stream_next_fulfill(ScrStreamState *st, ScrPromise *w /*moves*/, void *chunk /*moves or NULL=EOF*/) { +static void scr_stream_next_str_fence(void) { + static const char msg[] = + "for-await over a stream with an encoding set is not supported yet in " + "typed code (chunks are Buffers here)"; + scr_throw_error_msg(SCR_ERR_ERROR, msg, sizeof msg - 1); +} + +/* Fulfills the parked iterator with a chunk taken from the buffer; is_str + * is its kind as taken. The typed lane's chunk is a Buffer, so a string + * chunk rejects with the encoding fence instead. */ +static void scr_stream_next_fulfill(ScrStreamState *st, ScrPromise *w /*moves*/, + void *chunk /*moves or NULL=EOF*/, bool is_str) { void *payload; bool dyn = st->r.next_dyn; + if (!dyn && chunk != NULL && is_str) { + scr_str_release((ScrStr *)chunk); + scr_stream_next_str_fence(); + scr_promise_reject_pending(w); + scr_promise_release(w); + return; + } if (dyn) { ScrDyn *d; if (chunk == NULL) { d = scr_dyn_retain(scr_dyn_undefined()); /* the EOF sentinel */ - } else if (st->r.encoded) { + } else if (is_str) { d = scr_dyn_new_str((ScrStr *)chunk); scr_str_release((ScrStr *)chunk); } else { @@ -1021,8 +1061,9 @@ static void scr_stream_settle_next(ScrStream *s) { if (st->r.length > 0) { ScrPromise *w = st->r.next_waiter; st->r.next_waiter = NULL; + bool is_str = st->r.encoded; void *chunk = scr_stream_rbuf_take(st, st->r.object_entries ? 1 : st->r.length); - scr_stream_next_fulfill(st, w, chunk); + scr_stream_next_fulfill(st, w, chunk, is_str); if (st->r.ended && st->r.length == 0) scr_stream_end_readable(s); return; } @@ -1037,7 +1078,7 @@ static void scr_stream_settle_next(ScrStream *s) { } ScrPromise *w = st->r.next_waiter; st->r.next_waiter = NULL; - scr_stream_next_fulfill(st, w, NULL); + scr_stream_next_fulfill(st, w, NULL, false); } } @@ -1046,7 +1087,7 @@ static ScrPromise *scr_stream_next_chunk_impl(ScrStream *s, bool dyn) { ScrPromise *p = scr_promise_new(); if (!st->has_r) { st->r.next_dyn = dyn; - scr_stream_next_fulfill(st, scr_promise_retain(p), NULL); + scr_stream_next_fulfill(st, scr_promise_retain(p), NULL, false); return p; } if (st->r.next_waiter) { @@ -1083,10 +1124,7 @@ ScrPromise *scr_stream_next_chunk(ScrStream *s) { if (st->has_r && st->r.encoded) { /* the typed lane's chunk is a Buffer; string chunks would confuse */ ScrPromise *p = scr_promise_new(); - static const char msg[] = - "for-await over a stream with an encoding set is not supported yet in " - "typed code (chunks are Buffers here)"; - scr_throw_error_msg(SCR_ERR_ERROR, msg, sizeof msg - 1); + scr_stream_next_str_fence(); scr_promise_reject_pending(p); return p; } @@ -1475,8 +1513,9 @@ static int scr_stream_add_entry(ScrStream *s, void *entry /*moves*/, bool is_str if (st->r.flowing == 1 && st->r.length == 0 && !st->r.in_read_sync && (scr_emitter_has((ScrEmitter *)s, "data") || st->pipes.n > 0)) { if (st->r.emitted_readable) st->r.emitted_readable = false; + bool is_str = st->r.encoded; scr_stream_emit_data(s, entry); - scr_stream_entry_release(st, entry); + scr_stream_entry_release_kind(entry, is_str); if (scr_exc_pending()) return 0; } else { scr_stream_rbuf_push(st, entry, st->r.encoded, false); @@ -4392,28 +4431,44 @@ ScrStream *scr_stream_on_dyn(ScrStream *s, ScrStr *name, ScrDyn *cb, bool once, ScrDyn *scr_stream_read_dyn(ScrStream *s, ScrDyn *size) { double n = size && size->kind != SCR_DYN_UNDEF ? scr_dyn_number_coerce(size) : -1; if (scr_exc_pending()) return scr_dyn_undefined(); - void *chunk = s->st->has_r ? scr_stream_read_n(s, n) : NULL; + bool is_str = false; + void *chunk = s->st->has_r ? scr_stream_read_n_kind(s, n, &is_str) : NULL; if (!chunk) return scr_dyn_new_null(); - ScrDyn *result = s->st->r.encoded ? scr_dyn_new_str(chunk) : scr_dyn_new_buffer(chunk); - scr_stream_entry_release(s->st, chunk); + ScrDyn *result = is_str ? scr_dyn_new_str(chunk) : scr_dyn_new_buffer(chunk); + if (is_str) scr_str_release(chunk); + else scr_bytes_release(chunk); return result; } +/* read() answers STRINGS once encoded: the static result type here is + * `Buffer | null`, so the honest answer is a fence, not a mistyped + * payload (SEMANTICS.md; 'data' listeners carry strings). */ +static void scr_stream_read_str_fence(void) { + static const char msg[] = + "read() on a stream with an encoding set is not supported yet " + "(consume 'data' events, which deliver strings)"; + scr_throw_error_msg(SCR_ERR_ERROR, msg, sizeof msg - 1); +} + ScrBytes *scr_stream_read(ScrStream *s, double size) { /* a destroyed readable still DRAINS its buffer (Node: destroy leaves * buffered chunks readable; only _read refills are blocked) */ if (!s->st->has_r) return NULL; if (s->st->r.encoded) { - /* read() answers STRINGS once encoded — the static result type here - * is `Buffer | null`, so the honest answer is a fence, not a - * mistyped payload (SEMANTICS.md; 'data' listeners carry strings). */ - static const char msg[] = - "read() on a stream with an encoding set is not supported yet " - "(consume 'data' events, which deliver strings)"; - scr_throw_error_msg(SCR_ERR_ERROR, msg, sizeof msg - 1); + scr_stream_read_str_fence(); return NULL; } - return scr_stream_read_n(s, size); + bool is_str; + void *chunk = scr_stream_read_n_kind(s, size, &is_str); + if (chunk && is_str) { + /* the read itself made the stream hold strings (a lazy Readable.from + * pull's first string, or setEncoding() inside _read): the same + * fence, never a string in the Buffer result */ + scr_str_release(chunk); + scr_stream_read_str_fence(); + return NULL; + } + return chunk; } /* setEncoding(enc): flips the readable buffer into string mode — Node @@ -5090,7 +5145,7 @@ static void scr_stream_run_tick(ScrStreamTick *t) { if (st->r.next_waiter && st->r.length == 0 && (st->r.ended || st->destroyed)) { ScrPromise *w = st->r.next_waiter; st->r.next_waiter = NULL; - scr_stream_next_fulfill(st, w, NULL); + scr_stream_next_fulfill(st, w, NULL, false); } else if (st->r.next_waiter) { scr_stream_settle_next(s); /* content raced in: deliver it */ } diff --git a/tests/corpus/stream-dynamic-entry-kinds.cjs b/tests/corpus/stream-dynamic-entry-kinds.cjs new file mode 100644 index 000000000..82d70b16a --- /dev/null +++ b/tests/corpus/stream-dynamic-entry-kinds.cjs @@ -0,0 +1,28 @@ +// @no-node-shims +// Readable.from over parsed JSON strings: the first lazy pull makes the +// stream hold strings, which reach 'data', for await, an operator and a +// pipe as strings; a mixed source decodes once setEncoding() is set. +'use strict'; +const { Readable, Writable } = require('node:stream'); +async function main() { + const fromJson = () => Readable.from(JSON.parse('["j1", "j2", "j3"]')); + const data = []; + const r = fromJson(); + r.on('data', (c) => data.push(typeof c + ':' + c)); + await new Promise((resolve) => r.on('end', resolve)); + console.log('data', data.join(' ')); + for await (const c of fromJson()) console.log('for await', typeof c, c); + console.log('toArray', await fromJson().map((c) => c.toUpperCase()).toArray()); + const hex = []; + const sink = new Writable({ write(chunk, enc, cb) { hex.push(chunk.toString('hex')); cb(); } }); + fromJson().pipe(sink); + await new Promise((resolve) => sink.on('finish', resolve)); + console.log('piped', hex.join(',')); + const mixed = Readable.from(['s1', Buffer.from('b1'), 's2']); + mixed.setEncoding('utf8'); + for await (const c of mixed) console.log('mixed', typeof c, c); + const switched = fromJson(); + switched.setEncoding('latin1'); + console.log('switched', await switched.toArray()); +} +main(); diff --git a/tests/corpus/stream-dynamic-entry-kinds.ts b/tests/corpus/stream-dynamic-entry-kinds.ts new file mode 100644 index 000000000..a891df8a2 --- /dev/null +++ b/tests/corpus/stream-dynamic-entry-kinds.ts @@ -0,0 +1,18 @@ +// A Readable.from stream over parsed JSON strings read through an any +// receiver: read() answers the strings, as in Node, and setEncoding() +// before the first read keeps a Buffer and string source as strings. +import { Readable } from "node:stream"; + +const source: any = JSON.parse('["t1", "t2"]'); +const r: any = Readable.from(source); +r.once("readable", () => { + let c: any; + while ((c = r.read()) !== null) console.log("read", typeof c, c, Buffer.isBuffer(c)); +}); +r.on("end", () => console.log("end")); + +const values: any = JSON.parse('["s1"]'); +values.unshift(Buffer.from("b1")); +const mixed = Readable.from(values); +mixed.setEncoding("utf8"); +mixed.on("data", (c: string) => console.log("mixed", typeof c, c)); diff --git a/tests/harness/stream-entry-kinds.test.ts b/tests/harness/stream-entry-kinds.test.ts new file mode 100644 index 000000000..f7333b298 --- /dev/null +++ b/tests/harness/stream-entry-kinds.test.ts @@ -0,0 +1,158 @@ +/* Stream chunk kinds Node cannot oracle — scriptc-only. + * + * A typed read() answers `Buffer | null`. When the read itself makes the + * stream hold strings (the first string a lazy Readable.from pull over a + * dynamic array produces, or setEncoding() inside _read), Node returns + * the string, which the typed result cannot hold, so the read throws the + * encoding fence instead (caught here, so the program exits normally). + * These pins keep that refusal exact and, under + * SCRIPTC_SAN=1 (ASan and the RC audit, like every harness lane), prove + * no string reaches Buffer code. The Node-observable cases are corpus + * programs (stream-dynamic-entry-kinds.*). + */ +import { execFile } from "node:child_process"; +import { createHash } from "node:crypto"; +import { mkdirSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { promisify } from "node:util"; +import { describe, expect, test } from "vitest"; +import { compile } from "@scriptc/compiler"; + +const execFileAsync = promisify(execFile); +const repoRoot = join(import.meta.dirname, "../.."); +const cacheDir = join(repoRoot, "node_modules/.cache/scriptc-tests"); +const sanitize = process.env["SCRIPTC_SAN"] === "1"; + +interface RunResult { + stdout: string; + stderr: string; + exitCode: number; +} + +async function compileAndRun(name: string, source: string, ext = "ts"): Promise { + const key = createHash("sha256") + .update(source) + .update(sanitize ? "san" : "plain") + .digest("hex") + .slice(0, 16); + const outDir = join(cacheDir, `stream-kinds-${key}`); + mkdirSync(outDir, { recursive: true }); + const file = join(outDir, `${name}.${ext}`); + writeFileSync(file, source); + const result = await compile(file, { + outPath: join(outDir, name), + outDir, + sanitize, + backend: "llvm", + }); + if (!result.ok) { + throw new Error( + "stream program failed to compile:\n" + + result.diagnostics.map((d) => `${d.code}: ${d.message}`).join("\n"), + ); + } + try { + const { stdout, stderr } = await execFileAsync(result.binaryPath, [], { encoding: "utf8" }); + return { stdout, stderr: stripAsanFiberWarning(stderr), exitCode: 0 }; + } catch (err) { + const e = err as { code?: unknown; stdout?: string; stderr?: string }; + if (typeof e.code !== "number") throw err; + return { + stdout: e.stdout ?? "", + stderr: stripAsanFiberWarning(e.stderr ?? ""), + exitCode: e.code, + }; + } +} + +/** Linux ASan's once-per-process swapcontext warning (see errors.test.ts). */ +function stripAsanFiberWarning(s: string): string { + return s.replace( + /^==\d+==WARNING: ASan doesn't fully support makecontext\/swapcontext.*\n/gm, + "", + ); +} + +const READ_FENCE_MESSAGE = + "read() on a stream with an encoding set is not supported yet (consume 'data' events, which deliver strings)"; + +describe(`typed stream reads of string chunks (scriptc-only${sanitize ? ", sanitized" : ""})`, () => { + test("a typed read() whose lazy pull yields the first string throws the encoding fence", async () => { + const r = await compileAndRun( + "typed-read-json-string", + `import { Readable } from "node:stream"; +const r = Readable.from(JSON.parse('["j1", "j2"]')); +try { + const chunk = r.read(); + console.log("hex", chunk === null ? "null" : chunk.toString("hex")); +} catch (err) { + console.log("caught", (err as Error).message); +} +`, + ); + expect(r.exitCode).toBe(0); + expect(r.stdout).toBe(`caught ${READ_FENCE_MESSAGE}\n`); + }); + + test("a typed read() whose _read calls setEncoding() throws the encoding fence", async () => { + const r = await compileAndRun( + "typed-read-encoding-in-read", + `import { Readable } from "node:stream"; +const r = new Readable({ + read() { + this.setEncoding("utf8"); + this.push(Buffer.from("ab")); + this.push(null); + }, +}); +try { + const chunk = r.read(); + console.log("hex", chunk === null ? "null" : chunk.toString("hex")); +} catch (err) { + console.log("caught", (err as Error).message); +} +`, + ); + expect(r.exitCode).toBe(0); + expect(r.stdout).toBe(`caught ${READ_FENCE_MESSAGE}\n`); + }); + + test("the JavaScript lane's read() on a JSON source throws the same fence", async () => { + const r = await compileAndRun( + "js-read-json-string", + `'use strict'; +const { Readable } = require('node:stream'); +const r = Readable.from(JSON.parse('["j1"]')); +r.once('readable', () => { + try { + const chunk = r.read(); + console.log('got', chunk === null ? 'null' : chunk.length); + } catch (err) { + console.log('caught', err.message); + } +}); +`, + "cjs", + ); + expect(r.exitCode).toBe(0); + expect(r.stdout).toBe(`caught ${READ_FENCE_MESSAGE}\n`); + }); + + test("a dynamic source that mixes Buffers and strings without a decoder is destroyed", async () => { + const r = await compileAndRun( + "mixed-dynamic-source", + `import { Readable } from "node:stream"; +const values: any = JSON.parse('["s1"]'); +values.unshift(Buffer.from("b1")); +const r = Readable.from(values); +r.on("data", (c: any) => console.log("data", Buffer.isBuffer(c), String(c))); +r.on("error", (err: Error) => console.log("error", err.message)); +r.on("close", () => console.log("close")); +`, + ); + expect(r.exitCode).toBe(0); + expect(r.stdout).toBe( + "data true b1\nerror a Readable.from source that mixes strings and Buffers is not supported yet\nclose\n", + ); + }); +}); From da27496ea90b1be71bf3d7497b2a3b3b5d5bb815 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 15 Oct 2026 17:47:34 -0500 Subject: [PATCH 6/7] Probe hashed number-key Map/Set lookups inline Number-key reads of a map without the direct index called scr_map_entry_f64 for every lookup. The internal sc_map_entry_f64 helper now calls sc_map_entry_f64_hashed for such maps instead: it hashes the key as scr_map_find_f64 does (SameValueZero key word, scr_key_mix, the absent-key filter), scans the entries of a small map or probes a compact bucket table, and calls the runtime lookup only for a table wider than the compact limit. It stays out of line so the direct-index path stays small at every call site. The runtime fallback is called as a read-only call, so both helpers only read memory and LLVM merges lookups of one key when nothing is stored in between, such as a has() followed by get(). The runtime benchmark suite gains number-maps: a node table indexed by dense ids, scope tables keyed by sparse ids, and get/has/set mixes. --- benchmarks/runtime/README.md | 1 + benchmarks/runtime/workloads.json | 7 + benchmarks/runtime/workloads/number-maps.ts | 90 +++++++++++++ packages/compiler/src/backend/llvm/emitter.ts | 123 +++++++++++++++++- .../backend/llvm/map-number-lookup.test.ts | 65 +++++++++ .../src/backend/llvm/map-number-lookup.ts | 6 +- .../5710-map-number-keys-hashed.ts.json | 6 + packages/runtime/src/scr_map.c | 6 +- packages/runtime/src/scr_map.h | 6 +- tests/corpus/5710-map-number-keys-hashed.ts | 104 +++++++++++++++ tests/harness/wasi-differential.test.ts | 2 + 11 files changed, 408 insertions(+), 8 deletions(-) create mode 100644 benchmarks/runtime/workloads/number-maps.ts create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5710-map-number-keys-hashed.ts.json create mode 100644 tests/corpus/5710-map-number-keys-hashed.ts diff --git a/benchmarks/runtime/README.md b/benchmarks/runtime/README.md index 23fa17905..1eb12ab40 100644 --- a/benchmarks/runtime/README.md +++ b/benchmarks/runtime/README.md @@ -18,6 +18,7 @@ Application-shaped workloads for measuring the speed of compiled executables. Ea | `value-objects` | cpu | small immutable records and class instances: vector helpers, method chains, particle updates | | `optional-fields` | cpu | records and class instances with `number \| undefined` and other scalar-union fields | | `record-boundaries` | cpu | records built from unchecked array reads passed to and returned from annotated functions | +| `number-maps` | cpu | number-keyed Map/Set: a dense-id table, sparse ids in many small maps, get/has/set mixes | | `validate-errors` | cpu | throw/catch on a fraction of inputs, small call chains | | `csv-numbers` | cpu | `split`, `Number`, `parseInt`, `toFixed` | | `log-lines` | cpu | 200k immediate `console.log`/`process.stdout.write` lines into the runner's pipe | diff --git a/benchmarks/runtime/workloads.json b/benchmarks/runtime/workloads.json index bb6313cf7..3dd9f1da1 100644 --- a/benchmarks/runtime/workloads.json +++ b/benchmarks/runtime/workloads.json @@ -105,6 +105,13 @@ "8" ] }, + { + "name": "number-maps", + "entry": "workloads/number-maps.ts", + "args": [ + "4" + ] + }, { "name": "validate-errors", "entry": "workloads/validate-errors.ts", diff --git a/benchmarks/runtime/workloads/number-maps.ts b/benchmarks/runtime/workloads/number-maps.ts new file mode 100644 index 000000000..3fae9943f --- /dev/null +++ b/benchmarks/runtime/workloads/number-maps.ts @@ -0,0 +1,90 @@ +// Number-keyed Map and Set work: a node table indexed by dense ids, scope +// tables keyed by sparse ids from one global counter, and caches that mix +// get, has and set on the same keys. +interface Item { + id: number; + parent: number; + flags: number; +} + +// Dense ids: items 0..n-1, walked through their parent chains. +function denseIds(n: number, rounds: number): number { + const items = new Map(); + for (let id = 0; id < n; id++) items.set(id, { id, parent: id === 0 ? -1 : (id * 7) >> 3, flags: id & 15 }); + let total = 0; + for (let r = 0; r < rounds; r++) { + for (let id = r & 7; id < n; id += 8) { + let item = items.get(id); + let depth = 0; + while (item !== undefined && depth < 4) { + total += item.flags; + item = items.get(item.parent); + depth++; + } + } + } + return total; +} + +// Sparse ids: one counter hands out ids to many scopes, so each scope's +// keys are spread far apart. Lookups test membership, then read. +function sparseIds(scopes: number, perScope: number, rounds: number): number { + const tables: Map[] = []; + const probes: number[] = []; + let next = 1000; + for (let s = 0; s < scopes; s++) tables.push(new Map()); + for (let i = 0; i < perScope; i++) { + for (let s = 0; s < scopes; s++) { + const id = next; + next += 1 + ((id * 31) & 3); + tables[s]!.set(id, { id, parent: s, flags: id & 7 }); + if ((i & 1) === 0) probes.push(id); + else probes.push(id + 1); + } + } + let total = 0; + for (let r = 0; r < rounds; r++) { + for (let p = 0; p < probes.length; p++) { + const id = probes[p]!; + const table = tables[p % scopes]!; + if (table.has(id)) total += table.get(id)!.flags; + else total++; + } + } + return total; +} + +// Mixed get/set: counting relation keys, a memo cache filled on misses, +// and a visited set. +function mixed(ids: number[], rounds: number): number { + const counts = new Map(); + const memo = new Map(); + let total = 0; + for (let r = 0; r < rounds; r++) { + const visited = new Set(); + for (let i = 0; i < ids.length; i++) { + const id = ids[i]!; + const relation = id * 4 + (r & 3); + counts.set(relation, (counts.get(relation) ?? 0) + 1); + let cached = memo.get(id); + if (cached === undefined) { + cached = (id * 2654435761) % 1000; + memo.set(id, cached); + } + total += cached; + if (!visited.has(id)) { + visited.add(id); + total++; + } + } + } + return total + counts.size + memo.size; +} + +const scale = Number(process.argv[2] ?? "1"); +const ids: number[] = []; +for (let i = 0; i < 20000; i++) ids.push(5000 + ((i * 7919) % 12000) * 3); +const dense = denseIds(50000, Math.floor(40 * scale)); +const sparse = sparseIds(64, 40, Math.floor(400 * scale)); +const mix = mixed(ids, Math.floor(30 * scale)); +console.log("dense", dense, "sparse", sparse, "mixed", mix); diff --git a/packages/compiler/src/backend/llvm/emitter.ts b/packages/compiler/src/backend/llvm/emitter.ts index 5bfe250fd..423ce450e 100644 --- a/packages/compiler/src/backend/llvm/emitter.ts +++ b/packages/compiler/src/backend/llvm/emitter.ts @@ -2940,9 +2940,13 @@ export class LlEmitter { // Number-key Map/Set lookups probe the runtime's direct index inline: // while ndense is nonzero every live key is an integer below it, so an // in-range integral key reads its slot (entry index + 1, 0 = absent) and - // any other key is absent. Maps without the index (ndense 0) and every - // non-integral probe outside the table take the runtime lookup, which - // keeps the complete SameValueZero semantics. + // any other key is absent. Maps without the index (ndense 0) hash the + // key in sc_map_entry_f64_hashed, which scans the entries of a small map + // or probes a compact bucket table as scr_map_find_f64 does, and calls + // the runtime lookup for a wide table. It stays out of line so that the + // direct-index path stays small at every call site. Both only read + // memory, so LLVM merges lookups of one key when nothing is stored in + // between (a has() followed by get(), for example). if (this.decls.has(NUMBER_MAP_ENTRY_DECL)) { const sz = this.sizeType; let mapTmp = 0; @@ -2978,7 +2982,118 @@ export class LlEmitter { ` %dense.any = icmp ne ${sz} %nd, 0`, ` br i1 %dense.any, label %miss, label %slow`, `slow:`, - ` %r = call ptr @scr_map_entry_f64(ptr %m, double %k)`, + ` %hashed = call ptr @sc_map_entry_f64_hashed(ptr %m, double %k)`, + ` ret ptr %hashed`, + `}`, + ``, + // Hashing (scr_map_find_f64): the SameValueZero key word, its + // scr_key_mix hash (zero remapped to one) and the absent-key filter. + `define internal ptr @sc_map_entry_f64_hashed(ptr %m, double %k) noinline ${FN_ATTRS} {`, + `entry:`, + ` %nan = fcmp uno double %k, %k`, + ` %zero = fcmp oeq double %k, 0.0`, + ` %kz = select i1 %zero, double 0.0, double %k`, + ` %kzb = bitcast double %kz to i64`, + ` %kb = select i1 %nan, i64 9221120237041090560, i64 %kzb`, + ` %x1s = lshr i64 %kb, 30`, + ` %x1 = xor i64 %kb, %x1s`, + ` %x2 = mul i64 %x1, -4658895280553007687`, + ` %x3s = lshr i64 %x2, 27`, + ` %x3 = xor i64 %x2, %x3s`, + ` %x4 = mul i64 %x3, -7723592293110705685`, + ` %x5s = lshr i64 %x4, 31`, + ` %x5 = xor i64 %x4, %x5s`, + ` %hz = icmp eq i64 %x5, 0`, + ` %h = select i1 %hz, i64 1, i64 %x5`, + ` %f1s = lshr i64 %h, 44`, + ` %f1a = and i64 %f1s, 63`, + ` %f1 = shl i64 1, %f1a`, + ` %f2s = lshr i64 %h, 50`, + ` %f2a = and i64 %f2s, 63`, + ` %f2 = shl i64 1, %f2a`, + ` %fb = or i64 %f1, %f2`, + ` %fp = getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 11`, + ` %filter = load i64, ptr %fp`, + ` %fm = and i64 %filter, %fb`, + ` %fin = icmp eq i64 %fm, %fb`, + ` br i1 %fin, label %table, label %miss`, + `table:`, + ` %nbp = getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 14`, + ` %nb = load ${sz}, ptr %nbp`, + ` %eps = getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 13`, + ` %ents = load ptr, ptr %eps`, + ` %linear = icmp eq ${sz} %nb, 0`, + ` br i1 %linear, label %scan, label %buckets`, + // Small maps keep no bucket table: scan the cached entry hashes. + `scan:`, + ` %nep = getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 9`, + ` %ne = load ${sz}, ptr %nep`, + ` br label %scan.loop`, + `scan.loop:`, + ` %si = phi ${sz} [ 0, %scan ], [ %si.next, %scan.next ]`, + ` %sdone = icmp eq ${sz} %si, %ne`, + ` br i1 %sdone, label %miss, label %scan.entry`, + `scan.entry:`, + ` %se = getelementptr inbounds %ScrMapEntry, ptr %ents, ${sz} %si`, + ` %seh = getelementptr inbounds %ScrMapEntry, ptr %se, i32 0, i32 2`, + ` %sh = load i64, ptr %seh`, + ` %sk = load i64, ptr %se`, + ` %shq = icmp eq i64 %sh, %h`, + ` %skq = icmp eq i64 %sk, %kb`, + ` %sq = and i1 %shq, %skq`, + ` br i1 %sq, label %scan.found, label %scan.next`, + `scan.found:`, + ` ret ptr %se`, + `scan.next:`, + ` %si.next = add ${sz} %si, 1`, + ` br label %scan.loop`, + // Compact tables tag each 24-bit entry index (+1) with the hash's + // top byte; wider tables take the runtime lookup. + `buckets:`, + ` %wide = icmp ugt ${sz} %nb, 16777216`, + ` br i1 %wide, label %runtime, label %probe`, + `probe:`, + ` %bp = getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 15`, + ` %bk = load ptr, ptr %bp`, + ` %mask = sub ${sz} %nb, 1`, + ` %ht = lshr i64 %h, 32`, + ` %ht32 = trunc i64 %ht to i32`, + ` %tag = and i32 %ht32, -16777216`, + sz === "i64" ? ` %hi = add i64 %h, 0` : ` %hi = trunc i64 %h to ${sz}`, + ` %i0 = and ${sz} %hi, %mask`, + ` br label %probe.loop`, + `probe.loop:`, + ` %pi = phi ${sz} [ %i0, %probe ], [ %pi.next, %probe.next ]`, + ` %bep = getelementptr inbounds i32, ptr %bk, ${sz} %pi`, + ` %b = load i32, ptr %bep`, + ` %bempty = icmp eq i32 %b, 0`, + ` br i1 %bempty, label %miss, label %probe.tag`, + `probe.tag:`, + ` %bt = and i32 %b, -16777216`, + ` %btq = icmp eq i32 %bt, %tag`, + ` br i1 %btq, label %probe.entry, label %probe.next`, + `probe.entry:`, + ` %bix = and i32 %b, 16777215`, + ` %bix1 = sub i32 %bix, 1`, + sz === "i32" ? ` %bi = add i32 %bix1, 0` : ` %bi = zext i32 %bix1 to ${sz}`, + ` %pe = getelementptr inbounds %ScrMapEntry, ptr %ents, ${sz} %bi`, + ` %peh = getelementptr inbounds %ScrMapEntry, ptr %pe, i32 0, i32 2`, + ` %ph = load i64, ptr %peh`, + ` %pk = load i64, ptr %pe`, + ` %phq = icmp eq i64 %ph, %h`, + ` %pkq = icmp eq i64 %pk, %kb`, + ` %pq = and i1 %phq, %pkq`, + ` br i1 %pq, label %probe.found, label %probe.next`, + `probe.found:`, + ` ret ptr %pe`, + `probe.next:`, + ` %pi.inc = add ${sz} %pi, 1`, + ` %pi.next = and ${sz} %pi.inc, %mask`, + ` br label %probe.loop`, + `miss:`, + ` ret ptr null`, + `runtime:`, + ` %r = call ptr @scr_map_entry_f64(ptr %m, double %k) memory(read) nounwind willreturn`, ` ret ptr %r`, `}`, ``, diff --git a/packages/compiler/src/backend/llvm/map-number-lookup.test.ts b/packages/compiler/src/backend/llvm/map-number-lookup.test.ts index dbaf66fed..3c30c7239 100644 --- a/packages/compiler/src/backend/llvm/map-number-lookup.test.ts +++ b/packages/compiler/src/backend/llvm/map-number-lookup.test.ts @@ -117,3 +117,68 @@ test("string keys and reference values are retained inline, boolean slots compar expect(text).toContain("@scr_str_retain"); expect(text).toContain("call ptr @scr_map_iter_val_ref("); }); + +/** work(map, k): map.has(k) on a number-keyed map. */ +function lookupFixture(): IrModule { + const map: IrType = { kind: "map", key: F64, value: F64 }; + const params = [ + { localId: "map", name: "map", type: map }, + { localId: "k", name: "k", type: F64 }, + ]; + const has: IrExpr = { + kind: "mapIntrinsic", + method: "has", + receiver: ref("map", map), + args: [ref("k", F64)], + type: BOOL, + loc, + }; + const work: IrFunction = { + name: "work", + params, + returnType: BOOL, + loc, + locals: params.map((p) => ({ id: p.localId, name: p.name, type: p.type, mutable: true })), + body: [{ kind: "return", value: has, loc }], + }; + return { + irVersion: 16, + sourceFile: loc.file, + entry: "main", + records: [], + unions: [], + functions: [{ name: "main", params: [], locals: [], returnType: VOID, body: [], loc }, work], + }; +} + +test("hashed number-key lookups probe inline and call the runtime only for wide tables", () => { + for (const bits of [32, 64] as const) { + const sz = bits === 64 ? "i64" : "i32"; + const { work, module } = emitted(lookupFixture(), bits); + expect(work).toContain("call ptr @sc_map_entry_f64("); + const helper = /^define internal ptr @sc_map_entry_f64\(ptr %m, double %k\)[^]*?^}/m.exec( + module, + )![0]; + expect(helper).toContain("call ptr @sc_map_entry_f64_hashed(ptr %m, double %k)"); + const hashed = + /^define internal ptr @sc_map_entry_f64_hashed\(ptr %m, double %k\) noinline [^]*?^}/m.exec( + module, + )![0]; + // The SameValueZero key word (canonical NaN, +0 for -0) and the + // scr_key_mix multipliers 0xbf58476d1ce4e5b9 and 0x94d049bb133111eb. + expect(hashed).toContain("select i1 %nan, i64 9221120237041090560, i64 %kzb"); + expect(hashed).toContain("mul i64 %x1, -4658895280553007687"); + expect(hashed).toContain("mul i64 %x3, -7723592293110705685"); + // Filter (field 11), small-map scan of the entry count (field 9), and + // compact buckets (fields 14 and 15) tagged with the hash's top byte. + expect(hashed).toContain("getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 11"); + expect(hashed).toContain(`%ne = load ${sz}, ptr %nep`); + expect(hashed).toContain(`%nb = load ${sz}, ptr %nbp`); + expect(hashed).toContain("getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 15"); + expect(hashed).toContain("and i32 %ht32, -16777216"); + expect(hashed).toContain(`icmp ugt ${sz} %nb, 16777216`); + expect(hashed).toContain( + "call ptr @scr_map_entry_f64(ptr %m, double %k) memory(read) nounwind willreturn", + ); + } +}); diff --git a/packages/compiler/src/backend/llvm/map-number-lookup.ts b/packages/compiler/src/backend/llvm/map-number-lookup.ts index abee9c007..fdbc4ba0d 100644 --- a/packages/compiler/src/backend/llvm/map-number-lookup.ts +++ b/packages/compiler/src/backend/llvm/map-number-lookup.ts @@ -3,8 +3,10 @@ * Number-keyed collections whose keys are small non-negative integers keep a * direct index in the runtime (scr_map.c). Reads go through the internal * sc_map_entry_f64 helper (emitter.ts helperDefs), which LLVM inlines: it - * resolves directly indexed maps without a call and defers to - * scr_map_entry_f64 otherwise. The generated code then reads the entry's + * resolves directly indexed maps without a call and other maps through + * the internal sc_map_entry_f64_hashed probe, which hashes the key as the + * runtime does and defers to scr_map_entry_f64 only for wide bucket + * tables. Both only read memory. The generated code then reads the entry's * value slot itself and retains a reference value with the value type's own * retain, which is the adapter the collection was constructed with. * String-keyed reads use the same entry contract through the internal diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5710-map-number-keys-hashed.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5710-map-number-keys-hashed.ts.json new file mode 100644 index 000000000..56107e4cc --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5710-map-number-keys-hashed.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/5710-map-number-keys-hashed.ts" + ], + "diags": [] +} diff --git a/packages/runtime/src/scr_map.c b/packages/runtime/src/scr_map.c index 463024a28..92c006416 100644 --- a/packages/runtime/src/scr_map.c +++ b/packages/runtime/src/scr_map.c @@ -258,7 +258,11 @@ static inline bool scr_map_lookup_eq(const ScrMap *m, uint64_t stored, uint64_t /* Compact tables keep a hash fingerprint beside each 24-bit entry index. * Rejecting a collision here avoids loading an unrelated ordered entry. * Larger tables keep full-width indices; collection size remains limited - * only by the allocator. Zero is empty in compact tables, so store index+1. */ + * only by the allocator. Zero is empty in compact tables, so store index+1. + * The LLVM backend's inline number-key probe (sc_map_entry_f64_hashed) + * reads this encoding for tables of at most 1 << 24 buckets, hashes keys + * as scr_map_find_f64 does, and calls scr_map_entry_f64 for wider tables; + * only the runtime unit test builds a smaller bound. */ #ifndef SCR_MAP_COMPACT_BUCKETS #define SCR_MAP_COMPACT_BUCKETS ((size_t)1 << 24) #endif diff --git a/packages/runtime/src/scr_map.h b/packages/runtime/src/scr_map.h index cc0322609..b987b4990 100644 --- a/packages/runtime/src/scr_map.h +++ b/packages/runtime/src/scr_map.h @@ -119,7 +119,8 @@ _Static_assert(offsetof(ScrMap, key_trace) == offsetof(ScrMap, val_trace) + 3 * /* The LLVM backend's inline number-key lookup (sc_map_entry_f64) reads * entries, dense and ndense through %ScrMapIx fields 13, 19 and 20, and * entries as %ScrMapEntry { key, val, hash }; %ScrMapIx field 11 is the - * i64 filter. */ + * i64 filter. Its hashed probe (sc_map_entry_f64_hashed) also reads the + * entry count, nbuckets and buckets as fields 9, 14 and 15. */ _Static_assert(offsetof(ScrMap, filter) == (offsetof(ScrMap, nlive) + sizeof(size_t) + _Alignof(uint64_t) - 1) / _Alignof(uint64_t) * _Alignof(uint64_t), "%ScrMapIx field 11 is ScrMap.filter, directly after nlive"); @@ -132,6 +133,9 @@ _Static_assert(offsetof(ScrMap, nentries) + sizeof(size_t) == offsetof(ScrMap, n _Static_assert(offsetof(ScrMapEntry, hash) == 16, "inline iteration reads ScrMapEntry.hash at byte 16"); _Static_assert(offsetof(ScrMap, entries) == offsetof(ScrMap, filter) + sizeof(uint64_t) + sizeof(size_t), "inline number-key lookups read ScrMap.entries as %ScrMapIx field 13"); +_Static_assert(offsetof(ScrMap, nbuckets) == offsetof(ScrMap, entries) + sizeof(void *) && + offsetof(ScrMap, buckets) == offsetof(ScrMap, nbuckets) + sizeof(size_t), + "inline number-key probes read ScrMap.nbuckets and buckets as %ScrMapIx fields 14 and 15"); _Static_assert(offsetof(ScrMap, dense) == offsetof(ScrMap, entries) + 6 * sizeof(void *), "inline number-key lookups read ScrMap.dense as %ScrMapIx field 19"); _Static_assert(offsetof(ScrMap, ndense) == offsetof(ScrMap, dense) + sizeof(void *), diff --git a/tests/corpus/5710-map-number-keys-hashed.ts b/tests/corpus/5710-map-number-keys-hashed.ts new file mode 100644 index 000000000..df67ef240 --- /dev/null +++ b/tests/corpus/5710-map-number-keys-hashed.ts @@ -0,0 +1,104 @@ +// Number-keyed Map and Set lookups on hashed storage: small maps that +// scan their entries, bucket tables, keys that need SameValueZero +// normalization, deletes and clears between reads, and maps that move +// between the direct index and hashing. + +const special = [0, -0, NaN, Infinity, -Infinity, 0.5, -1, 2 ** 53, 1e300, Number.MIN_VALUE, Number.MAX_VALUE]; + +function describe(m: Map, keys: number[]): string { + return keys.map((k) => `${Object.is(k, -0) ? "-0" : k}:${m.has(k) ? m.get(k) : "-"}`).join(" "); +} + +// Small maps keep no bucket table. +const small = new Map(); +small.set(1e300, "big"); +small.set(-0, "zero"); +small.set(NaN, "nan"); +console.log("small", small.size, describe(small, special)); +console.log("small keys", [...small.keys()].map((k) => (Object.is(k, -0) ? "-0" : String(k))).join(",")); +small.delete(0); +console.log("small after delete", small.size, describe(small, special)); +small.set(0, "again"); +console.log("small re-add", describe(small, [0, -0, NaN, 1e300])); + +// Bucket tables of sparse ids, with hits, misses and tombstones. +const table = new Map(); +for (let i = 0; i < 3000; i++) table.set(100000 + i * 7919, i); +let hits = 0, + misses = 0, + sum = 0; +for (let i = 0; i < 6000; i++) { + const k = 100000 + i * 7919; + const v = table.get(k); + if (v === undefined) misses++; + else { + hits++; + sum += v; + } + if (table.has(k + 1)) hits += 1000000; +} +console.log("table", table.size, hits, misses, sum); +for (let i = 0; i < 3000; i += 3) table.delete(100000 + i * 7919); +let live = 0; +for (let i = 0; i < 3000; i++) if (table.has(100000 + i * 7919)) live++; +console.log("table after deletes", table.size, live, table.get(100000), table.get(100000 + 7919)); +for (const k of special) table.set(k, -1); +console.log("table specials", table.size, special.map((k) => table.get(k)).join(","), table.get(0.25)); +table.clear(); +console.log("table cleared", table.size, table.get(100000 + 7919), table.has(NaN)); + +// A read, a write of the same key, then the read again. +const m = new Map(); +for (let i = 0; i < 64; i++) m.set(i * 1000003, i); +const key = 5 * 1000003; +const before = m.get(key); +m.delete(key); +const afterDelete = m.get(key); +const hadIt = m.has(key); +m.set(key, 99); +const afterSet = m.get(key); +for (let i = 64; i < 400; i++) m.set(i * 1000003, i); // grows the bucket table +const afterGrowth = m.get(key); +console.log("reread", before, afterDelete, hadIt, afterSet, afterGrowth, m.size); + +// Direct index to hashing and back. +const ids = new Map(); +for (let i = 0; i < 40; i++) ids.set(i, `id${i}`); +console.log("dense", ids.get(0), ids.get(39), ids.get(40), ids.has(-1), ids.get(-0)); +ids.set(-5, "neg"); +ids.set(1.5, "half"); +console.log("hashed", ids.size, ids.get(0), ids.get(39), ids.get(-5), ids.get(1.5), ids.get(2.5)); +ids.delete(-5); +ids.delete(1.5); +for (let i = 40; i < 200; i++) ids.set(i, `id${i}`); +console.log("regrown", ids.size, ids.get(0), ids.get(150), ids.get(-5), ids.get(1.5)); + +// Sets and boolean and reference values. +const seen = new Set(); +for (let i = 0; i < 500; i++) seen.add((i * 2654435761) % 4294967296); +let found = 0; +for (let i = 0; i < 1000; i++) if (seen.has((i * 2654435761) % 4294967296)) found++; +console.log("set", seen.size, found, seen.has(NaN), seen.has(-0)); +seen.add(-0); +seen.add(NaN); +console.log("set specials", seen.has(0), seen.has(NaN), seen.size); + +const flags = new Map(); +for (let i = 0; i < 100; i++) flags.set(i * 37 + 1e6, i % 3 === 0); +let truthy = 0; +for (let i = 0; i < 100; i++) if (flags.get(i * 37 + 1e6)) truthy++; +console.log("flags", truthy, flags.get(1e6), flags.get(1e6 + 37), flags.get(5)); + +interface Node { + id: number; + label: string; +} +const nodes = new Map(); +for (let i = 0; i < 300; i++) nodes.set(50000 + i * 13, { id: i, label: `n${i}` }); +let labels = ""; +for (let i = 0; i < 300; i += 37) { + const node = nodes.get(50000 + i * 13); + labels += node === undefined ? "?" : node.label; + if (nodes.has(50000 + i * 13 + 1)) labels += "!"; +} +console.log("nodes", nodes.size, labels, nodes.get(49999)?.label ?? "none"); diff --git a/tests/harness/wasi-differential.test.ts b/tests/harness/wasi-differential.test.ts index 7a898b19d..94b096f7e 100644 --- a/tests/harness/wasi-differential.test.ts +++ b/tests/harness/wasi-differential.test.ts @@ -269,6 +269,8 @@ describe.skipIf(!zigOnPath())("wasm32-wasi differential", () => { "5317-map-iterate-while-mutating.ts", "5318-set-iterate-after-deletes.ts", "5319-map-iteration-values.ts", + // Hashed number-key lookups probe ScrMap through size_t-width fields. + "5710-map-number-keys-hashed.ts", // JSON writers read array slots as pointers. "5305-json-stringify-typed-arrays.ts", // Comparator sorts move 64-bit slots of 32-bit pointers. From cdf0598e6e1d6a8258ac651d1106f98a298a014c Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 15 Oct 2026 17:51:48 -0500 Subject: [PATCH 7/7] Overwrite present number keys in the lookup that checks for growth When appending one more entry would grow a hashed number-key map's index, Map.set and Set.add first look the key up to decide whether the map should move to the direct index. A key that lookup finds was then looked up again by the generic set. The found entry now takes the new value directly, so overwriting a present key in a map at its growth boundary probes the table once. --- ...712-map-number-overwrite-at-growth.ts.json | 6 ++ packages/runtime/src/scr_map.c | 13 +++- .../5712-map-number-overwrite-at-growth.ts | 61 +++++++++++++++++++ 3 files changed, 78 insertions(+), 2 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5712-map-number-overwrite-at-growth.ts.json create mode 100644 tests/corpus/5712-map-number-overwrite-at-growth.ts diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5712-map-number-overwrite-at-growth.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5712-map-number-overwrite-at-growth.ts.json new file mode 100644 index 000000000..5f8dcf6b4 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5712-map-number-overwrite-at-growth.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/5712-map-number-overwrite-at-growth.ts" + ], + "diags": [] +} diff --git a/packages/runtime/src/scr_map.c b/packages/runtime/src/scr_map.c index 463024a28..7803494b0 100644 --- a/packages/runtime/src/scr_map.c +++ b/packages/runtime/src/scr_map.c @@ -999,10 +999,19 @@ static inline bool scr_map_append_grows_index(const ScrMap *m) { /* Number keys arrive normalized (+0 for -0, canonical NaN). A directly * indexed map overwrites or appends through its table; a key the table - * cannot hold converts it to hashing first. */ + * cannot hold converts it to hashing first. When an append would grow a + * hashed map's index, the lookup that decides whether the key is new also + * overwrites a key that is already present. */ static void scr_map_set_f64_slot(ScrMap *m, uint64_t key, uint64_t val) { double k = scr_map_slot_to_f64(key); - if (!m->ndense && scr_map_append_grows_index(m) && scr_map_find_f64(m, k) == SCR_MAP_EMPTY) { + if (!m->ndense && scr_map_append_grows_index(m)) { + size_t e = scr_map_find_f64(m, k); + if (e != SCR_MAP_EMPTY) { + uint64_t old = m->entries[e].val; + m->entries[e].val = val; /* unlink before releasing (cycle collector) */ + scr_map_release_val(m, old); + return; + } scr_map_enter_dense(m, k); /* the append would grow the index */ } if (m->ndense) { diff --git a/tests/corpus/5712-map-number-overwrite-at-growth.ts b/tests/corpus/5712-map-number-overwrite-at-growth.ts new file mode 100644 index 000000000..acbdb06e7 --- /dev/null +++ b/tests/corpus/5712-map-number-overwrite-at-growth.ts @@ -0,0 +1,61 @@ +// Number-key Map.set and Set.add of present keys while the map is at its +// bucket-table growth boundary (the next new key grows the index): +// overwrites keep the entry, its position and the size, and release the +// value they replace. + +interface Box { + label: string; +} + +function sparse(i: number): number { + return 1000 + i * 7919; +} + +for (const size of [8, 64, 1024]) { + const nums = new Map(); + const boxes = new Map(); + const names = new Map(); + const seen = new Set(); + for (let i = 0; i < size; i++) { + nums.set(sparse(i), i); + boxes.set(sparse(i), { label: `b${i}` }); + names.set(sparse(i), `n${i}`); + seen.add(sparse(i)); + } + for (let round = 0; round < 3; round++) { + for (let i = size - 1; i >= 0; i--) { + nums.set(sparse(i), i * 10 + round); + boxes.set(sparse(i), { label: `b${i}.${round}` }); + names.set(sparse(i), `n${i}.${round}`.repeat(2)); + seen.add(sparse(i)); + } + } + let sum = 0; + for (const v of nums.values()) sum += v; + const firstKeys = [...nums.keys()].slice(0, 3).join(","); + console.log( + size, + nums.size, + boxes.size, + names.size, + seen.size, + sum, + firstKeys, + boxes.get(sparse(size - 1))?.label, + names.get(sparse(0)), + [...seen].slice(0, 2).join(","), + ); + // One more key grows the index; existing entries are unchanged. + nums.set(-1, -1); + boxes.set(-1, { label: "new" }); + console.log(size, "grown", nums.size, nums.get(sparse(size >> 1)), nums.get(-1), boxes.get(-1)?.label, boxes.get(sparse(1))?.label); +} + +// At the boundary after deletes: a delete frees no bucket, so re-adding +// the deleted key compacts instead of growing. +const churn = new Map(); +for (let i = 0; i < 64; i++) churn.set(sparse(i), i); +churn.delete(sparse(10)); +churn.set(sparse(11), 111); +churn.set(sparse(10), 110); +console.log("churn", churn.size, churn.get(sparse(10)), churn.get(sparse(11)), [...churn.keys()].indexOf(sparse(10)));