diff --git a/benchmarks/runtime/README.md b/benchmarks/runtime/README.md index 23fa17905..1eb12ab40 100644 --- a/benchmarks/runtime/README.md +++ b/benchmarks/runtime/README.md @@ -18,6 +18,7 @@ Application-shaped workloads for measuring the speed of compiled executables. Ea | `value-objects` | cpu | small immutable records and class instances: vector helpers, method chains, particle updates | | `optional-fields` | cpu | records and class instances with `number \| undefined` and other scalar-union fields | | `record-boundaries` | cpu | records built from unchecked array reads passed to and returned from annotated functions | +| `number-maps` | cpu | number-keyed Map/Set: a dense-id table, sparse ids in many small maps, get/has/set mixes | | `validate-errors` | cpu | throw/catch on a fraction of inputs, small call chains | | `csv-numbers` | cpu | `split`, `Number`, `parseInt`, `toFixed` | | `log-lines` | cpu | 200k immediate `console.log`/`process.stdout.write` lines into the runner's pipe | diff --git a/benchmarks/runtime/workloads.json b/benchmarks/runtime/workloads.json index bb6313cf7..3dd9f1da1 100644 --- a/benchmarks/runtime/workloads.json +++ b/benchmarks/runtime/workloads.json @@ -105,6 +105,13 @@ "8" ] }, + { + "name": "number-maps", + "entry": "workloads/number-maps.ts", + "args": [ + "4" + ] + }, { "name": "validate-errors", "entry": "workloads/validate-errors.ts", diff --git a/benchmarks/runtime/workloads/number-maps.ts b/benchmarks/runtime/workloads/number-maps.ts new file mode 100644 index 000000000..3fae9943f --- /dev/null +++ b/benchmarks/runtime/workloads/number-maps.ts @@ -0,0 +1,90 @@ +// Number-keyed Map and Set work: a node table indexed by dense ids, scope +// tables keyed by sparse ids from one global counter, and caches that mix +// get, has and set on the same keys. +interface Item { + id: number; + parent: number; + flags: number; +} + +// Dense ids: items 0..n-1, walked through their parent chains. +function denseIds(n: number, rounds: number): number { + const items = new Map(); + for (let id = 0; id < n; id++) items.set(id, { id, parent: id === 0 ? -1 : (id * 7) >> 3, flags: id & 15 }); + let total = 0; + for (let r = 0; r < rounds; r++) { + for (let id = r & 7; id < n; id += 8) { + let item = items.get(id); + let depth = 0; + while (item !== undefined && depth < 4) { + total += item.flags; + item = items.get(item.parent); + depth++; + } + } + } + return total; +} + +// Sparse ids: one counter hands out ids to many scopes, so each scope's +// keys are spread far apart. Lookups test membership, then read. +function sparseIds(scopes: number, perScope: number, rounds: number): number { + const tables: Map[] = []; + const probes: number[] = []; + let next = 1000; + for (let s = 0; s < scopes; s++) tables.push(new Map()); + for (let i = 0; i < perScope; i++) { + for (let s = 0; s < scopes; s++) { + const id = next; + next += 1 + ((id * 31) & 3); + tables[s]!.set(id, { id, parent: s, flags: id & 7 }); + if ((i & 1) === 0) probes.push(id); + else probes.push(id + 1); + } + } + let total = 0; + for (let r = 0; r < rounds; r++) { + for (let p = 0; p < probes.length; p++) { + const id = probes[p]!; + const table = tables[p % scopes]!; + if (table.has(id)) total += table.get(id)!.flags; + else total++; + } + } + return total; +} + +// Mixed get/set: counting relation keys, a memo cache filled on misses, +// and a visited set. +function mixed(ids: number[], rounds: number): number { + const counts = new Map(); + const memo = new Map(); + let total = 0; + for (let r = 0; r < rounds; r++) { + const visited = new Set(); + for (let i = 0; i < ids.length; i++) { + const id = ids[i]!; + const relation = id * 4 + (r & 3); + counts.set(relation, (counts.get(relation) ?? 0) + 1); + let cached = memo.get(id); + if (cached === undefined) { + cached = (id * 2654435761) % 1000; + memo.set(id, cached); + } + total += cached; + if (!visited.has(id)) { + visited.add(id); + total++; + } + } + } + return total + counts.size + memo.size; +} + +const scale = Number(process.argv[2] ?? "1"); +const ids: number[] = []; +for (let i = 0; i < 20000; i++) ids.push(5000 + ((i * 7919) % 12000) * 3); +const dense = denseIds(50000, Math.floor(40 * scale)); +const sparse = sparseIds(64, 40, Math.floor(400 * scale)); +const mix = mixed(ids, Math.floor(30 * scale)); +console.log("dense", dense, "sparse", sparse, "mixed", mix); diff --git a/docs/content/limitations/builtin-apis/streams-and-readline.mdx b/docs/content/limitations/builtin-apis/streams-and-readline.mdx index 2c84dd378..b822da8d2 100644 --- a/docs/content/limitations/builtin-apis/streams-and-readline.mdx +++ b/docs/content/limitations/builtin-apis/streams-and-readline.mdx @@ -1 +1 @@ -- **Streams and readline:** Stream chunks are Buffers or strings. Object-mode streams of other values, `Readable.from` over arrays of other values, the `Readable` iterator helpers (`map`, `filter`, `toArray`, and the others), `stream.compose`, and generator stages in `pipeline` are unsupported. `pipeline` from `node:stream/promises` accepts a trailing `{ signal, end }` object, and `finished` accepts `readable`, `writable`, `error`, `cleanup`, and `signal` options, with boolean literals for the flags. Stream constructors accept a `signal` option in an options object literal, and `stream.addAbortSignal` accepts any stream. `node:readline` and `node:readline/promises` interfaces read `process.stdin` or a readable stream without terminal mode: `question`, `prompt`, `write`, `pause`, `resume`, `'line'`/`'close'`/`'pause'`/`'resume'`/`'error'` listeners, and `for await` compile. Terminal mode, `completer`, history options, and chained listener registration are unsupported. +- **Streams and readline:** Stream chunks are Buffers or strings. Object-mode streams of other values, `Readable.from` over arrays of other values or over generators and other iterables, and generator stages in `pipeline` are unsupported. `Readable.from` accepts `objectMode` and `highWaterMark` options. The `Readable` operators (`map`, `filter`, `forEach`, `toArray`, `some`, `find`, `every`, `flatMap`, `drop`, `take`, and `reduce`) accept `concurrency`, `highWaterMark`, and `signal` options; their callbacks take only the chunk, `map` returns strings or Buffers, and `flatMap` returns strings or arrays of them. `readable.compose` is unsupported. `stream.compose` accepts two or more streams, or one `Duplex`; functions, iterables, and web streams as stages are unsupported, and the `ERR_INVALID_ARG_VALUE` message for a stream that is no longer readable or writable names the stream's class instead of printing the stream. `pipeline` from `node:stream/promises` accepts a trailing `{ signal, end }` object, and `finished` accepts `readable`, `writable`, `error`, `cleanup`, and `signal` options, with boolean literals for the flags. Stream constructors accept a `signal` option in an options object literal, and `stream.addAbortSignal` accepts any stream. `node:readline` and `node:readline/promises` interfaces read `process.stdin` or a readable stream without terminal mode: `question`, `prompt`, `write`, `pause`, `resume`, `'line'`/`'close'`/`'pause'`/`'resume'`/`'error'` listeners, and `for await` compile. Terminal mode, `completer`, history options, and chained listener registration are unsupported. diff --git a/docs/src/generated/node-v24-compatibility-meta.json b/docs/src/generated/node-v24-compatibility-meta.json index 95f04b58c..676daf38a 100644 --- a/docs/src/generated/node-v24-compatibility-meta.json +++ b/docs/src/generated/node-v24-compatibility-meta.json @@ -2,6 +2,6 @@ "schemaVersion": 3, "nodeVersion": "24.22.0", "nodeCommit": "f14cd3aa7bf9ca8df2cc5b8bb14beb0f335b1d93", - "artifactVersion": "c7c956da1d50af685bf1", + "artifactVersion": "cce4b946b6f8d3072e83", "rowCount": 3880 } diff --git a/docs/src/generated/node-v24-compatibility.json b/docs/src/generated/node-v24-compatibility.json index 7e4f9f668..c2469341a 100644 --- a/docs/src/generated/node-v24-compatibility.json +++ b/docs/src/generated/node-v24-compatibility.json @@ -659,9 +659,9 @@ "apiEntries": 140, "static": { "supported": 0, - "partial": 38, + "partial": 53, "refused": 0, - "not-implemented": 102, + "not-implemented": 87, "by-design": 0, "unreviewed": 0, "not-applicable": 0 @@ -61525,9 +61525,9 @@ "inherited": true }, "static": { - "status": "not-implemented", - "detail": "No static compiler lowering is registered for this API yet.", - "verification": "registry-gap" + "status": "partial", + "detail": "Implemented for the documented static async-iteration subset.", + "verification": "test-backed" } }, { @@ -61640,9 +61640,9 @@ "inherited": true }, "static": { - "status": "not-implemented", - "detail": "No static compiler lowering is registered for this API yet.", - "verification": "registry-gap" + "status": "partial", + "detail": "Implemented for the documented static async-iteration subset.", + "verification": "test-backed" } }, { @@ -62031,9 +62031,9 @@ "inherited": true }, "static": { - "status": "not-implemented", - "detail": "No static compiler lowering is registered for this API yet.", - "verification": "registry-gap" + "status": "partial", + "detail": "Implemented for the documented static async-iteration subset.", + "verification": "test-backed" } }, { @@ -62146,9 +62146,9 @@ "inherited": true }, "static": { - "status": "not-implemented", - "detail": "No static compiler lowering is registered for this API yet.", - "verification": "registry-gap" + "status": "partial", + "detail": "Implemented for the documented static async-iteration subset.", + "verification": "test-backed" } }, { @@ -63388,9 +63388,9 @@ "inherited": false }, "static": { - "status": "not-implemented", - "detail": "No static compiler lowering is registered for this API yet.", - "verification": "registry-gap" + "status": "partial", + "detail": "Implemented for the documented static async-iteration subset.", + "verification": "test-backed" } }, { @@ -63411,9 +63411,9 @@ "inherited": false }, "static": { - "status": "not-implemented", - "detail": "No static compiler lowering is registered for this API yet.", - "verification": "registry-gap" + "status": "partial", + "detail": "Implemented for the documented static async-iteration subset.", + "verification": "test-backed" } }, { @@ -63434,9 +63434,9 @@ "inherited": false }, "static": { - "status": "not-implemented", - "detail": "No static compiler lowering is registered for this API yet.", - "verification": "registry-gap" + "status": "partial", + "detail": "Implemented for the documented static async-iteration subset.", + "verification": "test-backed" } }, { @@ -63457,9 +63457,9 @@ "inherited": false }, "static": { - "status": "not-implemented", - "detail": "No static compiler lowering is registered for this API yet.", - "verification": "registry-gap" + "status": "partial", + "detail": "Implemented for the documented static async-iteration subset.", + "verification": "test-backed" } }, { @@ -63480,9 +63480,9 @@ "inherited": false }, "static": { - "status": "not-implemented", - "detail": "No static compiler lowering is registered for this API yet.", - "verification": "registry-gap" + "status": "partial", + "detail": "Implemented for the documented static async-iteration subset.", + "verification": "test-backed" } }, { @@ -63503,9 +63503,9 @@ "inherited": false }, "static": { - "status": "not-implemented", - "detail": "No static compiler lowering is registered for this API yet.", - "verification": "registry-gap" + "status": "partial", + "detail": "Implemented for the documented static async-iteration subset.", + "verification": "test-backed" } }, { @@ -63526,9 +63526,9 @@ "inherited": false }, "static": { - "status": "not-implemented", - "detail": "No static compiler lowering is registered for this API yet.", - "verification": "registry-gap" + "status": "partial", + "detail": "Implemented for the documented static async-iteration subset.", + "verification": "test-backed" } }, { @@ -63549,9 +63549,9 @@ "inherited": false }, "static": { - "status": "not-implemented", - "detail": "No static compiler lowering is registered for this API yet.", - "verification": "registry-gap" + "status": "partial", + "detail": "Implemented for the documented static async-iteration subset.", + "verification": "test-backed" } }, { @@ -63572,9 +63572,9 @@ "inherited": false }, "static": { - "status": "not-implemented", - "detail": "No static compiler lowering is registered for this API yet.", - "verification": "registry-gap" + "status": "partial", + "detail": "Implemented for the documented static async-iteration subset.", + "verification": "test-backed" } }, { @@ -63595,9 +63595,9 @@ "inherited": false }, "static": { - "status": "not-implemented", - "detail": "No static compiler lowering is registered for this API yet.", - "verification": "registry-gap" + "status": "partial", + "detail": "Implemented for the documented static async-iteration subset.", + "verification": "test-backed" } }, { @@ -63618,9 +63618,9 @@ "inherited": false }, "static": { - "status": "not-implemented", - "detail": "No static compiler lowering is registered for this API yet.", - "verification": "registry-gap" + "status": "partial", + "detail": "Implemented for the documented static async-iteration subset.", + "verification": "test-backed" } }, { diff --git a/internal/compatibility/generated/node-v24-backlog.json b/internal/compatibility/generated/node-v24-backlog.json index 2ae89c3b4..638b91770 100644 --- a/internal/compatibility/generated/node-v24-backlog.json +++ b/internal/compatibility/generated/node-v24-backlog.json @@ -8,8 +8,8 @@ "tiers": { "static": { "replace-refusal": 45, - "verify-gap": 2362, - "audit-partial": 978, + "verify-gap": 2347, + "audit-partial": 993, "classify": 2, "implement": 85 } @@ -61820,12 +61820,15 @@ "priority": "high", "tiers": { "static": { - "status": "not-implemented", - "action": "verify-gap", - "verification": "registry-gap", - "confidence": "low", - "source": "compiler-unmatched:stream", - "tests": [] + "status": "partial", + "action": "audit-partial", + "verification": "test-backed", + "confidence": "high", + "source": "compiler-feature:stream.stream.compose", + "tests": [ + "tests/corpus/5342-stream-compose.ts", + "tests/corpus/5343-stream-compose-edges.ts" + ] } } }, @@ -61945,12 +61948,15 @@ "priority": "high", "tiers": { "static": { - "status": "not-implemented", - "action": "verify-gap", - "verification": "registry-gap", - "confidence": "low", - "source": "compiler-unmatched:stream", - "tests": [] + "status": "partial", + "action": "audit-partial", + "verification": "test-backed", + "confidence": "high", + "source": "compiler-feature:stream.stream.Readable.from", + "tests": [ + "tests/corpus/5340-readable-from-options.ts", + "tests/corpus/5341-readable-from-dynamic.cjs" + ] } } }, @@ -62409,12 +62415,15 @@ "priority": "high", "tiers": { "static": { - "status": "not-implemented", - "action": "verify-gap", - "verification": "registry-gap", - "confidence": "low", - "source": "compiler-unmatched:stream", - "tests": [] + "status": "partial", + "action": "audit-partial", + "verification": "test-backed", + "confidence": "high", + "source": "compiler-feature:stream.stream.compose", + "tests": [ + "tests/corpus/5342-stream-compose.ts", + "tests/corpus/5343-stream-compose-edges.ts" + ] } } }, @@ -62534,12 +62543,15 @@ "priority": "high", "tiers": { "static": { - "status": "not-implemented", - "action": "verify-gap", - "verification": "registry-gap", - "confidence": "low", - "source": "compiler-unmatched:stream", - "tests": [] + "status": "partial", + "action": "audit-partial", + "verification": "test-backed", + "confidence": "high", + "source": "compiler-feature:stream.stream.Readable.from", + "tests": [ + "tests/corpus/5340-readable-from-options.ts", + "tests/corpus/5341-readable-from-dynamic.cjs" + ] } } }, @@ -63990,12 +64002,17 @@ "priority": "low", "tiers": { "static": { - "status": "not-implemented", - "action": "verify-gap", - "verification": "registry-gap", - "confidence": "low", - "source": "compiler-unmatched:stream", - "tests": [] + "status": "partial", + "action": "audit-partial", + "verification": "test-backed", + "confidence": "high", + "source": "compiler-feature:stream.readable.map", + "tests": [ + "tests/corpus/5335-stream-operators-values.ts", + "tests/corpus/5336-stream-operators-promise-timing.ts", + "tests/corpus/5337-stream-operators-concurrency.ts", + "tests/corpus/5338-stream-operators-checks.cjs" + ] } } }, @@ -64015,12 +64032,17 @@ "priority": "low", "tiers": { "static": { - "status": "not-implemented", - "action": "verify-gap", - "verification": "registry-gap", - "confidence": "low", - "source": "compiler-unmatched:stream", - "tests": [] + "status": "partial", + "action": "audit-partial", + "verification": "test-backed", + "confidence": "high", + "source": "compiler-feature:stream.readable.filter", + "tests": [ + "tests/corpus/5335-stream-operators-values.ts", + "tests/corpus/5336-stream-operators-promise-timing.ts", + "tests/corpus/5337-stream-operators-concurrency.ts", + "tests/corpus/5338-stream-operators-checks.cjs" + ] } } }, @@ -64040,12 +64062,17 @@ "priority": "low", "tiers": { "static": { - "status": "not-implemented", - "action": "verify-gap", - "verification": "registry-gap", - "confidence": "low", - "source": "compiler-unmatched:stream", - "tests": [] + "status": "partial", + "action": "audit-partial", + "verification": "test-backed", + "confidence": "high", + "source": "compiler-feature:stream.readable.forEach", + "tests": [ + "tests/corpus/5335-stream-operators-values.ts", + "tests/corpus/5336-stream-operators-promise-timing.ts", + "tests/corpus/5337-stream-operators-concurrency.ts", + "tests/corpus/5338-stream-operators-checks.cjs" + ] } } }, @@ -64065,12 +64092,17 @@ "priority": "low", "tiers": { "static": { - "status": "not-implemented", - "action": "verify-gap", - "verification": "registry-gap", - "confidence": "low", - "source": "compiler-unmatched:stream", - "tests": [] + "status": "partial", + "action": "audit-partial", + "verification": "test-backed", + "confidence": "high", + "source": "compiler-feature:stream.readable.toArray", + "tests": [ + "tests/corpus/5335-stream-operators-values.ts", + "tests/corpus/5336-stream-operators-promise-timing.ts", + "tests/corpus/5337-stream-operators-concurrency.ts", + "tests/corpus/5338-stream-operators-checks.cjs" + ] } } }, @@ -64090,12 +64122,17 @@ "priority": "low", "tiers": { "static": { - "status": "not-implemented", - "action": "verify-gap", - "verification": "registry-gap", - "confidence": "low", - "source": "compiler-unmatched:stream", - "tests": [] + "status": "partial", + "action": "audit-partial", + "verification": "test-backed", + "confidence": "high", + "source": "compiler-feature:stream.readable.some", + "tests": [ + "tests/corpus/5335-stream-operators-values.ts", + "tests/corpus/5336-stream-operators-promise-timing.ts", + "tests/corpus/5337-stream-operators-concurrency.ts", + "tests/corpus/5338-stream-operators-checks.cjs" + ] } } }, @@ -64115,12 +64152,17 @@ "priority": "low", "tiers": { "static": { - "status": "not-implemented", - "action": "verify-gap", - "verification": "registry-gap", - "confidence": "low", - "source": "compiler-unmatched:stream", - "tests": [] + "status": "partial", + "action": "audit-partial", + "verification": "test-backed", + "confidence": "high", + "source": "compiler-feature:stream.readable.find", + "tests": [ + "tests/corpus/5335-stream-operators-values.ts", + "tests/corpus/5336-stream-operators-promise-timing.ts", + "tests/corpus/5337-stream-operators-concurrency.ts", + "tests/corpus/5338-stream-operators-checks.cjs" + ] } } }, @@ -64140,12 +64182,17 @@ "priority": "low", "tiers": { "static": { - "status": "not-implemented", - "action": "verify-gap", - "verification": "registry-gap", - "confidence": "low", - "source": "compiler-unmatched:stream", - "tests": [] + "status": "partial", + "action": "audit-partial", + "verification": "test-backed", + "confidence": "high", + "source": "compiler-feature:stream.readable.every", + "tests": [ + "tests/corpus/5335-stream-operators-values.ts", + "tests/corpus/5336-stream-operators-promise-timing.ts", + "tests/corpus/5337-stream-operators-concurrency.ts", + "tests/corpus/5338-stream-operators-checks.cjs" + ] } } }, @@ -64165,12 +64212,17 @@ "priority": "low", "tiers": { "static": { - "status": "not-implemented", - "action": "verify-gap", - "verification": "registry-gap", - "confidence": "low", - "source": "compiler-unmatched:stream", - "tests": [] + "status": "partial", + "action": "audit-partial", + "verification": "test-backed", + "confidence": "high", + "source": "compiler-feature:stream.readable.flatMap", + "tests": [ + "tests/corpus/5335-stream-operators-values.ts", + "tests/corpus/5336-stream-operators-promise-timing.ts", + "tests/corpus/5337-stream-operators-concurrency.ts", + "tests/corpus/5338-stream-operators-checks.cjs" + ] } } }, @@ -64190,12 +64242,17 @@ "priority": "low", "tiers": { "static": { - "status": "not-implemented", - "action": "verify-gap", - "verification": "registry-gap", - "confidence": "low", - "source": "compiler-unmatched:stream", - "tests": [] + "status": "partial", + "action": "audit-partial", + "verification": "test-backed", + "confidence": "high", + "source": "compiler-feature:stream.readable.drop", + "tests": [ + "tests/corpus/5335-stream-operators-values.ts", + "tests/corpus/5336-stream-operators-promise-timing.ts", + "tests/corpus/5337-stream-operators-concurrency.ts", + "tests/corpus/5338-stream-operators-checks.cjs" + ] } } }, @@ -64215,12 +64272,17 @@ "priority": "low", "tiers": { "static": { - "status": "not-implemented", - "action": "verify-gap", - "verification": "registry-gap", - "confidence": "low", - "source": "compiler-unmatched:stream", - "tests": [] + "status": "partial", + "action": "audit-partial", + "verification": "test-backed", + "confidence": "high", + "source": "compiler-feature:stream.readable.take", + "tests": [ + "tests/corpus/5335-stream-operators-values.ts", + "tests/corpus/5336-stream-operators-promise-timing.ts", + "tests/corpus/5337-stream-operators-concurrency.ts", + "tests/corpus/5338-stream-operators-checks.cjs" + ] } } }, @@ -64240,12 +64302,17 @@ "priority": "low", "tiers": { "static": { - "status": "not-implemented", - "action": "verify-gap", - "verification": "registry-gap", - "confidence": "low", - "source": "compiler-unmatched:stream", - "tests": [] + "status": "partial", + "action": "audit-partial", + "verification": "test-backed", + "confidence": "high", + "source": "compiler-feature:stream.readable.reduce", + "tests": [ + "tests/corpus/5335-stream-operators-values.ts", + "tests/corpus/5336-stream-operators-promise-timing.ts", + "tests/corpus/5337-stream-operators-concurrency.ts", + "tests/corpus/5338-stream-operators-checks.cjs" + ] } } }, diff --git a/internal/compatibility/generated/node-v24-internal.json b/internal/compatibility/generated/node-v24-internal.json index dc46d2363..3fd2317c4 100644 --- a/internal/compatibility/generated/node-v24-internal.json +++ b/internal/compatibility/generated/node-v24-internal.json @@ -716,9 +716,9 @@ "apiEntries": 140, "static": { "supported": 0, - "partial": 38, + "partial": 53, "refused": 0, - "not-implemented": 102, + "not-implemented": 87, "by-design": 0, "unreviewed": 0, "not-applicable": 0 @@ -85802,7 +85802,15 @@ "tests/corpus/5300-stream-add-abort-signal.ts", "tests/corpus/5301-stream-add-abort-signal-errors.cjs", "tests/corpus/5302-stream-promises-options.ts", - "tests/corpus/5303-stream-finished-callback-opts.cjs" + "tests/corpus/5303-stream-finished-callback-opts.cjs", + "tests/corpus/5335-stream-operators-values.ts", + "tests/corpus/5336-stream-operators-promise-timing.ts", + "tests/corpus/5337-stream-operators-concurrency.ts", + "tests/corpus/5338-stream-operators-checks.cjs", + "tests/corpus/5340-readable-from-options.ts", + "tests/corpus/5341-readable-from-dynamic.cjs", + "tests/corpus/5342-stream-compose.ts", + "tests/corpus/5343-stream-compose-edges.ts" ] }, "anchorSource": "chapter" @@ -86120,8 +86128,12 @@ "inherited": true }, "static": { - "status": "not-implemented", - "evidence": "compiler-unmatched:stream" + "status": "partial", + "evidence": "compiler-feature:stream.stream.compose", + "tests": [ + "tests/corpus/5342-stream-compose.ts", + "tests/corpus/5343-stream-compose-edges.ts" + ] }, "anchorSource": "exact" }, @@ -86235,8 +86247,12 @@ "inherited": true }, "static": { - "status": "not-implemented", - "evidence": "compiler-unmatched:stream" + "status": "partial", + "evidence": "compiler-feature:stream.stream.Readable.from", + "tests": [ + "tests/corpus/5340-readable-from-options.ts", + "tests/corpus/5341-readable-from-dynamic.cjs" + ] }, "anchorSource": "exact" }, @@ -86695,8 +86711,12 @@ "inherited": true }, "static": { - "status": "not-implemented", - "evidence": "compiler-unmatched:stream" + "status": "partial", + "evidence": "compiler-feature:stream.stream.compose", + "tests": [ + "tests/corpus/5342-stream-compose.ts", + "tests/corpus/5343-stream-compose-edges.ts" + ] }, "anchorSource": "ancestor" }, @@ -86810,8 +86830,12 @@ "inherited": true }, "static": { - "status": "not-implemented", - "evidence": "compiler-unmatched:stream" + "status": "partial", + "evidence": "compiler-feature:stream.stream.Readable.from", + "tests": [ + "tests/corpus/5340-readable-from-options.ts", + "tests/corpus/5341-readable-from-dynamic.cjs" + ] }, "anchorSource": "ancestor" }, @@ -88294,8 +88318,14 @@ "inherited": false }, "static": { - "status": "not-implemented", - "evidence": "compiler-unmatched:stream" + "status": "partial", + "evidence": "compiler-feature:stream.readable.map", + "tests": [ + "tests/corpus/5335-stream-operators-values.ts", + "tests/corpus/5336-stream-operators-promise-timing.ts", + "tests/corpus/5337-stream-operators-concurrency.ts", + "tests/corpus/5338-stream-operators-checks.cjs" + ] }, "anchorSource": "exact" }, @@ -88317,8 +88347,14 @@ "inherited": false }, "static": { - "status": "not-implemented", - "evidence": "compiler-unmatched:stream" + "status": "partial", + "evidence": "compiler-feature:stream.readable.filter", + "tests": [ + "tests/corpus/5335-stream-operators-values.ts", + "tests/corpus/5336-stream-operators-promise-timing.ts", + "tests/corpus/5337-stream-operators-concurrency.ts", + "tests/corpus/5338-stream-operators-checks.cjs" + ] }, "anchorSource": "exact" }, @@ -88340,8 +88376,14 @@ "inherited": false }, "static": { - "status": "not-implemented", - "evidence": "compiler-unmatched:stream" + "status": "partial", + "evidence": "compiler-feature:stream.readable.forEach", + "tests": [ + "tests/corpus/5335-stream-operators-values.ts", + "tests/corpus/5336-stream-operators-promise-timing.ts", + "tests/corpus/5337-stream-operators-concurrency.ts", + "tests/corpus/5338-stream-operators-checks.cjs" + ] }, "anchorSource": "exact" }, @@ -88363,8 +88405,14 @@ "inherited": false }, "static": { - "status": "not-implemented", - "evidence": "compiler-unmatched:stream" + "status": "partial", + "evidence": "compiler-feature:stream.readable.toArray", + "tests": [ + "tests/corpus/5335-stream-operators-values.ts", + "tests/corpus/5336-stream-operators-promise-timing.ts", + "tests/corpus/5337-stream-operators-concurrency.ts", + "tests/corpus/5338-stream-operators-checks.cjs" + ] }, "anchorSource": "exact" }, @@ -88386,8 +88434,14 @@ "inherited": false }, "static": { - "status": "not-implemented", - "evidence": "compiler-unmatched:stream" + "status": "partial", + "evidence": "compiler-feature:stream.readable.some", + "tests": [ + "tests/corpus/5335-stream-operators-values.ts", + "tests/corpus/5336-stream-operators-promise-timing.ts", + "tests/corpus/5337-stream-operators-concurrency.ts", + "tests/corpus/5338-stream-operators-checks.cjs" + ] }, "anchorSource": "exact" }, @@ -88409,8 +88463,14 @@ "inherited": false }, "static": { - "status": "not-implemented", - "evidence": "compiler-unmatched:stream" + "status": "partial", + "evidence": "compiler-feature:stream.readable.find", + "tests": [ + "tests/corpus/5335-stream-operators-values.ts", + "tests/corpus/5336-stream-operators-promise-timing.ts", + "tests/corpus/5337-stream-operators-concurrency.ts", + "tests/corpus/5338-stream-operators-checks.cjs" + ] }, "anchorSource": "exact" }, @@ -88432,8 +88492,14 @@ "inherited": false }, "static": { - "status": "not-implemented", - "evidence": "compiler-unmatched:stream" + "status": "partial", + "evidence": "compiler-feature:stream.readable.every", + "tests": [ + "tests/corpus/5335-stream-operators-values.ts", + "tests/corpus/5336-stream-operators-promise-timing.ts", + "tests/corpus/5337-stream-operators-concurrency.ts", + "tests/corpus/5338-stream-operators-checks.cjs" + ] }, "anchorSource": "exact" }, @@ -88455,8 +88521,14 @@ "inherited": false }, "static": { - "status": "not-implemented", - "evidence": "compiler-unmatched:stream" + "status": "partial", + "evidence": "compiler-feature:stream.readable.flatMap", + "tests": [ + "tests/corpus/5335-stream-operators-values.ts", + "tests/corpus/5336-stream-operators-promise-timing.ts", + "tests/corpus/5337-stream-operators-concurrency.ts", + "tests/corpus/5338-stream-operators-checks.cjs" + ] }, "anchorSource": "exact" }, @@ -88478,8 +88550,14 @@ "inherited": false }, "static": { - "status": "not-implemented", - "evidence": "compiler-unmatched:stream" + "status": "partial", + "evidence": "compiler-feature:stream.readable.drop", + "tests": [ + "tests/corpus/5335-stream-operators-values.ts", + "tests/corpus/5336-stream-operators-promise-timing.ts", + "tests/corpus/5337-stream-operators-concurrency.ts", + "tests/corpus/5338-stream-operators-checks.cjs" + ] }, "anchorSource": "exact" }, @@ -88501,8 +88579,14 @@ "inherited": false }, "static": { - "status": "not-implemented", - "evidence": "compiler-unmatched:stream" + "status": "partial", + "evidence": "compiler-feature:stream.readable.take", + "tests": [ + "tests/corpus/5335-stream-operators-values.ts", + "tests/corpus/5336-stream-operators-promise-timing.ts", + "tests/corpus/5337-stream-operators-concurrency.ts", + "tests/corpus/5338-stream-operators-checks.cjs" + ] }, "anchorSource": "exact" }, @@ -88524,8 +88608,14 @@ "inherited": false }, "static": { - "status": "not-implemented", - "evidence": "compiler-unmatched:stream" + "status": "partial", + "evidence": "compiler-feature:stream.readable.reduce", + "tests": [ + "tests/corpus/5335-stream-operators-values.ts", + "tests/corpus/5336-stream-operators-promise-timing.ts", + "tests/corpus/5337-stream-operators-concurrency.ts", + "tests/corpus/5338-stream-operators-checks.cjs" + ] }, "anchorSource": "exact" }, diff --git a/internal/compatibility/static-support.json b/internal/compatibility/static-support.json index 9d5da0935..d356f63e8 100644 --- a/internal/compatibility/static-support.json +++ b/internal/compatibility/static-support.json @@ -186,6 +186,9 @@ {"chapter": "readline", "symbols": ["readlinePromises.Interface", "readlinePromises.createInterface", "rl.question"], "status": "partial", "evidence": ["tests/corpus/5295-readline-promises-question.ts", "tests/corpus/5296-readline-lines.ts", "tests/corpus/5298-readline-abort.cjs", "tests/corpus/5299-readline-stdin.ts"]}, {"chapter": "readline", "symbols": ["readline.Interface", "rl.pause", "rl.resume", "rl.prompt", "rl.setPrompt", "rl.getPrompt", "rl.write", "rl", "'close'", "'error'", "'line'", "'pause'", "'resume'"], "status": "partial", "evidence": ["tests/corpus/5295-readline-promises-question.ts", "tests/corpus/5296-readline-lines.ts", "tests/corpus/5297-readline-callback-events.cjs", "tests/corpus/5299-readline-stdin.ts"]}, {"chapter": "stream", "symbols": ["stream.addAbortSignal"], "status": "partial", "evidence": ["tests/corpus/5300-stream-add-abort-signal.ts", "tests/corpus/5301-stream-add-abort-signal-errors.cjs"]}, + {"chapter": "stream", "symbols": ["readable.map", "readable.filter", "readable.forEach", "readable.toArray", "readable.some", "readable.find", "readable.every", "readable.flatMap", "readable.drop", "readable.take", "readable.reduce"], "status": "partial", "evidence": ["tests/corpus/5335-stream-operators-values.ts", "tests/corpus/5336-stream-operators-promise-timing.ts", "tests/corpus/5337-stream-operators-concurrency.ts", "tests/corpus/5338-stream-operators-checks.cjs"]}, + {"chapter": "stream", "symbols": ["stream.Readable.from"], "status": "partial", "evidence": ["tests/corpus/5340-readable-from-options.ts", "tests/corpus/5341-readable-from-dynamic.cjs"]}, + {"chapter": "stream", "symbols": ["stream.compose"], "status": "partial", "evidence": ["tests/corpus/5342-stream-compose.ts", "tests/corpus/5343-stream-compose-edges.ts"]}, {"chapter": "worker_threads", "symbols": ["Worker"], "status": "partial", "evidence": ["tests/corpus/native-worker-messages.ts", "tests/corpus/native-worker-commonjs.cjs", "tests/corpus/native-worker-environment.ts", "tests/corpus/native-worker-roots/main.ts"]}, {"chapter": "worker_threads", "symbols": ["MessagePort", "parentPort", "worker_threads.workerData", "isMainThread", "threadId"], "status": "partial", "evidence": ["tests/corpus/native-worker-commonjs.cjs", "tests/corpus/native-worker-port-lifetime.ts", "tests/corpus/native-worker-references.ts"]}, {"chapter": "worker_threads", "symbols": ["worker.postMessage", "port.postMessage"], "status": "partial", "evidence": ["tests/corpus/native-worker-message-graphs.ts", "tests/corpus/native-worker-shared-views.ts", "tests/corpus/native-worker-clone-errors.ts", "tests/corpus/native-worker-delayed-messages.ts"]}, diff --git a/packages/compiler/ambient/scriptc-node-fallback.d.ts b/packages/compiler/ambient/scriptc-node-fallback.d.ts index d37c4983e..5a1daf770 100644 --- a/packages/compiler/ambient/scriptc-node-fallback.d.ts +++ b/packages/compiler/ambient/scriptc-node-fallback.d.ts @@ -5527,6 +5527,14 @@ declare module "stream" { signal?: AbortSignal; } + /* The operators' options: concurrency and highWaterMark bound map's + * in-flight callbacks and queued results. */ + interface ArrayOptions { + concurrency?: number; + highWaterMark?: number; + signal?: AbortSignal; + } + class Readable extends EventEmitter { constructor(opts?: ReadableOptions); /* The underscore-method surface (@types/node declares these on the @@ -5557,6 +5565,51 @@ declare module "stream" { readonly destroyed: boolean; readonly errored: Error | null; static from(iterable: unknown, options?: ReadableOptions): Readable; + /* The Readable operators: chunks are strings or Buffers, map and + * flatMap callbacks return strings or Buffers (flatMap: arrays of + * them, or strings), and callbacks that declare the options argument + * are refused. */ + map( + fn: (data: any, options?: Pick) => any, + options?: ArrayOptions, + ): Readable; + filter( + fn: (data: any, options?: Pick) => boolean | Promise, + options?: ArrayOptions, + ): Readable; + forEach( + fn: (data: any, options?: Pick) => void | Promise, + options?: ArrayOptions, + ): Promise; + toArray(options?: Pick): Promise; + some( + fn: (data: any, options?: Pick) => boolean | Promise, + options?: ArrayOptions, + ): Promise; + find( + fn: (data: any, options?: Pick) => boolean | Promise, + options?: ArrayOptions, + ): Promise; + every( + fn: (data: any, options?: Pick) => boolean | Promise, + options?: ArrayOptions, + ): Promise; + flatMap( + fn: (data: any, options?: Pick) => any, + options?: ArrayOptions, + ): Readable; + drop(limit: number, options?: Pick): Readable; + take(limit: number, options?: Pick): Readable; + reduce( + fn: (previous: any, data: any, options?: Pick) => T, + initial?: undefined, + options?: Pick, + ): Promise; + reduce( + fn: (previous: T, data: any, options?: Pick) => T, + initial: T, + options?: Pick, + ): Promise; on(event: "data", listener: (chunk: any) => void): this; on(event: "end" | "close" | "readable" | "pause" | "resume", listener: () => void): this; on(event: "error", listener: (err: Error) => void): this; @@ -5773,6 +5826,9 @@ declare module "stream" { function addAbortSignal(signal: AbortSignal, stream: T): T; /* Declared so requires typecheck; each fences at its use site. */ function pipeline(...streams: unknown[]): unknown; + /* compose(...streams): a Duplex that writes to the first stream and + * reads from the last, with the streams running as a pipeline. */ + function compose(...streams: Array): Duplex; /* finished's options: readable/writable choose which ends count, * error: false ignores 'error' events, signal aborts the wait. */ interface FinishedOptions { @@ -5808,6 +5864,7 @@ declare module "stream" { Stream, addAbortSignal, pipeline, + compose, finished, getDefaultHighWaterMark, promises, diff --git a/packages/compiler/src/backend/llvm/emitter.ts b/packages/compiler/src/backend/llvm/emitter.ts index 5bfe250fd..423ce450e 100644 --- a/packages/compiler/src/backend/llvm/emitter.ts +++ b/packages/compiler/src/backend/llvm/emitter.ts @@ -2940,9 +2940,13 @@ export class LlEmitter { // Number-key Map/Set lookups probe the runtime's direct index inline: // while ndense is nonzero every live key is an integer below it, so an // in-range integral key reads its slot (entry index + 1, 0 = absent) and - // any other key is absent. Maps without the index (ndense 0) and every - // non-integral probe outside the table take the runtime lookup, which - // keeps the complete SameValueZero semantics. + // any other key is absent. Maps without the index (ndense 0) hash the + // key in sc_map_entry_f64_hashed, which scans the entries of a small map + // or probes a compact bucket table as scr_map_find_f64 does, and calls + // the runtime lookup for a wide table. It stays out of line so that the + // direct-index path stays small at every call site. Both only read + // memory, so LLVM merges lookups of one key when nothing is stored in + // between (a has() followed by get(), for example). if (this.decls.has(NUMBER_MAP_ENTRY_DECL)) { const sz = this.sizeType; let mapTmp = 0; @@ -2978,7 +2982,118 @@ export class LlEmitter { ` %dense.any = icmp ne ${sz} %nd, 0`, ` br i1 %dense.any, label %miss, label %slow`, `slow:`, - ` %r = call ptr @scr_map_entry_f64(ptr %m, double %k)`, + ` %hashed = call ptr @sc_map_entry_f64_hashed(ptr %m, double %k)`, + ` ret ptr %hashed`, + `}`, + ``, + // Hashing (scr_map_find_f64): the SameValueZero key word, its + // scr_key_mix hash (zero remapped to one) and the absent-key filter. + `define internal ptr @sc_map_entry_f64_hashed(ptr %m, double %k) noinline ${FN_ATTRS} {`, + `entry:`, + ` %nan = fcmp uno double %k, %k`, + ` %zero = fcmp oeq double %k, 0.0`, + ` %kz = select i1 %zero, double 0.0, double %k`, + ` %kzb = bitcast double %kz to i64`, + ` %kb = select i1 %nan, i64 9221120237041090560, i64 %kzb`, + ` %x1s = lshr i64 %kb, 30`, + ` %x1 = xor i64 %kb, %x1s`, + ` %x2 = mul i64 %x1, -4658895280553007687`, + ` %x3s = lshr i64 %x2, 27`, + ` %x3 = xor i64 %x2, %x3s`, + ` %x4 = mul i64 %x3, -7723592293110705685`, + ` %x5s = lshr i64 %x4, 31`, + ` %x5 = xor i64 %x4, %x5s`, + ` %hz = icmp eq i64 %x5, 0`, + ` %h = select i1 %hz, i64 1, i64 %x5`, + ` %f1s = lshr i64 %h, 44`, + ` %f1a = and i64 %f1s, 63`, + ` %f1 = shl i64 1, %f1a`, + ` %f2s = lshr i64 %h, 50`, + ` %f2a = and i64 %f2s, 63`, + ` %f2 = shl i64 1, %f2a`, + ` %fb = or i64 %f1, %f2`, + ` %fp = getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 11`, + ` %filter = load i64, ptr %fp`, + ` %fm = and i64 %filter, %fb`, + ` %fin = icmp eq i64 %fm, %fb`, + ` br i1 %fin, label %table, label %miss`, + `table:`, + ` %nbp = getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 14`, + ` %nb = load ${sz}, ptr %nbp`, + ` %eps = getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 13`, + ` %ents = load ptr, ptr %eps`, + ` %linear = icmp eq ${sz} %nb, 0`, + ` br i1 %linear, label %scan, label %buckets`, + // Small maps keep no bucket table: scan the cached entry hashes. + `scan:`, + ` %nep = getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 9`, + ` %ne = load ${sz}, ptr %nep`, + ` br label %scan.loop`, + `scan.loop:`, + ` %si = phi ${sz} [ 0, %scan ], [ %si.next, %scan.next ]`, + ` %sdone = icmp eq ${sz} %si, %ne`, + ` br i1 %sdone, label %miss, label %scan.entry`, + `scan.entry:`, + ` %se = getelementptr inbounds %ScrMapEntry, ptr %ents, ${sz} %si`, + ` %seh = getelementptr inbounds %ScrMapEntry, ptr %se, i32 0, i32 2`, + ` %sh = load i64, ptr %seh`, + ` %sk = load i64, ptr %se`, + ` %shq = icmp eq i64 %sh, %h`, + ` %skq = icmp eq i64 %sk, %kb`, + ` %sq = and i1 %shq, %skq`, + ` br i1 %sq, label %scan.found, label %scan.next`, + `scan.found:`, + ` ret ptr %se`, + `scan.next:`, + ` %si.next = add ${sz} %si, 1`, + ` br label %scan.loop`, + // Compact tables tag each 24-bit entry index (+1) with the hash's + // top byte; wider tables take the runtime lookup. + `buckets:`, + ` %wide = icmp ugt ${sz} %nb, 16777216`, + ` br i1 %wide, label %runtime, label %probe`, + `probe:`, + ` %bp = getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 15`, + ` %bk = load ptr, ptr %bp`, + ` %mask = sub ${sz} %nb, 1`, + ` %ht = lshr i64 %h, 32`, + ` %ht32 = trunc i64 %ht to i32`, + ` %tag = and i32 %ht32, -16777216`, + sz === "i64" ? ` %hi = add i64 %h, 0` : ` %hi = trunc i64 %h to ${sz}`, + ` %i0 = and ${sz} %hi, %mask`, + ` br label %probe.loop`, + `probe.loop:`, + ` %pi = phi ${sz} [ %i0, %probe ], [ %pi.next, %probe.next ]`, + ` %bep = getelementptr inbounds i32, ptr %bk, ${sz} %pi`, + ` %b = load i32, ptr %bep`, + ` %bempty = icmp eq i32 %b, 0`, + ` br i1 %bempty, label %miss, label %probe.tag`, + `probe.tag:`, + ` %bt = and i32 %b, -16777216`, + ` %btq = icmp eq i32 %bt, %tag`, + ` br i1 %btq, label %probe.entry, label %probe.next`, + `probe.entry:`, + ` %bix = and i32 %b, 16777215`, + ` %bix1 = sub i32 %bix, 1`, + sz === "i32" ? ` %bi = add i32 %bix1, 0` : ` %bi = zext i32 %bix1 to ${sz}`, + ` %pe = getelementptr inbounds %ScrMapEntry, ptr %ents, ${sz} %bi`, + ` %peh = getelementptr inbounds %ScrMapEntry, ptr %pe, i32 0, i32 2`, + ` %ph = load i64, ptr %peh`, + ` %pk = load i64, ptr %pe`, + ` %phq = icmp eq i64 %ph, %h`, + ` %pkq = icmp eq i64 %pk, %kb`, + ` %pq = and i1 %phq, %pkq`, + ` br i1 %pq, label %probe.found, label %probe.next`, + `probe.found:`, + ` ret ptr %pe`, + `probe.next:`, + ` %pi.inc = add ${sz} %pi, 1`, + ` %pi.next = and ${sz} %pi.inc, %mask`, + ` br label %probe.loop`, + `miss:`, + ` ret ptr null`, + `runtime:`, + ` %r = call ptr @scr_map_entry_f64(ptr %m, double %k) memory(read) nounwind willreturn`, ` ret ptr %r`, `}`, ``, diff --git a/packages/compiler/src/backend/llvm/lib-shared.ts b/packages/compiler/src/backend/llvm/lib-shared.ts index 84f039f2c..d5251337a 100644 --- a/packages/compiler/src/backend/llvm/lib-shared.ts +++ b/packages/compiler/src/backend/llvm/lib-shared.ts @@ -941,6 +941,19 @@ export const LIB_FN_SYMS: Record = { "readable.nextChunk": "scr_stream_next_chunk", "readable.nextChunkDyn": "scr_stream_next_chunk_dyn", "readable.fromArr": "scr_stream_from_arr", + "readable.fromDyn": "scr_stream_from_dyn", + // The Readable operators: borrowed operands; +1 stream or promise. + "readable.opMap": "scr_stream_op_map", + "readable.opFilter": "scr_stream_op_filter", + "readable.opFlatMap": "scr_stream_op_flat_map", + "readable.opDrop": "scr_stream_op_drop", + "readable.opTake": "scr_stream_op_take", + "readable.opToArray": "scr_stream_op_to_array", + "readable.opForEach": "scr_stream_op_for_each", + "readable.opSome": "scr_stream_op_some", + "readable.opEvery": "scr_stream_op_every", + "readable.opFind": "scr_stream_op_find", + "readable.opReduce": "scr_stream_op_reduce", "readable.pipe": "scr_stream_pipe", "writable.cork": "scr_stream_cork", "writable.uncork": "scr_stream_uncork", @@ -1323,6 +1336,18 @@ export const USES_TIMERS_LIB_FNS = new Set([ "readable.nextChunk", "readable.nextChunkDyn", "readable.fromArr", + "readable.fromDyn", + "readable.opMap", + "readable.opFilter", + "readable.opFlatMap", + "readable.opDrop", + "readable.opTake", + "readable.opToArray", + "readable.opForEach", + "readable.opSome", + "readable.opEvery", + "readable.opFind", + "readable.opReduce", "readable.resume", "readable.pipe", "readable.unpipe", @@ -1347,6 +1372,7 @@ export const USES_TIMERS_LIB_FNS = new Set([ "stream.finishedDyn", "stream.pipeline", "stream.pipelineDyn", + "stream.compose", "sp.finished", "sp.pipeline", "sp.pipelineOpts", diff --git a/packages/compiler/src/backend/llvm/lib-stream.ts b/packages/compiler/src/backend/llvm/lib-stream.ts index 4bf1df6fc..ce8ac6d6a 100644 --- a/packages/compiler/src/backend/llvm/lib-stream.ts +++ b/packages/compiler/src/backend/llvm/lib-stream.ts @@ -612,6 +612,27 @@ export function emitStreamLibCall(host: LlvmEmitterContext, e: LibCallExpr): LlV host.emitPendingCheck(); return out; } + if (e.fn === "stream.compose") { + // compose(count, s1..sn): the stream list rides a stack array. + const countArg = e.args[0]!; + if (countArg.kind !== "numLit") + throw new InternalCompilerError(`llvm emitter bug: ${e.fn} count not a literal`); + const n = countArg.value; + const args = e.args.map((a) => host.emitExpr(a)); + const arr = B.slot(); + B.entryAllocas.push(`${arr} = alloca [${n} x ptr]`); + for (let i = 0; i < n; i++) { + const p = B.tmp(); + B.line(`${p} = getelementptr inbounds [${n} x ptr], ptr ${arr}, i64 0, i64 ${i}`); + B.line(`store ptr ${args[1 + i]!.name}, ptr ${p}`); + } + const t = B.tmp(); + host.declare(`declare ptr @scr_stream_compose(double, ptr)`); + B.line(`${t} = call ptr @scr_stream_compose(double ${f64Lit(n)}, ptr ${arr})`); + const out = host.own({ name: t, type: e.type }); + host.emitPendingCheck(); + return out; + } if (e.fn === "stream.pipeline" || e.fn === "stream.pipelineDyn") { // pipeline(count, s1..sn, cb): the destination answers +1. The // stream list rides a stack array (the C compound literal). diff --git a/packages/compiler/src/backend/llvm/map-number-lookup.test.ts b/packages/compiler/src/backend/llvm/map-number-lookup.test.ts index dbaf66fed..3c30c7239 100644 --- a/packages/compiler/src/backend/llvm/map-number-lookup.test.ts +++ b/packages/compiler/src/backend/llvm/map-number-lookup.test.ts @@ -117,3 +117,68 @@ test("string keys and reference values are retained inline, boolean slots compar expect(text).toContain("@scr_str_retain"); expect(text).toContain("call ptr @scr_map_iter_val_ref("); }); + +/** work(map, k): map.has(k) on a number-keyed map. */ +function lookupFixture(): IrModule { + const map: IrType = { kind: "map", key: F64, value: F64 }; + const params = [ + { localId: "map", name: "map", type: map }, + { localId: "k", name: "k", type: F64 }, + ]; + const has: IrExpr = { + kind: "mapIntrinsic", + method: "has", + receiver: ref("map", map), + args: [ref("k", F64)], + type: BOOL, + loc, + }; + const work: IrFunction = { + name: "work", + params, + returnType: BOOL, + loc, + locals: params.map((p) => ({ id: p.localId, name: p.name, type: p.type, mutable: true })), + body: [{ kind: "return", value: has, loc }], + }; + return { + irVersion: 16, + sourceFile: loc.file, + entry: "main", + records: [], + unions: [], + functions: [{ name: "main", params: [], locals: [], returnType: VOID, body: [], loc }, work], + }; +} + +test("hashed number-key lookups probe inline and call the runtime only for wide tables", () => { + for (const bits of [32, 64] as const) { + const sz = bits === 64 ? "i64" : "i32"; + const { work, module } = emitted(lookupFixture(), bits); + expect(work).toContain("call ptr @sc_map_entry_f64("); + const helper = /^define internal ptr @sc_map_entry_f64\(ptr %m, double %k\)[^]*?^}/m.exec( + module, + )![0]; + expect(helper).toContain("call ptr @sc_map_entry_f64_hashed(ptr %m, double %k)"); + const hashed = + /^define internal ptr @sc_map_entry_f64_hashed\(ptr %m, double %k\) noinline [^]*?^}/m.exec( + module, + )![0]; + // The SameValueZero key word (canonical NaN, +0 for -0) and the + // scr_key_mix multipliers 0xbf58476d1ce4e5b9 and 0x94d049bb133111eb. + expect(hashed).toContain("select i1 %nan, i64 9221120237041090560, i64 %kzb"); + expect(hashed).toContain("mul i64 %x1, -4658895280553007687"); + expect(hashed).toContain("mul i64 %x3, -7723592293110705685"); + // Filter (field 11), small-map scan of the entry count (field 9), and + // compact buckets (fields 14 and 15) tagged with the hash's top byte. + expect(hashed).toContain("getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 11"); + expect(hashed).toContain(`%ne = load ${sz}, ptr %nep`); + expect(hashed).toContain(`%nb = load ${sz}, ptr %nbp`); + expect(hashed).toContain("getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 15"); + expect(hashed).toContain("and i32 %ht32, -16777216"); + expect(hashed).toContain(`icmp ugt ${sz} %nb, 16777216`); + expect(hashed).toContain( + "call ptr @scr_map_entry_f64(ptr %m, double %k) memory(read) nounwind willreturn", + ); + } +}); diff --git a/packages/compiler/src/backend/llvm/map-number-lookup.ts b/packages/compiler/src/backend/llvm/map-number-lookup.ts index abee9c007..fdbc4ba0d 100644 --- a/packages/compiler/src/backend/llvm/map-number-lookup.ts +++ b/packages/compiler/src/backend/llvm/map-number-lookup.ts @@ -3,8 +3,10 @@ * Number-keyed collections whose keys are small non-negative integers keep a * direct index in the runtime (scr_map.c). Reads go through the internal * sc_map_entry_f64 helper (emitter.ts helperDefs), which LLVM inlines: it - * resolves directly indexed maps without a call and defers to - * scr_map_entry_f64 otherwise. The generated code then reads the entry's + * resolves directly indexed maps without a call and other maps through + * the internal sc_map_entry_f64_hashed probe, which hashes the key as the + * runtime does and defers to scr_map_entry_f64 only for wide bucket + * tables. Both only read memory. The generated code then reads the entry's * value slot itself and retains a reference value with the value type's own * retain, which is the adapter the collection was constructed with. * String-keyed reads use the same entry contract through the internal diff --git a/packages/compiler/src/backend/native/executable.ts b/packages/compiler/src/backend/native/executable.ts index f57de66ee..30e425fcc 100644 --- a/packages/compiler/src/backend/native/executable.ts +++ b/packages/compiler/src/backend/native/executable.ts @@ -549,6 +549,7 @@ export async function compileCInternal( // The zlib stream classes reference both the zlib engine and the // stream unit. ...(opts.zlib && opts.stream ? [rt(join(rtDir, "scr_zlib_stream.c"))] : []), + ...(opts.stream && opts.dynAsync ? [rt(join(rtDir, "scr_stream_ops.c"))] : []), // The readiness-poller backends (scr_platform.h): kqueue on macOS/BSD, // epoll on Linux, WSAPoll on Windows — each TU is empty off its // platform, so all three link whenever a poller-using unit does and diff --git a/packages/compiler/src/frontend/lowering/class-dynamic-dispatch.ts b/packages/compiler/src/frontend/lowering/class-dynamic-dispatch.ts index 507c3d243..df87fbf36 100644 --- a/packages/compiler/src/frontend/lowering/class-dynamic-dispatch.ts +++ b/packages/compiler/src/frontend/lowering/class-dynamic-dispatch.ts @@ -59,13 +59,14 @@ import { import { classStaticData } from "./class-static-data.js"; import { isJsSourceFile } from "../program.js"; import * as ts from "../ts7/adapter.js"; -import { SYMBOL_T } from "../../ir/ir.js"; +import { SYMBOL_T, type IrLibFn } from "../../ir/ir.js"; import { isClassCallback } from "./class-callbacks.js"; import { CaughtValueDispatch } from "./caught-value-dispatch.js"; import { refreshDescriptorGuards } from "./class-descriptors.js"; import { ClassConstructionDispatch } from "./class-construction.js"; import { emitterRooted } from "./lower-event-emitter.js"; import { protocolArm, protocolFieldArm } from "./class-protocol-dispatch.js"; +import { STREAM_OPERATORS } from "./lower-stream.js"; const STREAM_METHODS = new Set([ "on", @@ -1089,7 +1090,12 @@ export class ClassDynamicDispatch { const matching = this.propertyReceivers(lowerer, true).flatMap(({ info }) => { if (!info || info.fields.has(method) || info.builtinError) return []; if (emitterRooted(lowerer, info) && EMITTER_METHODS.has(method)) return [info]; - if (info.builtinStream) return STREAM_METHODS.has(method) ? [info] : []; + if (info.builtinStream) { + return STREAM_METHODS.has(method) || + (STREAM_OPERATORS.has(method) && info.builtinStream !== "w") + ? [info] + : []; + } return findMethodOn(lowerer, info, method) || findGenericMethodOn(lowerer, info, method) ? [info] : []; @@ -3969,6 +3975,48 @@ export class ClassDynamicDispatch { type: method === "isPaused" ? BOOL : receiver.type, loc, }); + if (STREAM_OPERATORS.has(method) && info.builtinStream && info.builtinStream !== "w") { + // The Readable operators over a checked-dynamic receiver: every + // argument is already a checked value. + const arg = (i: number): IrExpr => incoming[i] ?? dynUndefinedExpr(loc); + const readableT: IrType = { kind: "object", className: "%Readable" }; + const op = (fn: string, args: IrExpr[], type: IrType): IrStmt[] => + ret({ kind: "libCall", fn: fn as IrLibFn, args: [receiver, ...args], type, loc }); + switch (method) { + case "map": + return op("readable.opMap", [arg(0), arg(1)], readableT); + case "filter": + return op("readable.opFilter", [arg(0), arg(1)], readableT); + case "flatMap": + return op("readable.opFlatMap", [arg(0), arg(1)], readableT); + case "drop": + return op("readable.opDrop", [arg(0), arg(1)], readableT); + case "take": + return op("readable.opTake", [arg(0), arg(1)], readableT); + case "toArray": + return op("readable.opToArray", [arg(0)], { kind: "promise", inner: DYN }); + case "forEach": + return op("readable.opForEach", [arg(0), arg(1)], { kind: "promise", inner: VOID }); + case "some": + return op("readable.opSome", [arg(0), arg(1)], { kind: "promise", inner: BOOL }); + case "every": + return op("readable.opEvery", [arg(0), arg(1)], { kind: "promise", inner: BOOL }); + case "find": + return op("readable.opFind", [arg(0), arg(1)], { kind: "promise", inner: DYN }); + case "reduce": + return op( + "readable.opReduce", + [ + arg(0), + arg(1), + { kind: "boolLit", value: dispatch.source.args.length >= 2, type: BOOL, loc }, + arg(2), + { kind: "numLit", value: 0, type: F64, loc }, + ], + { kind: "promise", inner: DYN }, + ); + } + } if (method === "destroy" && (!incoming.length || incoming[0]?.kind === "unitLit")) return ret({ kind: "libCall", diff --git a/packages/compiler/src/frontend/lowering/lower-async-iteration.ts b/packages/compiler/src/frontend/lowering/lower-async-iteration.ts index bdf7e3d70..daade569b 100644 --- a/packages/compiler/src/frontend/lowering/lower-async-iteration.ts +++ b/packages/compiler/src/frontend/lowering/lower-async-iteration.ts @@ -317,6 +317,19 @@ function boolRef(localId: string, loc: ReturnType): IrExpr { return { kind: "varRef", localId, type: BOOL, loc }; } +/** The loop's stream: a JavaScript value typed as a stream can arrive as a + * checked-dynamic value (returned through an untyped function), which + * checks back into the stream class. */ +function streamSource( + lowerer: Lowerer, + plan: NodeReadablePlan, + loc: ReturnType, +): IrExpr { + const v = lowerer.lowerExpr(plan.source); + if (v.type.kind !== "dyn") return v; + return { kind: "dynCheck", value: v, type: plan.type, loc }; +} + function lowerNodeReadable( lowerer: Lowerer, stmt: ts.ForOfStatement, @@ -423,7 +436,7 @@ function lowerNodeReadable( return { kind: "block", body: [ - { kind: "varDecl", localId: stream.id, init: lowerer.lowerExpr(plan.source), loc }, + { kind: "varDecl", localId: stream.id, init: streamSource(lowerer, plan, loc), loc }, { kind: "varDecl", localId: active.id, diff --git a/packages/compiler/src/frontend/lowering/lower-calls.ts b/packages/compiler/src/frontend/lowering/lower-calls.ts index ef4d9702d..daf75f727 100644 --- a/packages/compiler/src/frontend/lowering/lower-calls.ts +++ b/packages/compiler/src/frontend/lowering/lower-calls.ts @@ -150,6 +150,7 @@ import { } from "./lower-console.js"; import { STREAM_API_MEMBERS, + STREAM_OPERATORS, lowerStreamMethodCall, lowerStreamStaticCall, streamSidesOf, @@ -8592,7 +8593,7 @@ export function lowerObjectMethodCall( if ( info && !found && - STREAM_API_MEMBERS.has(access.name.text) && + (STREAM_API_MEMBERS.has(access.name.text) || STREAM_OPERATORS.has(access.name.text)) && streamSidesOf(lowerer, info) !== null ) { const stream = lowerStreamMethodCall(lowerer, call, access, info); diff --git a/packages/compiler/src/frontend/lowering/lower-stream.ts b/packages/compiler/src/frontend/lowering/lower-stream.ts index 607681fc0..dfab11cc9 100644 --- a/packages/compiler/src/frontend/lowering/lower-stream.ts +++ b/packages/compiler/src/frontend/lowering/lower-stream.ts @@ -38,6 +38,8 @@ import { appendImplicitUndefinedReturn } from "./function-returns.js"; import { bufEncoding, knownBufEncoding } from "./containers/bytes.js"; import { tryLowerExpression } from "./expressions/try-lower-expression.js"; import { sideEffectFreeOptionValue } from "./surfaces.js"; +import { lowerDynObjectLiteral } from "./expressions/object-literals.js"; +import { dynUndefinedExpr } from "../../ir/build.js"; import { BOOL, DYN, @@ -1346,12 +1348,14 @@ export function lowerStreamUnderscoreAssign( /* ── the static surface (Readable.from) ───────────────────────────────── */ -/** `Readable.from(iterable)` — the array-seeded object-entry form: - * string[] / Buffer[] values (or literals) become a fully-seeded - * readable delivering one WHOLE chunk per element (Node's objectMode - * accounting; hwm 1), and a single string/Buffer argument is a - * one-chunk stream (Node's special case). Called from the method-call - * chain once the receiver resolved to a stream class value. */ +/** `Readable.from(iterable, options)`: string[] / Buffer[] sources + * become a readable that pushes one element per _read (whole entries in + * object mode, hwm 1), and a single string/Buffer argument is a + * one-chunk stream (from.js's special case, the mode's default hwm). The + * options lower when they are a literal of `objectMode` and + * `highWaterMark`; the JS lane and `any` sources take the runtime's + * checked-dynamic form. Called from the method-call chain once the + * receiver resolved to a stream class value. */ export function lowerStreamStaticCall( lowerer: Lowerer, call: ts.CallExpression, @@ -1410,49 +1414,97 @@ export function lowerStreamStaticCall( lowerer.noLowering(`${cls.slice(1)}.from`, call, "Readable.from is the lowered form"); } const args = call.arguments; - if (args.length !== 1) { + if (args.length > 2) { lowerer.noLowering( `Readable.from with ${args.length} arguments`, call, - "the one-argument form (an array, string, or Buffer) is supported", + "Readable.from(iterable, options) is the supported form", ); } - const srcT = lowerer.mapTypeOf(lowerer.typeOf(args[0]!)); const type: IrType = { kind: "object", className: "%Readable" }; - if ( - srcT?.kind === "array" && - (srcT.elem.kind === "string" || (srcT.elem.kind === "bytes" && srcT.elem.elem === "u8")) - ) { - const arr = lowerer.lowerExpr(args[0]!); - const strings = srcT.elem.kind === "string"; + const js = isJsSourceFile(call.getSourceFile()); + const srcNode = args[0]; + const optsNode = args[1]; + const srcT = srcNode ? lowerer.mapTypeOf(lowerer.typeOf(srcNode)) : null; + const typedElem = (t: IrType | null | undefined): "string" | "bytes" | null => + t?.kind === "string" ? "string" : t?.kind === "bytes" && t.elem === "u8" ? "bytes" : null; + const single = typedElem(srcT); + const elem = srcT?.kind === "array" ? typedElem(srcT.elem) : null; + const opts = optsNode ? fromOptions(lowerer, optsNode) : { flags: 0, hwm: null }; + if (srcNode && (single || elem) && opts) { + const v = lowerer.lowerExpr(srcNode); + const hwm = opts.hwm ? lowerer.lowerExprExpecting(opts.hwm, F64) : numLit(0, loc); + const arr: IrExpr = single ? { kind: "arrayLit", elems: [v], type: arrayOf(v.type), loc } : v; + // SCR_FROM_STRINGS 1, SCR_FROM_SINGLE 2 (scr_runtime.h); the options + // add SCR_FROM_BYTE_MODE 4 and SCR_FROM_HAS_HWM 8. + const flags = ((single ?? elem) === "string" ? 1 : 0) | (single ? 2 : 0) | opts.flags; return { kind: "libCall", fn: "readable.fromArr", - args: [arr, boolLit(strings, loc)], + args: [arr, numLit(flags, loc), hwm], type, loc, }; } - if (srcT?.kind === "string" || (srcT?.kind === "bytes" && srcT.elem === "u8")) { - // Node special-cases a single string/Buffer: ONE whole chunk. - const v = lowerer.lowerExpr(args[0]!); - const strings = srcT.kind === "string"; - const arr: IrExpr = { kind: "arrayLit", elems: [v], type: arrayOf(v.type), loc }; - return { - kind: "libCall", - fn: "readable.fromArr", - args: [arr, boolLit(strings, loc)], - type, - loc, - }; + const anySrc = + srcT?.kind === "dyn" || + (srcNode !== undefined && + srcT === null && + (lowerer.typeOf(srcNode).flags & ts.TypeFlags.Any) !== 0); + if (js || anySrc) { + // The checked-dynamic form: the runtime takes from.js's ladder + // (ERR_INVALID_ARG_TYPE for a non-iterable, the options, null values). + const src = srcNode ? lowerer.lowerExprExpecting(srcNode, DYN) : dynUndefinedExpr(loc); + const o = lowerOperatorValue(lowerer, optsNode, "Readable.from options", loc, true); + return { kind: "libCall", fn: "readable.fromDyn", args: [src, o], type, loc }; + } + if (srcNode && (single || elem)) { + lowerer.noLowering( + "Readable.from with these options", + optsNode!, + "an object literal with 'objectMode' (true or false) and a number 'highWaterMark' is supported", + ); } lowerer.noLowering( `Readable.from over a '${srcT ? lowerer.fmt(srcT) : "?"}' source`, - args[0]!, + srcNode ?? call, "string[] / Buffer[] arrays (and a single string or Buffer) are the supported sources — generators and async iterables have no lowering yet", ); } +/** Readable.from's typed options: an object literal of `objectMode` + * (a boolean literal) and `highWaterMark` (a number), as SCR_FROM_* flag + * bits and the hwm expression; null for any other shape. */ +function fromOptions( + lowerer: Lowerer, + node: ts.Expression, +): { flags: number; hwm: ts.Expression | null } | null { + if (!ts.isObjectLiteralExpression(node)) return null; + let flags = 0; + let hwm: ts.Expression | null = null; + for (const prop of node.properties) { + if ( + !ts.isPropertyAssignment(prop) || + !(ts.isIdentifier(prop.name) || ts.isStringLiteral(prop.name)) + ) { + return null; + } + const value = prop.initializer; + if (prop.name.text === "objectMode") { + if (value.kind === ts.SyntaxKind.FalseKeyword) flags |= 4; + else if (value.kind === ts.SyntaxKind.TrueKeyword) flags &= ~4; + else return null; + } else if (prop.name.text === "highWaterMark") { + if (lowerer.mapTypeOf(lowerer.typeOf(value))?.kind !== "f64") return null; + flags |= 8; + hwm = value; + } else { + return null; + } + } + return { flags, hwm }; +} + /** `const Writable = stream.Writable` — a runtime stream CLASS bound * through a namespace member: alias plumbing with no storage (both * declaration walks skip it; uses resolve through builtinStreamInfoOf's @@ -1491,7 +1543,7 @@ const PIPELINE_STAGE_EXPECTED = function lowerStreamArg( lowerer: Lowerer, node: ts.Expression, - what: "finished" | "pipeline" | "addAbortSignal", + what: "finished" | "pipeline" | "addAbortSignal" | "compose", ): IrExpr { const v = lowerer.lowerExpr(node); const info = v.type.kind === "object" ? lowerer.classes.get(v.type.className) : undefined; @@ -1546,7 +1598,7 @@ function lowerStreamArg( lowerer.noLowering( `${what} over a '${lowerer.fmt(v.type)}'`, node, - what === "pipeline" + what === "pipeline" || what === "compose" ? "stream stages are the lowered surface — Node also accepts iterables (strings, arrays), generators, plain functions, and { readable, writable } pairs, but those have no lowering yet" : "stream arguments are the lowered surface (iterables, generators, and plain functions have no lowering yet)", ); @@ -2005,11 +2057,295 @@ export function lowerStreamModuleCall( }; } + if (bi.member === "compose") return lowerStreamCompose(lowerer, call, loc); + return null; } +/** `stream.compose(...streams)` over stream instances: a Duplex writing + * to the first stream and reading from the last (scr_stream_compose). The + * classes fix which positions can be readable and writable; a stream in + * the wrong position, a single Readable or Writable (Duplex.from wraps + * those), and functions, iterables or web streams have no lowering. */ +function lowerStreamCompose(lowerer: Lowerer, call: ts.CallExpression, loc: SrcLoc): IrExpr { + const args = call.arguments; + const duplex: IrType = { kind: "object", className: "%Duplex" }; + if (args.some((a) => ts.isSpreadElement(a))) { + lowerer.noLowering("compose with spread arguments", call, "pass the streams directly"); + } + if (args.length === 0) { + return nodeThrowExpr( + 1, + "ERR_MISSING_ARGS", + 'The "streams" argument must be specified', + duplex, + loc, + ); + } + let streams: IrExpr[]; + try { + streams = args.map((a) => lowerStreamArg(lowerer, a, "compose")); + } catch (e) { + if (e instanceof StreamArgTypeThrow) return e.expr; + throw e; + } + const sides = streams.map((s) => + s.type.kind === "object" ? streamSidesOf(lowerer, lowerer.classes.get(s.type.className)) : null, + ); + if (streams.length === 1) { + // Duplex.from answers a duplex itself. + if (sides[0] === "rw") return streams[0]!; + lowerer.noLowering( + `compose of a single ${sides[0] === "r" ? "Readable" : "Writable"}`, + args[0]!, + "Duplex.from wraps a lone Readable or Writable, which has no lowering yet; one Duplex, Transform or PassThrough answers itself", + ); + } + for (let i = 0; i < streams.length; i++) { + const s = sides[i]; + const want = + i < streams.length - 1 && s === "w" ? "readable" : i > 0 && s === "r" ? "writable" : null; + if (want) { + lowerer.noLowering( + `compose with a stream that is not ${want} at position ${i}`, + args[i]!, + `Node throws ERR_INVALID_ARG_VALUE ('streams[${i}]' must be ${want}); every stream but the last must be readable and every stream but the first writable`, + ); + } + } + return { + kind: "libCall", + fn: "stream.compose", + args: [numLit(streams.length, loc), ...streams], + type: duplex, + loc, + }; +} + /* ── the method surface ───────────────────────────────────────────────── */ +/* ── the Readable operators ──────────────────────────────────────────── */ + +/** The Readable operators (scr_stream_ops.c — Node's operators.js). */ +export const STREAM_OPERATORS: ReadonlySet = new Set([ + "map", + "filter", + "flatMap", + "drop", + "take", + "toArray", + "forEach", + "some", + "every", + "find", + "reduce", +]); + +/** An operator callback as a checked-dynamic function value. Callbacks + * that declare the options argument ({ signal }) are refused: the + * AbortSignal Node passes there has no representation. map and flatMap + * callbacks must produce strings or Buffers (flatMap: arrays of them, or + * strings) — or checked-dynamic values, which the runtime checks. */ +function lowerOperatorCallback( + lowerer: Lowerer, + node: ts.Expression, + member: string, + maxParams: number, +): IrExpr { + const loc = locOf(node); + const optionsParam = (at: ts.Node): never => + lowerer.noLowering( + `${member} callbacks that declare the options argument`, + at, + "the { signal } argument has no lowering — take only the chunk", + ); + if ( + (ts.isArrowFunction(node) || ts.isFunctionExpression(node)) && + node.parameters.length > maxParams + ) { + optionsParam(node.parameters[maxParams]!); + } + const v = lowerer.lowerExpr(node); + if (v.type.kind === "dyn") return v; + if ( + v.type.kind !== "func" && + isJsSourceFile(node.getSourceFile()) && + lowerer.dynConvertible(v.type) + ) { + // JavaScript passes any value; Node's validateFunction throws. + return { kind: "dynFrom", value: v, type: DYN, loc }; + } + if (v.type.kind !== "func") { + lowerer.noLowering( + `${member} with a '${lowerer.fmt(v.type)}' callback`, + node, + "pass a function", + ); + } + if (v.type.params.length > maxParams) optionsParam(node); + if (member === "map" || member === "flatMap") { + const ret = v.type.ret.kind === "promise" ? v.type.ret.inner : v.type.ret; + const chunk = (t: IrType): boolean => + t.kind === "string" || t.kind === "dyn" || (t.kind === "bytes" && t.elem === "u8"); + const ok = + member === "map" + ? chunk(ret) + : ret.kind === "dyn" || ret.kind === "string" || (ret.kind === "array" && chunk(ret.elem)); + if (!ok) { + lowerer.noLowering( + `${member} callbacks that return '${lowerer.fmt(ret)}'`, + node, + member === "map" + ? "map results are stream chunks: strings or Buffers (object-mode streams of other values are not lowered)" + : "flatMap results are arrays of strings or Buffers, or strings", + ); + } + } + const getRecord = (id: string) => lowerer.shapes.get(id); + const getUnion = (id: string) => lowerer.unions.get(id); + if (!canBoxFuncIntoDyn(v.type, getRecord, getUnion)) { + lowerer.noLowering( + `${member} with this callback`, + node, + "the callback's parameter and result types must cross the checked-dynamic boundary", + ); + } + return { kind: "dynFrom", value: v, type: DYN, loc }; +} + +/** An operator's options (or reduce's initial value): an object literal + * becomes a checked-dynamic object, checked-dynamic values pass through, + * absent arguments are undefined. */ +function lowerOperatorValue( + lowerer: Lowerer, + node: ts.Expression | undefined, + what: string, + loc: SrcLoc, + literalObject: boolean, +): IrExpr { + if (node === undefined) return dynUndefinedExpr(loc); + if (literalObject && ts.isObjectLiteralExpression(node)) + return lowerDynObjectLiteral(lowerer, node); + const v = lowerer.lowerExpr(node); + if (v.type.kind === "dyn") return v; + if (literalObject && v.type.kind !== "undefinedT" && v.type.kind !== "nullT") { + lowerer.noLowering( + `${what} that are not an object literal`, + node, + "write the options inline, e.g. { concurrency: 2 }", + ); + } + if (!lowerer.dynConvertible(v.type)) { + lowerer.noLowering(`${what} of type '${lowerer.fmt(v.type)}'`, node); + } + return { kind: "dynFrom", value: v, type: DYN, loc: locOf(node) }; +} + +function lowerStreamOperator( + lowerer: Lowerer, + call: ts.CallExpression, + access: ts.PropertyAccessExpression, + member: string, +): IrExpr { + const loc = locOf(call); + const args = call.arguments; + const recv = lowerer.lowerExpr(access.expression); + const readable: IrType = { kind: "object", className: "%Readable" }; + const opts = (i: number): IrExpr => + lowerOperatorValue(lowerer, args[i], `${member} options`, loc, true); + const fnArg = (maxParams: number): IrExpr => { + if (args[0] === undefined) return dynUndefinedExpr(loc); + return lowerOperatorCallback(lowerer, args[0], member, maxParams); + }; + const lib = (fn: string, a: IrExpr[], type: IrType): IrExpr => ({ + kind: "libCall", + fn: fn as IrLibFn, + args: a, + type, + loc, + }); + switch (member) { + case "map": + case "filter": + case "flatMap": { + const fn = fnArg(1); + const fnName = + member === "map" + ? "readable.opMap" + : member === "filter" + ? "readable.opFilter" + : "readable.opFlatMap"; + return lib(fnName, [recv, fn, opts(1)], readable); + } + case "drop": + case "take": { + const n = lowerOperatorValue(lowerer, args[0], `${member}'s count`, loc, false); + return lib( + member === "drop" ? "readable.opDrop" : "readable.opTake", + [recv, n, opts(1)], + readable, + ); + } + case "toArray": + return lib("readable.opToArray", [recv, opts(0)], { kind: "promise", inner: DYN }); + case "forEach": + return lib("readable.opForEach", [recv, fnArg(1), opts(1)], { kind: "promise", inner: VOID }); + case "some": + case "every": { + const fn = fnArg(1); + return lib(member === "some" ? "readable.opSome" : "readable.opEvery", [recv, fn, opts(1)], { + kind: "promise", + inner: BOOL, + }); + } + case "find": { + const t = lowerer.typeOf(call); + const inner = lowerer.checker.getTypeArguments(t as ts.TypeReference)[0]; + if (inner && !(inner.flags & (ts.TypeFlags.Any | ts.TypeFlags.Unknown))) { + lowerer.noLowering( + "find with a type-guard callback", + call, + "the result is a checked-dynamic value — use a boolean predicate and narrow the result", + ); + } + return lib("readable.opFind", [recv, fnArg(1), opts(1)], { kind: "promise", inner: DYN }); + } + case "reduce": { + const t = lowerer.typeOf(call); + const resultTs = lowerer.checker.getTypeArguments(t as ts.TypeReference)[0]; + let kind = 0; + let inner: IrType = DYN; + if (resultTs && !(resultTs.flags & (ts.TypeFlags.Any | ts.TypeFlags.Unknown))) { + const mapped = lowerer.mapTypeOf(resultTs); + if (mapped?.kind === "f64") { + kind = 1; + inner = F64; + } else if (mapped?.kind === "bool") { + kind = 2; + inner = BOOL; + } else if (mapped?.kind === "string") { + kind = 3; + inner = STRING; + } else if (mapped?.kind !== "dyn") { + lowerer.noLowering( + `reduce with a '${mapped ? lowerer.fmt(mapped) : lowerer.checker.typeToString(resultTs)}' result`, + call, + "reduce results are numbers, booleans, strings, or checked-dynamic values", + ); + } + } + const fn = fnArg(2); + const init = lowerOperatorValue(lowerer, args[1], "reduce's initial value", loc, false); + return lib( + "readable.opReduce", + [recv, fn, init, boolLit(args.length >= 2, loc), opts(2), numLit(kind, loc)], + { kind: "promise", inner }, + ); + } + } + throw new InternalCompilerError(`unknown stream operator ${member}`); +} + /** The one supported encoding argument: utf8 (Node's default), as a * compile-time literal or absent. */ function checkUtf8Encoding(lowerer: Lowerer, member: string, arg: ts.Expression | undefined): void { @@ -2032,6 +2368,11 @@ export function lowerStreamMethodCall( info: ClassInfo, ): IrExpr | null { const member = access.name.text; + if (STREAM_OPERATORS.has(member)) { + const opSides = streamSidesOf(lowerer, info); + if (opSides === "r" || opSides === "rw") + return lowerStreamOperator(lowerer, call, access, member); + } if (!STREAM_API_MEMBERS.has(member)) return null; const sides = streamSidesOf(lowerer, info); if (!sides) return null; diff --git a/packages/compiler/src/frontend/lowering/lowerer.ts b/packages/compiler/src/frontend/lowering/lowerer.ts index 80d5cbe83..1c8d92a0a 100644 --- a/packages/compiler/src/frontend/lowering/lowerer.ts +++ b/packages/compiler/src/frontend/lowering/lowerer.ts @@ -1145,8 +1145,21 @@ export function dynFallbackType(lowerer: Lowerer, node: ts.Node, t: ts.Type): Ir symbol && lowerer.isStdlibSymbol(symbol) ? lowerer.checker.getTypeArguments(t as ts.TypeReference) : []; - if (symbol?.name === "Promise" && args[0] && args[0].flags & ts.TypeFlags.Any) - return jsFallbackPromiseType(lowerer, t); + // Promise and Promise (an any[] is itself a checked + // value below): a promise of a checked value. + const anyArray = (x: ts.Type): boolean => + lowerer.checker.isArrayType(x) && + ((lowerer.checker.getTypeArguments(x as ts.TypeReference)[0]?.flags ?? 0) & + ts.TypeFlags.Any) !== + 0; + if ( + symbol?.name === "Promise" && + args[0] && + (args[0].flags & ts.TypeFlags.Any || anyArray(args[0])) + ) + return symbol.name === "Promise" && anyArray(args[0]) + ? { kind: "promise", inner: DYN } + : jsFallbackPromiseType(lowerer, t); if ( symbol && ["Generator", "AsyncGenerator", "IterableIterator"].includes(symbol.name) && diff --git a/packages/compiler/src/frontend/tsc-codes.ts b/packages/compiler/src/frontend/tsc-codes.ts index 49e4a59e1..fe48c8855 100644 --- a/packages/compiler/src/frontend/tsc-codes.ts +++ b/packages/compiler/src/frontend/tsc-codes.ts @@ -71,6 +71,7 @@ export const JS_RELAXED_TSC_CODES: ReadonlySet = new Set([ 2556, 2769, 2740, // 2322's elaborated "missing the following properties" form + 2559, // a value with no properties in common with an all-optional options type 2305, 2551, 2724, diff --git a/packages/compiler/src/ir/builtin-effects.ts b/packages/compiler/src/ir/builtin-effects.ts index 01d9cad07..b494aa2b9 100644 --- a/packages/compiler/src/ir/builtin-effects.ts +++ b/packages/compiler/src/ir/builtin-effects.ts @@ -242,6 +242,7 @@ export const MAY_THROW_LIB_FNS: ReadonlySet = new Set([ "stream.finishedDyn", "stream.pipeline", "stream.pipelineDyn", + "stream.compose", "sp.finished", "sp.pipeline", "sc.text", @@ -384,6 +385,13 @@ export const MAY_THROW_LIB_FNS: ReadonlySet = new Set([ "https.requestFn", "https.requestFnCb", "rl.question", + "readable.fromArr", + "readable.fromDyn", + "readable.opMap", + "readable.opFilter", + "readable.opFlatMap", + "readable.opDrop", + "readable.opTake", "rl.createSignal", "stream.finishedOpts", "stream.finishedOptsSignal", diff --git a/packages/compiler/src/ir/builtin-signatures.ts b/packages/compiler/src/ir/builtin-signatures.ts index 017f1752a..27650ecd0 100644 --- a/packages/compiler/src/ir/builtin-signatures.ts +++ b/packages/compiler/src/ir/builtin-signatures.ts @@ -3222,6 +3222,10 @@ export const LIB_FN_SIGS = defineLibFnSignatures({ "stream.finishedDyn": { argTypes: [null, DYN], result: VOID }, "stream.pipeline": { argTypes: [F64], result: VOID }, "stream.pipelineDyn": { argTypes: [F64], result: VOID }, + /** stream.compose(count, s1..sn): the Duplex over a pipeline of the + * streams (+1). MAY THROW (ERR_INVALID_ARG_VALUE for a stream that is + * no longer readable or writable). */ + "stream.compose": { argTypes: [F64], result: VOID }, /** node:stream/promises — the promise forms over the same machinery: * sp.finished: (s) → a pending void promise the terminal watcher * settles; sp.pipeline: (count, s1..sn) → the callback pipeline's @@ -3304,11 +3308,32 @@ export const LIB_FN_SIGS = defineLibFnSignatures({ * +1 promise of the next chunk — buffered content, the EOF sentinel * (empty Buffer / dyn undefined), or a rejection with the stream's * error. The Dyn twin boxes chunks by runtime tag (the JS lane). - * readable.fromArr is Readable.from(array): a fully-seeded object- - * entry stream (one whole chunk per element; strings per the flag). */ + * readable.fromArr is Readable.from over a typed array, string or + * Buffer: [array, SCR_FROM_* flags, highWaterMark]. readable.fromDyn is + * the JS lane's [source, options]. */ "readable.nextChunk": { argTypes: [null], result: VOID }, "readable.nextChunkDyn": { argTypes: [null], result: VOID }, - "readable.fromArr": { argTypes: [null, BOOL], result: VOID }, + "readable.fromArr": { argTypes: [null, F64, F64], result: VOID }, + "readable.fromDyn": { argTypes: [DYN, DYN], result: VOID }, + /** The Readable operators (scr_stream_ops.c): [stream, ...] with every + * other argument a checked-dynamic value (the callback, the count, the + * options). map/filter/flatMap/drop/take answer the object-mode output + * stream (validated as a stream class; THROW Node's validation errors); + * toArray settles a dyn array, find a dyn value, some/every a boolean, + * forEach nothing; reduce: [stream, fn, init, hasInit, options, kind] + * with the payload kind (0 dyn, 1 number, 2 boolean, 3 string). The + * promise-returning ones reject instead of throwing. */ + "readable.opMap": { argTypes: [null, DYN, DYN], result: VOID }, + "readable.opFilter": { argTypes: [null, DYN, DYN], result: VOID }, + "readable.opFlatMap": { argTypes: [null, DYN, DYN], result: VOID }, + "readable.opDrop": { argTypes: [null, DYN, DYN], result: VOID }, + "readable.opTake": { argTypes: [null, DYN, DYN], result: VOID }, + "readable.opToArray": { argTypes: [null, DYN], result: { kind: "promise", inner: DYN } }, + "readable.opForEach": { argTypes: [null, DYN, DYN], result: { kind: "promise", inner: VOID } }, + "readable.opSome": { argTypes: [null, DYN, DYN], result: { kind: "promise", inner: BOOL } }, + "readable.opEvery": { argTypes: [null, DYN, DYN], result: { kind: "promise", inner: BOOL } }, + "readable.opFind": { argTypes: [null, DYN, DYN], result: { kind: "promise", inner: DYN } }, + "readable.opReduce": { argTypes: [null, DYN, DYN, BOOL, DYN, F64], result: VOID }, "readable.resume": { argTypes: [null], result: VOID }, "readable.isPaused": { argTypes: [null], result: BOOL }, "readable.pipe": { argTypes: [null, null, BOOL], result: VOID }, diff --git a/packages/compiler/src/ir/runtime-features.ts b/packages/compiler/src/ir/runtime-features.ts index af8c64a77..b7b4c6569 100644 --- a/packages/compiler/src/ir/runtime-features.ts +++ b/packages/compiler/src/ir/runtime-features.ts @@ -231,6 +231,9 @@ function scanRuntimeFeatures(mod: IrModule, stopAt?: keyof RuntimeFeatures): Run ) features.stream = true; if (fn.startsWith("zlib.")) features.zlib = true; + // The Readable operators (scr_stream_ops.c) await through the + // dyn-async unit. + if (fn.startsWith("readable.op")) features.dynAsync = true; if (fn.startsWith("dc.")) features.dc = true; if (fn.startsWith("assert.")) features.assert = true; if ( diff --git a/packages/compiler/src/ir/validate.ts b/packages/compiler/src/ir/validate.ts index 205949013..bdea755bd 100644 --- a/packages/compiler/src/ir/validate.ts +++ b/packages/compiler/src/ir/validate.ts @@ -3960,6 +3960,7 @@ function validateFunction( e.fn === "passthrough.initDyn" || e.fn === "stream.pipeline" || e.fn === "stream.pipelineDyn" || + e.fn === "stream.compose" || e.fn === "sp.pipeline" || e.fn === "sp.pipelineOpts" || e.fn === "sp.pipelineOptsSignal" || @@ -5239,6 +5240,7 @@ function validateFunction( e.fn === "stream.finishedDyn" || e.fn === "stream.pipeline" || e.fn === "stream.pipelineDyn" || + e.fn === "stream.compose" || e.fn === "stream.addAbortSignal" || e.fn === "stream.addAbortSignalFn" || e.fn === "stream.finishedOpts" || @@ -5329,6 +5331,22 @@ function validateFunction( } return true; } + if (e.fn === "stream.compose") { + // (count, s1..sn) → a Duplex (n >= 2), or the lone stream. + const count = e.args[0]; + const n = count?.kind === "numLit" ? count.value : -1; + if (n < 1 || e.args.length !== n + 1) { + err(`libCall ${e.fn} count/arity mismatch`, e.loc); + return true; + } + for (let i = 1; i <= n; i++) { + if (!isStreamObject(e.args[i]?.type)) { + err(`libCall ${e.fn} stage ${i} must be a stream-hierarchy object`, e.loc); + } + } + if (!isStreamObject(e.type)) err(`libCall ${e.fn} must return a stream`, e.loc); + return true; + } if (e.fn === "stream.pipeline" || e.fn === "stream.pipelineDyn") { // (count, s1..sn, cb) → the destination's type. const count = e.args[0]; @@ -5354,10 +5372,35 @@ function validateFunction( } return true; } - if (e.fn === "readable.fromArr") { - // No receiver: the seed array leads; the result is the class. - if (e.args[0]?.type.kind !== "array" || !isStreamObject(e.type)) { - err(`libCall readable.fromArr must take an array and return a stream class`, e.loc); + if ( + e.fn === "readable.opMap" || + e.fn === "readable.opFilter" || + e.fn === "readable.opFlatMap" || + e.fn === "readable.opDrop" || + e.fn === "readable.opTake" + ) { + if (!isStreamObject(e.args[0]?.type) || !isStreamObject(e.type)) { + err(`libCall ${e.fn} must take a stream and return a stream class`, e.loc); + } + return true; + } + if (e.fn === "readable.opReduce") { + const inner = e.type.kind === "promise" ? e.type.inner : null; + const kind = e.args[5]?.kind === "numLit" ? e.args[5].value : -1; + const want = [DYN, F64, BOOL, STRING][kind]; + if (!isStreamObject(e.args[0]?.type) || !inner || !want || !typeEquals(inner, want)) { + err(`libCall readable.opReduce must return the promise its kind names`, e.loc); + } + return true; + } + if (e.fn === "readable.fromArr" || e.fn === "readable.fromDyn") { + // No receiver: the source leads; the result is the class. + const src = e.fn === "readable.fromArr" ? "array" : "dyn"; + if (e.args[0]?.type.kind !== src || !isStreamObject(e.type)) { + err(`libCall ${e.fn} must take a ${src} source and return a stream class`, e.loc); + } + if (e.fn === "readable.fromArr" && e.args[1]?.kind !== "numLit") { + err(`libCall readable.fromArr flags must be a number literal`, e.loc); } return true; } diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5335-stream-operators-values.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5335-stream-operators-values.ts.json new file mode 100644 index 000000000..16992748f --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5335-stream-operators-values.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/5335-stream-operators-values.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5336-stream-operators-promise-timing.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5336-stream-operators-promise-timing.ts.json new file mode 100644 index 000000000..047f67160 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5336-stream-operators-promise-timing.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/5336-stream-operators-promise-timing.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5337-stream-operators-concurrency.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5337-stream-operators-concurrency.ts.json new file mode 100644 index 000000000..940a8d8ad --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5337-stream-operators-concurrency.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/5337-stream-operators-concurrency.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5338-stream-operators-checks.cjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5338-stream-operators-checks.cjs.json new file mode 100644 index 000000000..d177f7d00 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5338-stream-operators-checks.cjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/5338-stream-operators-checks.cjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5339-stream-for-await-hops.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5339-stream-for-await-hops.ts.json new file mode 100644 index 000000000..6a408e78a --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5339-stream-for-await-hops.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/5339-stream-for-await-hops.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5340-readable-from-options.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5340-readable-from-options.ts.json new file mode 100644 index 000000000..e2391f004 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5340-readable-from-options.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/5340-readable-from-options.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5341-readable-from-dynamic.cjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5341-readable-from-dynamic.cjs.json new file mode 100644 index 000000000..ab5412d9d --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5341-readable-from-dynamic.cjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/5341-readable-from-dynamic.cjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5342-stream-compose.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5342-stream-compose.ts.json new file mode 100644 index 000000000..f556fd730 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5342-stream-compose.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/5342-stream-compose.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5343-stream-compose-edges.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5343-stream-compose-edges.ts.json new file mode 100644 index 000000000..284409138 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5343-stream-compose-edges.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/5343-stream-compose-edges.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5344-stream-pipeline-timing.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5344-stream-pipeline-timing.ts.json new file mode 100644 index 000000000..c264a1d5e --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5344-stream-pipeline-timing.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/5344-stream-pipeline-timing.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5710-map-number-keys-hashed.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5710-map-number-keys-hashed.ts.json new file mode 100644 index 000000000..56107e4cc --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5710-map-number-keys-hashed.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/5710-map-number-keys-hashed.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5712-map-number-overwrite-at-growth.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5712-map-number-overwrite-at-growth.ts.json new file mode 100644 index 000000000..5f8dcf6b4 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/5712-map-number-overwrite-at-growth.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/5712-map-number-overwrite-at-growth.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/stream-dynamic-entry-kinds.cjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/stream-dynamic-entry-kinds.cjs.json new file mode 100644 index 000000000..31e4202c6 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/stream-dynamic-entry-kinds.cjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/stream-dynamic-entry-kinds.cjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/stream-dynamic-entry-kinds.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/stream-dynamic-entry-kinds.ts.json new file mode 100644 index 000000000..3f64353a3 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/stream-dynamic-entry-kinds.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/stream-dynamic-entry-kinds.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/stream-encoded-from-entries.cjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/stream-encoded-from-entries.cjs.json new file mode 100644 index 000000000..15f7272e4 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/stream-encoded-from-entries.cjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/stream-encoded-from-entries.cjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/stream-encoded-from-entries.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/stream-encoded-from-entries.ts.json new file mode 100644 index 000000000..daa9e6e5c --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/stream-encoded-from-entries.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/stream-encoded-from-entries.ts" + ], + "diags": [] +} diff --git a/packages/runtime-darwin-arm64/runtime-pack-matrix.mjs b/packages/runtime-darwin-arm64/runtime-pack-matrix.mjs index 5014bb6ae..ad678650f 100644 --- a/packages/runtime-darwin-arm64/runtime-pack-matrix.mjs +++ b/packages/runtime-darwin-arm64/runtime-pack-matrix.mjs @@ -90,6 +90,8 @@ const optional = [ ["scr_stream.c", "stream"], // The zlib stream classes reference both the zlib engine and the stream unit. ["scr_zlib_stream.c", all("zlib", "stream")], + // The Readable operators await through the dyn-async unit. + ["scr_stream_ops.c", all("stream", "dynAsync")], ["scr_loop_kqueue.c", any("netEffective", "dgram")], ["scr_loop_epoll.c", any("netEffective", "dgram")], ["scr_loop_wsapoll.c", any("netEffective", "dgram")], diff --git a/packages/runtime-pack-common/runtime-pack-matrix.mjs b/packages/runtime-pack-common/runtime-pack-matrix.mjs index 8554bd870..3da4e2154 100644 --- a/packages/runtime-pack-common/runtime-pack-matrix.mjs +++ b/packages/runtime-pack-common/runtime-pack-matrix.mjs @@ -80,6 +80,8 @@ const OPTIONAL = [ ["scr_stream.c", "stream"], // The zlib stream classes reference both the zlib engine and the stream unit. ["scr_zlib_stream.c", all("zlib", "stream")], + // The Readable operators await through the dyn-async unit. + ["scr_stream_ops.c", all("stream", "dynAsync")], ["scr_loop_kqueue.c", any("netEffective", "dgram")], ["scr_loop_epoll.c", any("netEffective", "dgram")], ["scr_loop_wsapoll.c", any("netEffective", "dgram")], diff --git a/packages/runtime/src/scr_dyn.h b/packages/runtime/src/scr_dyn.h index 171256e62..704151ca3 100644 --- a/packages/runtime/src/scr_dyn.h +++ b/packages/runtime/src/scr_dyn.h @@ -533,6 +533,9 @@ void scr_dyn_arr_push_hole(ScrDyn *arr); ScrDyn *scr_dyn_new_obj(void); ScrDyn *scr_dyn_iterator(const ScrDyn *value, const ScrStr *spell); ScrDyn *scr_dyn_iterator_optional(const ScrDyn *value); +/* from.js's probe: 2 when value[Symbol.asyncIterator] is set, 1 when + * value[Symbol.iterator] is, 0 for neither, -1 with an error pending. */ +int scr_dyn_iterable_kind(const ScrDyn *value); void scr_dyn_install_iterator_symbol(ScrDyn *key); void scr_dyn_install_async_iterator_symbol(ScrDyn *key); ScrDyn *scr_dyn_array_values_function(void); diff --git a/packages/runtime/src/scr_json.c b/packages/runtime/src/scr_json.c index 82c14b7c4..11f2ff09a 100644 --- a/packages/runtime/src/scr_json.c +++ b/packages/runtime/src/scr_json.c @@ -4152,6 +4152,23 @@ static ScrDyn *scr_iterator_method(const ScrDyn *source) { return scr_builtin_iterable(source) ? scr_builtin_iterator_method(source) : scr_dyn_retain(scr_dyn_undefined()); } +int scr_dyn_iterable_kind(const ScrDyn *value) { + if (value->kind == SCR_DYN_NULL || value->kind == SCR_DYN_UNDEF) return 0; + if (scr_dyn_generator(value)) return value->v.typed_ref.type_key[0] == 'a' ? 2 : 1; + if (scr_async_iterator_symbol) { + ScrDyn *method = scr_dyn_symbol_key_get(value, scr_async_iterator_symbol, false); + if (!method) return -1; + bool set = method->kind != SCR_DYN_UNDEF && method->kind != SCR_DYN_NULL; + scr_dyn_release(method); + if (set) return 2; + } + ScrDyn *method = scr_iterator_method(value); + if (!method) return -1; + bool set = method->kind != SCR_DYN_UNDEF && method->kind != SCR_DYN_NULL; + scr_dyn_release(method); + return set ? 1 : 0; +} + /* Compiled class methods are dispatched before this checked-object path. */ ScrDyn *scr_dyn_iterator_optional(const ScrDyn *source) { ScrDyn *method = scr_iterator_method(source); diff --git a/packages/runtime/src/scr_map.c b/packages/runtime/src/scr_map.c index 463024a28..aaedb535e 100644 --- a/packages/runtime/src/scr_map.c +++ b/packages/runtime/src/scr_map.c @@ -258,7 +258,11 @@ static inline bool scr_map_lookup_eq(const ScrMap *m, uint64_t stored, uint64_t /* Compact tables keep a hash fingerprint beside each 24-bit entry index. * Rejecting a collision here avoids loading an unrelated ordered entry. * Larger tables keep full-width indices; collection size remains limited - * only by the allocator. Zero is empty in compact tables, so store index+1. */ + * only by the allocator. Zero is empty in compact tables, so store index+1. + * The LLVM backend's inline number-key probe (sc_map_entry_f64_hashed) + * reads this encoding for tables of at most 1 << 24 buckets, hashes keys + * as scr_map_find_f64 does, and calls scr_map_entry_f64 for wider tables; + * only the runtime unit test builds a smaller bound. */ #ifndef SCR_MAP_COMPACT_BUCKETS #define SCR_MAP_COMPACT_BUCKETS ((size_t)1 << 24) #endif @@ -999,10 +1003,19 @@ static inline bool scr_map_append_grows_index(const ScrMap *m) { /* Number keys arrive normalized (+0 for -0, canonical NaN). A directly * indexed map overwrites or appends through its table; a key the table - * cannot hold converts it to hashing first. */ + * cannot hold converts it to hashing first. When an append would grow a + * hashed map's index, the lookup that decides whether the key is new also + * overwrites a key that is already present. */ static void scr_map_set_f64_slot(ScrMap *m, uint64_t key, uint64_t val) { double k = scr_map_slot_to_f64(key); - if (!m->ndense && scr_map_append_grows_index(m) && scr_map_find_f64(m, k) == SCR_MAP_EMPTY) { + if (!m->ndense && scr_map_append_grows_index(m)) { + size_t e = scr_map_find_f64(m, k); + if (e != SCR_MAP_EMPTY) { + uint64_t old = m->entries[e].val; + m->entries[e].val = val; /* unlink before releasing (cycle collector) */ + scr_map_release_val(m, old); + return; + } scr_map_enter_dense(m, k); /* the append would grow the index */ } if (m->ndense) { diff --git a/packages/runtime/src/scr_map.h b/packages/runtime/src/scr_map.h index cc0322609..b987b4990 100644 --- a/packages/runtime/src/scr_map.h +++ b/packages/runtime/src/scr_map.h @@ -119,7 +119,8 @@ _Static_assert(offsetof(ScrMap, key_trace) == offsetof(ScrMap, val_trace) + 3 * /* The LLVM backend's inline number-key lookup (sc_map_entry_f64) reads * entries, dense and ndense through %ScrMapIx fields 13, 19 and 20, and * entries as %ScrMapEntry { key, val, hash }; %ScrMapIx field 11 is the - * i64 filter. */ + * i64 filter. Its hashed probe (sc_map_entry_f64_hashed) also reads the + * entry count, nbuckets and buckets as fields 9, 14 and 15. */ _Static_assert(offsetof(ScrMap, filter) == (offsetof(ScrMap, nlive) + sizeof(size_t) + _Alignof(uint64_t) - 1) / _Alignof(uint64_t) * _Alignof(uint64_t), "%ScrMapIx field 11 is ScrMap.filter, directly after nlive"); @@ -132,6 +133,9 @@ _Static_assert(offsetof(ScrMap, nentries) + sizeof(size_t) == offsetof(ScrMap, n _Static_assert(offsetof(ScrMapEntry, hash) == 16, "inline iteration reads ScrMapEntry.hash at byte 16"); _Static_assert(offsetof(ScrMap, entries) == offsetof(ScrMap, filter) + sizeof(uint64_t) + sizeof(size_t), "inline number-key lookups read ScrMap.entries as %ScrMapIx field 13"); +_Static_assert(offsetof(ScrMap, nbuckets) == offsetof(ScrMap, entries) + sizeof(void *) && + offsetof(ScrMap, buckets) == offsetof(ScrMap, nbuckets) + sizeof(size_t), + "inline number-key probes read ScrMap.nbuckets and buckets as %ScrMapIx fields 14 and 15"); _Static_assert(offsetof(ScrMap, dense) == offsetof(ScrMap, entries) + 6 * sizeof(void *), "inline number-key lookups read ScrMap.dense as %ScrMapIx field 19"); _Static_assert(offsetof(ScrMap, ndense) == offsetof(ScrMap, dense) + sizeof(void *), diff --git a/packages/runtime/src/scr_stream.c b/packages/runtime/src/scr_stream.c index 7ec455753..9856ac17c 100644 --- a/packages/runtime/src/scr_stream.c +++ b/packages/runtime/src/scr_stream.c @@ -111,7 +111,13 @@ struct ScrStreamState { bool reading, in_read_sync, ended, end_emitted, end_scheduled; bool need_readable, emitted_readable, readable_listening; bool resume_scheduled, maybe_more_scheduled; - bool encoded; /* string-chunk mode (decoder active) */ + /* The buffer's representation: encoded means every entry is a ScrStr, + * otherwise every entry is a ScrBytes (scr_stream_rbuf_push checks + * it). decoding: setEncoding() installed a decoder, so Buffer entries + * decode on the way in; a string object stream without one + * (Readable.from over strings) holds strings but decodes nothing. */ + bool encoded; + bool decoding; ScrStr *enc; /* owned canonical encoding name, or NULL */ double dec_pending; /* scr_strdec packed pending state */ /* The defaultEncoding option's push-side effect: how push(string) @@ -122,11 +128,17 @@ struct ScrStreamState { * toward length/hwm and delivers undivided — Node's objectMode * accounting for the from() surface; hwm is 1). */ bool object_entries; + /* An operator output's entry kind is decided by its first push. */ + bool object_kind_set; /* The parked for-await next() promise (at most one — the loop awaits * each chunk before asking again). next_dyn picks the resolution * shape: a dyn boxed by tag (the JS lane), or bytes. */ ScrPromise *next_waiter; bool next_dyn; + /* The iterator's own read() is calling _read: synchronous pushes + * buffer, and the waiter takes the whole buffer afterwards (Node's + * read() returns everything _read pushed). */ + bool next_reading; ScrClosure *read_cb; ScrStreamReadInv read_inv; } r; @@ -139,6 +151,7 @@ struct ScrStreamState { int corked; bool writing, wsync, need_drain, after_scheduled; bool ending, finished, prefinished, final_called, finish_scheduled; + bool final_pending; /* _final called, its callback not yet run */ ScrClosure *inflight_cb; /* owned or NULL */ ScrClosure *write_cb; ScrStreamChunkInv write_inv; @@ -154,6 +167,7 @@ struct ScrStreamState { ScrClosure *flush_cb; ScrStreamPlainInv flush_inv; bool flush_called; + bool flush_pending; /* _flush called, its callback not yet run */ /* A _final that completes at once (zlib's ZlibBase): 'prefinish' starts * the flush and 'finish' does not wait for it. */ bool flush_on_prefinish; @@ -240,6 +254,13 @@ static void scr_stream_entry_release(const ScrStreamState *st, void *e) { else scr_bytes_release((ScrBytes *)e); } +/* Releases an entry by the kind it had when it left the buffer: a 'data' + * listener may call setEncoding() while the entry is out. */ +static void scr_stream_entry_release_kind(void *e, bool is_str) { + if (is_str) scr_str_release((ScrStr *)e); + else scr_bytes_release((ScrBytes *)e); +} + static void scr_stream_state_drop(ScrStreamState *st, bool gc) { if (!st) return; if (st->errored) scr_error_release(st->errored); @@ -358,6 +379,7 @@ typedef enum { SCR_ST_CLOSE, /* 'close' (when emitClose) */ SCR_ST_END_W, /* half-open: end the writable side */ SCR_ST_WCB_ERR, /* a failed write's user cb (called plain) */ + SCR_ST_WRITE_ERR, /* onwriteError after a synchronous failed write */ SCR_ST_NEXT_EOF, /* a parked for-await's EOF sentinel, AFTER 'end' */ SCR_ST_FIN, /* notify finished()/pipeline watchers (already- * terminal registration) */ @@ -367,7 +389,7 @@ typedef struct ScrStreamTick { ScrStream *s; /* owned */ ScrStreamTickOp op; ScrError *err; /* owned or NULL (SCR_ST_ERROR) */ - ScrClosure *cb; /* owned or NULL (SCR_ST_AFTER_WRITE / SCR_ST_WCB_ERR) */ + ScrClosure *cb; /* owned or NULL (SCR_ST_AFTER_WRITE / SCR_ST_WCB_ERR / SCR_ST_WRITE_ERR) */ struct ScrStreamTick *next; } ScrStreamTick; @@ -432,6 +454,7 @@ static ScrError *scr_stream_mkerr(const char *code, const char *msg) { /* ── forward decls ────────────────────────────────────────────────────── */ static void scr_stream_flow(ScrStream *s); +static int scr_stream_add_entry(ScrStream *s, void *entry /*moves*/, bool is_str); static void scr_stream_emit_readable_now(ScrStream *s); static void scr_stream_maybe_read_more(ScrStream *s); static void scr_stream_finish_maybe(ScrStream *s); @@ -439,6 +462,7 @@ static void scr_stream_clear_buffer(ScrStream *s); static void scr_stream_do_destroy(ScrStream *s, ScrError *err /*borrowed*/); static void scr_stream_notify_finished(ScrStream *s); static void *scr_stream_read_n(ScrStream *s, double size); +static void *scr_stream_read_n_kind(ScrStream *s, double size, bool *is_str); static void scr_stream_settle_next(ScrStream *s); static ScrStream *scr_stream_alloc(const ScrVt *vt, const char *cls, bool has_r, bool has_w, double rhwm, double whwm, bool auto_destroy, @@ -463,7 +487,13 @@ static void scr_stream_error_or_destroy(ScrStream *s, ScrError *err) { /* ── readable internals ───────────────────────────────────────────────── */ -static void scr_stream_rbuf_push(ScrStreamState *st, void *chunk /*moves*/, bool front) { +static void scr_stream_rbuf_push(ScrStreamState *st, void *chunk /*moves*/, bool is_str, + bool front) { + if (is_str != st->r.encoded) { + /* every entry has the buffer's representation (encoded: ScrStr) */ + fputs("scriptc: internal error: stream buffer entry of the wrong kind\n", stderr); + abort(); + } if (st->r.n == st->r.cap) { st->r.cap = st->r.cap ? st->r.cap * 2 : 4; st->r.buf = realloc(st->r.buf, st->r.cap * sizeof *st->r.buf); @@ -680,11 +710,12 @@ static bool scr_stream_add_chunk(ScrStream *s, ScrBytes *chunk, bool front) { (scr_emitter_has((ScrEmitter *)s, "data") || st->pipes.n > 0)) { /* the direct-emit fast path: flowing, empty buffer, async push */ if (st->r.emitted_readable) st->r.emitted_readable = false; + bool is_str = st->r.encoded; scr_stream_emit_data(s, entry); - scr_stream_entry_release(st, entry); + scr_stream_entry_release_kind(entry, is_str); if (scr_exc_pending()) return false; } else { - scr_stream_rbuf_push(st, entry, front); + scr_stream_rbuf_push(st, entry, st->r.encoded, front); if (st->r.need_readable) scr_stream_emit_readable_nt(s); } if (!front) scr_stream_maybe_read_more(s); @@ -704,7 +735,18 @@ static void scr_stream_maybe_read_more(ScrStream *s) { * Answers a +1 Buffer — or a +1 string once encoded (lengths and the * size argument then count JS string units, Node's encoded accounting). */ static void *scr_stream_read_n(ScrStream *s, double size) { + bool is_str; + return scr_stream_read_n_kind(s, size, &is_str); +} + +/* read_n, also answering the chunk's kind as it was taken from the buffer + * (a string when the stream was encoded then). A _read or a lazy pull + * inside the read can change the representation, and a 'data' listener + * can call setEncoding() before this returns, so callers that need the + * kind read it here rather than from st->r.encoded afterwards. */ +static void *scr_stream_read_n_kind(ScrStream *s, double size, bool *is_str) { ScrStreamState *st = s->st; + *is_str = false; bool absent = size < 0; size_t n = 0; if (!absent) { @@ -761,6 +803,7 @@ static void *scr_stream_read_n(ScrStream *s, double size) { } } } + *is_str = st->r.encoded; void *ret = want > 0 && want <= st->r.length ? scr_stream_rbuf_take(st, want) : NULL; if (ret == NULL) { st->r.need_readable = st->r.length <= st->r.hwm; @@ -775,7 +818,7 @@ static void *scr_stream_read_n(ScrStream *s, double size) { if (ret != NULL && !st->error_emitted && !st->close_emitted) { scr_stream_emit_data(s, ret); if (scr_exc_pending()) { - scr_stream_entry_release(st, ret); + scr_stream_entry_release_kind(ret, *is_str); return NULL; } } @@ -785,9 +828,10 @@ static void *scr_stream_read_n(ScrStream *s, double size) { static void scr_stream_flow(ScrStream *s) { ScrStreamState *st = s->st; while (st->r.flowing == 1 && !scr_exc_pending()) { - void *b = scr_stream_read_n(s, -1); + bool is_str; + void *b = scr_stream_read_n_kind(s, -1, &is_str); if (b == NULL) break; - scr_stream_entry_release(st, b); + scr_stream_entry_release_kind(b, is_str); } } @@ -932,24 +976,70 @@ bool scr_stream_write_dyn(ScrStream *s, const ScrDyn *d, ScrClosure *cb) { /* Resolve the parked waiter with a taken chunk (moves), the EOF * sentinel, or dyn-boxed content per next_dyn. */ -static void scr_stream_next_fulfill(ScrStreamState *st, ScrPromise *w /*moves*/, void *chunk /*moves or NULL=EOF*/) { - if (st->r.next_dyn) { +/* Node's iterator is an async generator: `yield chunk` awaits the chunk, + * so next() settles one microtask after the chunk is read. The end (a + * bare return) and errors (a throw) settle without that await. caps[0] + * the waiter, caps[1] the payload (dyn or bytes, by caps[2]). */ +static void scr_stream_next_yield_job(ScrClosure *job) { + ScrPromise *w = scr_box_get_ref(job->caps[0]); + bool dyn = scr_box_get_bool(job->caps[2]); + void *payload = scr_box_get_ref(job->caps[1]); + if (dyn) scr_promise_fulfill_ref(w, payload, scr_dyn_retain_v, scr_dyn_release_v, NULL); + else scr_promise_fulfill_ref(w, payload, scr_bytes_retain_v, scr_bytes_release_v, NULL); + scr_promise_release(w); +} + +static void scr_stream_next_str_fence(void) { + static const char msg[] = + "for-await over a stream with an encoding set is not supported yet in " + "typed code (chunks are Buffers here)"; + scr_throw_error_msg(SCR_ERR_ERROR, msg, sizeof msg - 1); +} + +/* Fulfills the parked iterator with a chunk taken from the buffer; is_str + * is its kind as taken. The typed lane's chunk is a Buffer, so a string + * chunk rejects with the encoding fence instead. */ +static void scr_stream_next_fulfill(ScrStreamState *st, ScrPromise *w /*moves*/, + void *chunk /*moves or NULL=EOF*/, bool is_str) { + void *payload; + bool dyn = st->r.next_dyn; + if (!dyn && chunk != NULL && is_str) { + scr_str_release((ScrStr *)chunk); + scr_stream_next_str_fence(); + scr_promise_reject_pending(w); + scr_promise_release(w); + return; + } + if (dyn) { ScrDyn *d; if (chunk == NULL) { d = scr_dyn_retain(scr_dyn_undefined()); /* the EOF sentinel */ - } else if (st->r.encoded) { + } else if (is_str) { d = scr_dyn_new_str((ScrStr *)chunk); scr_str_release((ScrStr *)chunk); } else { d = scr_dyn_new_buffer((ScrBytes *)chunk); scr_bytes_release((ScrBytes *)chunk); } - scr_promise_fulfill_ref(w, d, scr_dyn_retain_v, scr_dyn_release_v, NULL); + payload = d; } else { - ScrBytes *b = chunk ? (ScrBytes *)chunk : scr_bytes_new(SCR_BYTES_U8, 0); - scr_promise_fulfill_ref(w, b, scr_bytes_retain_v, scr_bytes_release_v, NULL); + payload = chunk ? (ScrBytes *)chunk : scr_bytes_new(SCR_BYTES_U8, 0); } - scr_promise_release(w); + if (chunk == NULL) { + if (dyn) scr_promise_fulfill_ref(w, payload, scr_dyn_retain_v, scr_dyn_release_v, NULL); + else scr_promise_fulfill_ref(w, payload, scr_bytes_retain_v, scr_bytes_release_v, NULL); + scr_promise_release(w); + return; + } + ScrClosure *job = scr_closure_new((void *)&scr_stream_next_yield_job, 3); + job->caps[0] = scr_box_new_obj(scr_promise_retain_v, scr_promise_release_v, scr_promise_trace_v); + scr_box_set_ref(job->caps[0], w); /* moves */ + job->caps[1] = dyn ? scr_box_new_obj(scr_dyn_retain_v, scr_dyn_release_v, NULL) + : scr_box_new_obj(scr_bytes_retain_v, scr_bytes_release_v, NULL); + scr_box_set_ref(job->caps[1], payload); /* moves */ + job->caps[2] = scr_box_new(SCR_BOX_BOOL); + scr_box_set_bool(job->caps[2], dyn); + scr_queue_microtask(job); } /* Settle the parked waiter if the stream's state can answer now: an @@ -958,7 +1048,15 @@ static void scr_stream_next_fulfill(ScrStreamState *st, ScrPromise *w /*moves*/, * fulfills the sentinel. Called from push/eof/destroy transitions. */ static void scr_stream_settle_next(ScrStream *s) { ScrStreamState *st = s->st; - if (!st->r.next_waiter) return; + if (!st->r.next_waiter || st->r.next_reading) return; + if (st->errored && !st->error_emitted) { + /* Node's iterator learns of the error from its eos 'error' listener, + * after the listeners registered before it: the waiter rejects once + * the 'error' event is dispatched (SCR_ST_ERROR), and its listener + * keeps the error from being unhandled. */ + st->next_err_consumed = true; + return; + } if (st->errored) { ScrPromise *w = st->r.next_waiter; st->r.next_waiter = NULL; @@ -972,8 +1070,9 @@ static void scr_stream_settle_next(ScrStream *s) { if (st->r.length > 0) { ScrPromise *w = st->r.next_waiter; st->r.next_waiter = NULL; + bool is_str = st->r.encoded; void *chunk = scr_stream_rbuf_take(st, st->r.object_entries ? 1 : st->r.length); - scr_stream_next_fulfill(st, w, chunk); + scr_stream_next_fulfill(st, w, chunk, is_str); if (st->r.ended && st->r.length == 0) scr_stream_end_readable(s); return; } @@ -988,7 +1087,7 @@ static void scr_stream_settle_next(ScrStream *s) { } ScrPromise *w = st->r.next_waiter; st->r.next_waiter = NULL; - scr_stream_next_fulfill(st, w, NULL); + scr_stream_next_fulfill(st, w, NULL, false); } } @@ -997,7 +1096,7 @@ static ScrPromise *scr_stream_next_chunk_impl(ScrStream *s, bool dyn) { ScrPromise *p = scr_promise_new(); if (!st->has_r) { st->r.next_dyn = dyn; - scr_stream_next_fulfill(st, scr_promise_retain(p), NULL); + scr_stream_next_fulfill(st, scr_promise_retain(p), NULL, false); return p; } if (st->r.next_waiter) { @@ -1013,7 +1112,9 @@ static ScrPromise *scr_stream_next_chunk_impl(ScrStream *s, bool dyn) { scr_stream_settle_next(s); if (st->r.next_waiter && !st->r.reading && !st->r.ended && !st->destroyed) { st->r.need_readable = true; + st->r.next_reading = true; scr_stream_call_read(s); + st->r.next_reading = false; if (scr_exc_pending()) { ScrPromise *w = st->r.next_waiter; st->r.next_waiter = NULL; @@ -1032,10 +1133,7 @@ ScrPromise *scr_stream_next_chunk(ScrStream *s) { if (st->has_r && st->r.encoded) { /* the typed lane's chunk is a Buffer; string chunks would confuse */ ScrPromise *p = scr_promise_new(); - static const char msg[] = - "for-await over a stream with an encoding set is not supported yet in " - "typed code (chunks are Buffers here)"; - scr_throw_error_msg(SCR_ERR_ERROR, msg, sizeof msg - 1); + scr_stream_next_str_fence(); scr_promise_reject_pending(p); return p; } @@ -1046,25 +1144,405 @@ ScrPromise *scr_stream_next_chunk_dyn(ScrStream *s) { return scr_stream_next_chunk_impl(s, true); } -/* ── Readable.from (array-seeded object-entry streams) ────────────────── */ +/* ── Readable.from (array, string and Buffer sources) ─────────────────── */ -/* +1 stream fully seeded from the array's elements (strings or Buffers — - * one WHOLE entry per element, Node's objectMode delivery; hwm 1) and - * already EOF'd. Borrows arr. */ -ScrStream *scr_stream_from_arr(ScrArr *arr, bool strings) { - ScrStream *s = scr_stream_alloc(&scr_readable_vt, "Readable", true, false, - 1, -1, true, true, true); +/* from.js over an array source: the iterator position and the `reading` + * guard. The typed lane holds a ScrArr of ScrStr / ScrBytes elements; the + * JS lane holds the checked-dynamic array. `single` is from.js's string / + * Buffer special case: each _read pushes the value, then EOF. */ +enum { SCR_FROM_HWM_FROM, SCR_FROM_HWM_DEFAULT, SCR_FROM_HWM_SET }; + +typedef struct { + int rc; + ScrArr *arr; /* +1 or NULL */ + ScrDyn *dyn; /* +1 or NULL */ + size_t i; + bool strings, single, reading; +} ScrFromSrc; + +static ScrFromSrc *scr_from_src_retain(ScrFromSrc *f) { + f->rc++; + return f; +} + +static void scr_from_src_release(ScrFromSrc *f) { + if (--f->rc > 0) return; + if (f->arr) scr_arr_release(f->arr); + if (f->dyn) scr_dyn_release(f->dyn); + free(f); +} + +static void *scr_from_src_retain_v(void *f) { return scr_from_src_retain(f); } +static void scr_from_src_release_v(void *f) { scr_from_src_release(f); } + +/* A Readable.from source whose values no longer fit the buffer's kind + * (strings beside Buffers): destroyed with the "not supported yet" Error. */ +static void scr_from_destroy_mixed(ScrStream *s) { + static const char m[] = "a Readable.from source that mixes strings and Buffers is not supported yet"; + ScrStr *msg = scr_str_new(m, sizeof m - 1); + ScrError *e = scr_error_new(SCR_ERR_ERROR, msg); + scr_str_release(msg); + scr_stream_release(scr_stream_destroy(s, e)); + scr_error_release(e); +} + +/* One typed element into the stream: a whole entry in object mode, a + * chunk in byte mode. Moves e. Returns push's answer. */ +static bool scr_from_push_typed(ScrStream *s, void *e, bool strings) { + if (s->st->r.object_entries) { + int pushed = scr_stream_add_entry(s, e, strings); + if (pushed < 0) scr_from_destroy_mixed(s); + return pushed > 0; + } + bool more = strings ? scr_stream_push_str(s, e) : scr_stream_push(s, e); + if (strings) scr_str_release(e); + else scr_bytes_release(e); + return more; +} + +static void scr_from_read_inv(ScrClosure *cb, ScrStream *s, double size) { + (void)size; + ScrFromSrc *f = scr_box_get_ref(cb->caps[0]); + if (f->single) { + scr_from_push_typed(s, scr_arr_get_ref(f->arr, 0), f->strings); + if (!scr_exc_pending()) scr_stream_push_null(s); + } else if (!f->reading) { + /* nextSyncWithSyncValues: push while push() answers true */ + f->reading = true; + for (;;) { + if (f->i >= f->arr->len) { + scr_stream_push_null(s); + break; + } + void *e = scr_arr_get_ref(f->arr, (double)f->i++); + bool more = scr_from_push_typed(s, e, f->strings); + if (scr_exc_pending()) break; + if (!more) { + f->reading = false; + break; + } + } + } + scr_from_src_release(f); +} + +/* readable._destroy: close() is an async function with nothing to await + * for an array iterator (no return/throw methods), so the callback runs + * after its promise reaction and then a nextTick. */ +typedef struct { + int rc; + ScrStream *s; /* +1 */ + ScrError *error; /* +1 or NULL */ +} ScrFromDestroy; + +static void *scr_from_destroy_retain_v(void *p) { + ((ScrFromDestroy *)p)->rc++; + return p; +} + +static void scr_from_destroy_release_v(void *p) { + ScrFromDestroy *d = p; + if (--d->rc > 0) return; + scr_stream_release(d->s); + if (d->error) scr_error_release(d->error); + free(d); +} + +static ScrClosure *scr_from_destroy_closure(void *fn, ScrFromDestroy *d) { + ScrClosure *job = scr_closure_new(fn, 1); + job->caps[0] = scr_box_new_obj(&scr_from_destroy_retain_v, &scr_from_destroy_release_v, NULL); + scr_box_set_ref(job->caps[0], scr_from_destroy_retain_v(d)); + return job; +} + +static void scr_from_destroy_tick(ScrClosure *job) { + ScrFromDestroy *d = scr_box_get_ref(job->caps[0]); + scr_stream_destroy_done(d->s, d->error ? scr_error_retain(d->error) : NULL); + scr_from_destroy_release_v(d); +} + +static void scr_from_destroy_job(ScrClosure *job) { + ScrFromDestroy *d = scr_box_get_ref(job->caps[0]); + scr_next_tick(scr_from_destroy_closure((void *)&scr_from_destroy_tick, d)); + scr_from_destroy_release_v(d); +} + +static void scr_from_destroy_inv(ScrClosure *cb, ScrStream *s, ScrError *err) { + (void)cb; + ScrFromDestroy *d = calloc(1, sizeof *d); + if (!d) scr_stream_oom(); + d->rc = 1; + d->s = scr_stream_retain(s); + d->error = err ? scr_error_retain(err) : NULL; + scr_queue_microtask(scr_from_destroy_closure((void *)&scr_from_destroy_job, d)); + scr_from_destroy_release_v(d); +} + +/* getHighWaterMark: a non-negative integer, or ERR_INVALID_ARG_VALUE + * (thrown; false on return). */ +static bool scr_from_check_hwm(double hwm) { + if (hwm >= 0 && hwm <= 9007199254740991.0 && hwm == (double)(int64_t)hwm) return true; + char num[40]; + char msg[96]; + num[scr_f64_to_str(hwm, num)] = 0; + int n = snprintf(msg, sizeof msg, "The property 'options.highWaterMark' is invalid. Received %s", num); + scr_throw_error_msg_code(SCR_ERR_TYPE, msg, (size_t)n, "ERR_INVALID_ARG_VALUE"); + return false; +} + +/* The stream from.js builds: object mode unless the options turn it off; + * highWaterMark 1 for iterables, the mode's default for the string / + * Buffer case or a nullish option, or the option's value. +1, or NULL with the error + * pending. Moves f. */ +static ScrStream *scr_from_stream(ScrFromSrc *f, ScrStreamReadInv read_inv, bool object_mode, + int hwm_kind, double hwm, bool strings) { + if (hwm_kind == SCR_FROM_HWM_SET && !scr_from_check_hwm(hwm)) { + scr_from_src_release(f); + return NULL; + } + if (hwm_kind == SCR_FROM_HWM_FROM && !f->single) hwm = 1; + else if (hwm_kind != SCR_FROM_HWM_SET) hwm = object_mode ? 16 : -1; + ScrClosure *read = scr_closure_new((void *)read_inv, 1); + read->caps[0] = scr_box_new_obj(scr_from_src_retain_v, scr_from_src_release_v, NULL); + scr_box_set_ref(read->caps[0], f); /* moves */ + ScrClosure *destroy = NULL; + if (!f->single) destroy = scr_closure_new((void *)&scr_from_destroy_inv, 0); + ScrStream *s = scr_stream_new_readable(hwm, true, true, read, read_inv, destroy, + destroy ? &scr_from_destroy_inv : NULL); ScrStreamState *st = s->st; - st->r.object_entries = true; - st->r.encoded = strings; - if (strings) st->r.enc = scr_str_new("utf8", 4); - for (size_t i = 0; i < arr->len; i++) { - scr_stream_rbuf_push(st, scr_arr_get_ref(arr, (double)i), false); + if (object_mode) { + st->r.object_entries = true; + st->r.object_kind_set = strings || f->arr != NULL; + st->r.encoded = strings; + if (strings) st->r.enc = scr_str_new("utf8", 4); } - scr_stream_push_null(s); return s; } +ScrStream *scr_stream_from_arr(ScrArr *arr, double flags, double hwm) { + int bits = (int)flags; + ScrFromSrc *f = calloc(1, sizeof *f); + if (!f) scr_stream_oom(); + f->rc = 1; + f->arr = scr_arr_retain(arr); + f->strings = (bits & SCR_FROM_STRINGS) != 0; + f->single = (bits & SCR_FROM_SINGLE) != 0; + return scr_from_stream(f, &scr_from_read_inv, (bits & SCR_FROM_BYTE_MODE) == 0, + (bits & SCR_FROM_HAS_HWM) ? SCR_FROM_HWM_SET : SCR_FROM_HWM_FROM, hwm, + f->strings); +} + +/* The JS lane: one checked-dynamic value into the stream. null is + * ERR_STREAM_NULL_VALUES; in object mode, values other than strings and + * Buffers (or a mix of the two) are outside the stream model; byte mode + * rejects them with push()'s ERR_INVALID_ARG_TYPE. Each failure destroys + * the stream and answers -1. Borrows v. */ +static int scr_from_push_dyn(ScrStream *s, const ScrDyn *v) { + ScrError *e = NULL; + if (v->kind == SCR_DYN_NULL) { + static const char m[] = "May not write null values to stream"; + ScrStr *msg = scr_str_new(m, sizeof m - 1); + e = scr_error_new(SCR_ERR_TYPE, msg); + scr_str_release(msg); + scr_error_set_code(e, "ERR_STREAM_NULL_VALUES"); + } else if (s->st->r.object_entries) { + int pushed = scr_stream_push_object(s, v); + if (pushed >= 0) return pushed; + static const char mixed[] = "a Readable.from source that mixes strings and Buffers is not supported yet"; + static const char other[] = "Readable.from over values other than strings and Buffers is not supported yet"; + const char *m = pushed == -2 ? mixed : other; + ScrStr *msg = scr_str_new(m, strlen(m)); + e = scr_error_new(SCR_ERR_ERROR, msg); + scr_str_release(msg); + } else if (v->kind == SCR_DYN_STR || v->kind == SCR_DYN_BYTES) { + return scr_stream_push_dyn(s, v) ? 1 : 0; + } else { + char detail[256]; + char m[400]; + int n = snprintf(m, sizeof m, + "The \"chunk\" argument must be of type string or an instance of Buffer, " + "TypedArray, or DataView. Received %s", + scr_dyn_specific_type(v, detail, sizeof detail)); + ScrStr *msg = scr_str_new(m, (size_t)n); + e = scr_error_new(SCR_ERR_TYPE, msg); + scr_str_release(msg); + scr_error_set_code(e, "ERR_INVALID_ARG_TYPE"); + } + scr_stream_release(scr_stream_destroy(s, e)); + scr_error_release(e); + return -1; +} + +static void scr_from_read_dyn_inv(ScrClosure *cb, ScrStream *s, double size) { + (void)size; + ScrFromSrc *f = scr_box_get_ref(cb->caps[0]); + if (f->single) { + if (scr_from_push_dyn(s, f->dyn) >= 0 && !scr_exc_pending()) scr_stream_push_null(s); + } else if (!f->reading) { + f->reading = true; + for (;;) { + if (f->i >= f->dyn->v.arr.len) { + scr_stream_push_null(s); + break; + } + ScrDyn *v = scr_dyn_arr_at(f->dyn, (double)f->i++); + if (!v) break; + if (v->kind == SCR_DYN_NULL) f->reading = false; + int pushed = scr_from_push_dyn(s, v); + scr_dyn_release(v); + if (pushed < 0 || scr_exc_pending()) break; + if (!pushed) { + f->reading = false; + break; + } + } + } + scr_from_src_release(f); +} + +ScrStream *scr_stream_from_dyn(ScrDyn *src, ScrDyn *opts) { + bool single = src->kind == SCR_DYN_STR || src->kind == SCR_DYN_BYTES; + if (!single) { + int kind = scr_dyn_iterable_kind(src); + if (kind < 0) return NULL; + if (kind == 0) { + scr_dyn_arg_type_fail("iterable", "an instance of Iterable", src); + return NULL; + } + if (src->kind != SCR_DYN_ARR) { + static const char msg[] = + "Readable.from over an iterable other than an array, string, or Buffer is not supported yet"; + scr_throw_error_msg(SCR_ERR_ERROR, msg, sizeof msg - 1); + return NULL; + } + } + bool object_mode = true; + int hwm_kind = SCR_FROM_HWM_FROM; + double hwm = 0; + if (opts && opts->kind == SCR_DYN_OBJ) { + /* new Readable({ objectMode: true, highWaterMark: 1, ...opts }) */ + ScrDyn *keys = scr_dyn_own_keys(opts); + if (!keys) return NULL; + for (size_t i = 0; i < keys->v.arr.len; i++) { + ScrDyn *k = keys->v.arr.items[i]; + if (k->kind != SCR_DYN_STR) continue; + ScrStr *name = k->v.str; + if ((name->len == 10 && memcmp(name->data, "objectMode", 10) == 0) || + (name->len == 13 && memcmp(name->data, "highWaterMark", 13) == 0)) { + continue; + } + char m[200]; + int n = snprintf(m, sizeof m, "the Readable.from option '%.*s' is not supported yet", + (int)(name->len > 64 ? 64 : name->len), name->data); + scr_dyn_release(keys); + scr_throw_error_msg(SCR_ERR_ERROR, m, (size_t)n); + return NULL; + } + scr_dyn_release(keys); + ScrDyn *om = scr_dyn_obj_get(opts, "objectMode", 10); + if (om) object_mode = scr_dyn_truthy(om); + ScrDyn *h = scr_dyn_obj_get(opts, "highWaterMark", 13); + if (h && (h->kind == SCR_DYN_UNDEF || h->kind == SCR_DYN_NULL)) { + hwm_kind = SCR_FROM_HWM_DEFAULT; + } else if (h) { + double v = h->kind == SCR_DYN_NUM ? h->v.num : -1; + if (!(v >= 0 && v <= 9007199254740991.0 && v == (double)(int64_t)v)) { + scr_dyn_arg_value_fail("options.highWaterMark", NULL, h); + return NULL; + } + hwm_kind = SCR_FROM_HWM_SET; + hwm = v; + } + } + ScrFromSrc *f = calloc(1, sizeof *f); + if (!f) scr_stream_oom(); + f->rc = 1; + f->dyn = scr_dyn_retain(src); + f->single = single; + return scr_from_stream(f, &scr_from_read_dyn_inv, object_mode, hwm_kind, hwm, false); +} + +/* ── object-mode streams for the stream operators (scr_stream_ops.c) ──── */ + +/* Readable.from's stream (objectMode, highWaterMark 1) around native read + * and destroy callbacks. +1; the callbacks move. */ +ScrStream *scr_stream_new_object_readable(ScrClosure *read, ScrStreamReadInv read_inv, + ScrClosure *destroy, ScrStreamErrInv destroy_inv) { + ScrStream *s = scr_stream_new_readable(1, true, true, read, read_inv, destroy, destroy_inv); + s->st->r.object_entries = true; + return s; +} + +/* An object-mode entry in the buffer's representation: a Buffer on a + * stream with a decoder decodes through it (Node's decoder.write in + * readableAddChunkPushObjectMode); an entry of the other kind otherwise + * mixes strings and Buffers, which the buffer cannot hold. Moves entry; + * +1, or NULL (entry released). */ +static void *scr_stream_entry_convert(ScrStreamState *st, void *entry, bool is_str) { + if (is_str == st->r.encoded) return entry; + if (!is_str && st->r.decoding) { + ScrBytes *b = entry; + ScrStr *dec = scr_strdec_write(st->r.enc, st->r.dec_pending, b); + st->r.dec_pending = scr_strdec_next(st->r.enc, st->r.dec_pending, b); + scr_bytes_release(b); + return dec; + } + if (is_str) scr_str_release(entry); + else scr_bytes_release(entry); + return NULL; +} + +/* readableAddChunk in object mode: every value is one whole entry, empty + * strings and Buffers included. is_str gives entry's kind. Answers push()'s + * result, or -2 (entry released) when the kinds cannot mix. */ +static int scr_stream_add_entry(ScrStream *s, void *entry /*moves*/, bool is_str) { + ScrStreamState *st = s->st; + if (!st->r.object_kind_set && st->r.length == 0) { + /* the first entry picks the representation */ + st->r.object_kind_set = true; + if (is_str && !st->r.encoded) { + st->r.encoded = true; + if (!st->r.enc) st->r.enc = scr_str_new("utf8", 4); + } + } + entry = scr_stream_entry_convert(st, entry, is_str); + if (entry == NULL) return -2; + if (st->r.ended) { + scr_stream_entry_release(st, entry); + ScrError *e = scr_stream_mkerr("ERR_STREAM_PUSH_AFTER_EOF", "stream.push() after EOF"); + scr_stream_error_or_destroy(s, e); + scr_error_release(e); + return 0; + } + if (st->destroyed || st->errored) { + scr_stream_entry_release(st, entry); + return 0; + } + st->r.reading = false; + if (st->r.flowing == 1 && st->r.length == 0 && !st->r.in_read_sync && + (scr_emitter_has((ScrEmitter *)s, "data") || st->pipes.n > 0)) { + if (st->r.emitted_readable) st->r.emitted_readable = false; + bool is_str = st->r.encoded; + scr_stream_emit_data(s, entry); + scr_stream_entry_release_kind(entry, is_str); + if (scr_exc_pending()) return 0; + } else { + scr_stream_rbuf_push(st, entry, st->r.encoded, false); + if (st->r.need_readable) scr_stream_emit_readable_nt(s); + } + scr_stream_maybe_read_more(s); + scr_stream_settle_next(s); + return !st->r.ended && (st->r.length < st->r.hwm || st->r.length == 0); +} + +int scr_stream_push_object(ScrStream *s, const ScrDyn *v) { + bool is_str = v->kind == SCR_DYN_STR; + if (!is_str && v->kind != SCR_DYN_BYTES) return -1; + void *entry = is_str ? (void *)scr_str_retain(v->v.str) : (void *)scr_bytes_raw_view(v->v.bytes); + int pushed = scr_stream_add_entry(s, entry, is_str); + return pushed < 0 ? pushed : pushed ? 1 : 0; +} + /* ── the registration hook ('data'/'readable' listeners kick flow) ────── */ static void scr_stream_on_listener(ScrEmitter *em, ScrStr *name) { @@ -1198,7 +1676,9 @@ static void scr_stream_clear_buffer(ScrStream *s) { static void scr_stream_finish_maybe(ScrStream *s) { ScrStreamState *st = s->st; if (!st->w.ending || st->w.writing || st->w.head < st->w.n || st->w.finished || - st->destroyed || st->w.finish_scheduled) { + st->destroyed || st->w.finish_scheduled || st->w.final_pending || st->flush_pending) { + /* a pending _final/_flush callback holds the finish (Node's + * pendingcb) */ return; } if (!st->w.prefinished && st->flush_on_prefinish) { @@ -1217,6 +1697,7 @@ static void scr_stream_finish_maybe(ScrStream *s) { if ((st->is_transform || st->passthrough) && !st->flush_called) { st->flush_called = true; if (st->flush_cb) { + st->flush_pending = true; st->flush_inv(st->flush_cb, s); return; /* flush_done resumes the finish path */ } @@ -1226,6 +1707,7 @@ static void scr_stream_finish_maybe(ScrStream *s) { } if (st->w.final_cb && !st->w.final_called) { st->w.final_called = true; + st->w.final_pending = true; st->w.final_inv(st->w.final_cb, s); return; /* final_done resumes the finish path */ } @@ -2036,11 +2518,9 @@ void scr_stream_init_passthrough_dyn(ScrStream *s, ScrDyn *opts, * ERR_STREAM_PREMATURE_CLOSE ("Premature close") when the stream * closed before end/finish. finished returns a CLEANUP function that * unhooks the callback. - * - pipeline(a, b, c, cb) on a mid-stream error: the erroring stream - * destroys itself and emits 'error'+'close'; the pipeline then - * destroys every OTHER stream with the SAME error (pipeline order), - * each emitting 'error' then 'close'; cb(err) runs after the last - * 'close'. Success: cb(null) after the destination's 'close'. + * - pipeline(a, b, c, cb) follows pipeline.js (see scr_stream_pipeline_ex): + * cb runs on a nextTick once every stream's eos watcher finished, and + * the first error destroys the streams that have not finished. * - both register 'error' handlers, so an otherwise-unhandled stream * error no longer crashes (the SCR_ST_ERROR suppression above). */ @@ -2139,18 +2619,46 @@ void scr_stream_finished_dyn_inv(ScrClosure *cb, ScrStream *s, ScrError *err) { if (r != NULL) scr_dyn_release(r); } +static ScrError *scr_stream_abort_error(ScrDyn *signal); +static ScrDyn *scr_stream_signal_prop(ScrDyn *signal, const char *key); +static void *scr_stream_dyn_retain_v(void *p); +static void scr_stream_dyn_release_v(void *p); +static ScrClosure *scr_stream_eos_impl(ScrStream *s, ScrClosure *cb, ScrStreamErrInv inv, + ScrPromise *p, bool auto_cleanup, int readable, + int writable, bool on_error, ScrDyn *signal, + ScrAbortAddFn add); + +/* stream.pipeline over streams (lib/internal/streams/pipeline.js). Each + * stream gets the destroyer (a 'close' listener and an eos watcher over + * the sides it plays), an 'error' listener, and each adjacent pair the + * pipe() helper: src.pipe(dst, { end: false }), dst.end() on the source's + * 'end', a premature-close check on the destination's 'close', and eos + * watchers on both. The callback runs on a nextTick once every watcher + * finished, or the first error destroys the remaining streams. + * + * The context's listeners on the streams hold it, and it holds the + * streams until the final callback (or exit), which breaks the cycle. */ + typedef struct { size_t rc; - size_t n, closed; - size_t watched; /* stages with a watcher: all, or all but the last (end: false) */ - ScrStream **streams; /* owned */ - ScrClosure *cb; /* owned; NULL once fired */ + size_t n; + ScrStream **streams; /* +1 each; NULL after the final callback */ + bool *destroyer; /* stream i has a destroyer (destroys queue) */ + bool *finished; /* the destroyer's finished flag */ + bool *ended; /* pipe(streams[i-1], streams[i])'s ended flag */ + size_t destroy_head; /* the next destroys entry to run */ + long finish_count; + ScrError *error; /* +1 or NULL */ + ScrClosure *cb; /* +1 until the final callback is queued */ ScrStreamErrInv inv; - ScrError *err; /* first error, owned */ - ScrDyn *dispose; /* the signal listener's disposable (+1), or NULL */ - /* contexts holding a disposable: it holds the abort listener, which - * holds the context, so exit releases what a pipeline that never - * finished still holds */ + bool final_done; + ScrDyn *signal; /* +1 or NULL */ + ScrDyn *dispose; /* the abort listener's disposable (+1) or NULL */ + /* lastStreamCleanup: the last stream's eos cleanups and its 'error' + * listener, when that stream is readable */ + ScrClosure *last_cleanup[2]; + size_t last_cleanups; + ScrClosure *last_error_l; /* +1 */ void *live_prev, *live_next; } ScrStreamPipeCtx; @@ -2164,70 +2672,388 @@ static void scr_pipe_unlink(ScrStreamPipeCtx *ctx) { ctx->live_prev = ctx->live_next = NULL; } -static void scr_pipe_atexit(void) { - while (scr_pipe_live) { - ScrStreamPipeCtx *ctx = scr_pipe_live; - scr_pipe_unlink(ctx); - if (ctx->dispose) { - ScrDyn *d = ctx->dispose; - ctx->dispose = NULL; - scr_dyn_release(d); - } - } -} - static void *scr_stream_pipectx_retain_v(void *p) { ((ScrStreamPipeCtx *)p)->rc++; return p; } +/* What the context holds besides its flags: the streams, the pending + * callback, the signal's disposable and the cleanups. */ +static void scr_stream_pipectx_drop(ScrStreamPipeCtx *ctx) { + scr_pipe_unlink(ctx); + if (ctx->streams) { + for (size_t i = 0; i < ctx->n; i++) scr_stream_release(ctx->streams[i]); + free(ctx->streams); + ctx->streams = NULL; + } + if (ctx->cb) { + scr_closure_release(ctx->cb); + ctx->cb = NULL; + } + if (ctx->dispose) { + ScrDyn *d = ctx->dispose; + ctx->dispose = NULL; + scr_dyn_release(d); + } + if (ctx->signal) { + scr_dyn_release(ctx->signal); + ctx->signal = NULL; + } + for (size_t i = 0; i < ctx->last_cleanups; i++) scr_closure_release(ctx->last_cleanup[i]); + ctx->last_cleanups = 0; + if (ctx->last_error_l) { + scr_closure_release(ctx->last_error_l); + ctx->last_error_l = NULL; + } +} + static void scr_stream_pipectx_release_v(void *p) { ScrStreamPipeCtx *ctx = p; if (--ctx->rc > 0) return; - for (size_t i = 0; i < ctx->n; i++) scr_stream_release(ctx->streams[i]); - free(ctx->streams); - if (ctx->cb) scr_closure_release(ctx->cb); - if (ctx->err) scr_error_release(ctx->err); - if (ctx->dispose) scr_dyn_release(ctx->dispose); - scr_pipe_unlink(ctx); + scr_stream_pipectx_drop(ctx); + if (ctx->error) scr_error_release(ctx->error); + free(ctx->destroyer); + free(ctx->finished); + free(ctx->ended); free(ctx); } -static void scr_stream_pipectx_dispose(ScrStreamPipeCtx *ctx) { - if (!ctx->dispose) return; - scr_pipe_unlink(ctx); - ScrDyn *d = ctx->dispose; - ctx->dispose = NULL; - ScrDyn *r = scr_dyn_call(d, NULL, 0, "dispose"); - if (r) scr_dyn_release(r); - scr_dyn_release(d); +static void scr_pipe_atexit(void) { + while (scr_pipe_live) { + ScrStreamPipeCtx *ctx = scr_stream_pipectx_retain_v(scr_pipe_live); + scr_stream_pipectx_drop(ctx); + scr_stream_pipectx_release_v(ctx); + } } -static void scr_stream_pipeline_watch_inv(ScrClosure *cb, ScrStream *s, ScrError *err) { - ScrStreamPipeCtx *ctx = scr_box_get_ref(cb->caps[0]); /* +1 */ - ctx->closed++; - if (err != NULL && ctx->err == NULL) { - ctx->err = scr_error_retain(err); - scr_stream_pipectx_dispose(ctx); - /* Node's pipeline destroyer: every other stream goes down with the - * SAME error, pipeline order (each emits 'error' then 'close'). */ - for (size_t i = 0; i < ctx->watched; i++) { - if (ctx->streams[i] != s) { - ScrStream *chain = scr_stream_destroy(ctx->streams[i], ctx->err); - scr_stream_release(chain); +/* caps[0] the context, caps[1] the stream index. */ +static ScrClosure *scr_pipe_closure(void *fn, ScrStreamPipeCtx *ctx, size_t i) { + ScrClosure *c = scr_closure_new(fn, 2); + c->caps[0] = scr_box_new_obj(&scr_stream_pipectx_retain_v, &scr_stream_pipectx_release_v, NULL); + scr_box_set_ref(c->caps[0], scr_stream_pipectx_retain_v(ctx)); + c->caps[1] = scr_box_new(SCR_BOX_F64); + scr_box_set_f64(c->caps[1], (double)i); + return c; +} + +static ScrStreamPipeCtx *scr_pipe_of(ScrClosure *c, size_t *i) { + if (i) *i = (size_t)scr_box_get_f64(c->caps[1]); + return scr_box_get_ref(c->caps[0]); /* +1 */ +} + +static void scr_pipe_on(ScrStream *s, const char *name, ScrClosure *cb /*moves*/, ScrEeInvoke inv, + bool once) { + ScrStr *n = scr_str_new(name, strlen(name)); + scr_emitter_release(scr_emitter_on((ScrEmitter *)s, n, cb, inv, once, false)); + scr_str_release(n); +} + +static bool scr_pipe_err_is(const ScrError *e, const char *code, const char *name) { + if (code && e->code && e->code->len == strlen(code) && memcmp(e->code->data, code, e->code->len) == 0) + return true; + return name && e->name && e->name->len == strlen(name) && + memcmp(e->name->data, name, e->name->len) == 0; +} + +/* process.nextTick(callback, error): caps[0] the context, caps[1] the + * callback, caps[2] the last stream, caps[3] the error as it was when + * the tick was queued (an Error or NULL). */ +static void scr_pipe_callback_tick(ScrClosure *job) { + ScrStreamPipeCtx *ctx = scr_box_get_ref(job->caps[0]); + ScrClosure *cb = scr_box_get_ref(job->caps[1]); + ScrStream *last = scr_box_get_ref(job->caps[2]); + ScrError *err = job->ncaps > 3 ? scr_box_get_ref(job->caps[3]) : NULL; + ctx->inv(cb, last, err); + if (err) scr_error_release(err); + scr_closure_release(cb); + scr_stream_release(last); + scr_stream_pipectx_release_v(ctx); +} + +/* destroyImpl.destroyer(stream, err || new ERR_STREAM_DESTROYED('pipe')) */ +static void scr_pipe_destroy(ScrStream *s, ScrError *err) { + if (s->st->destroyed) return; + ScrError *e = err ? scr_error_retain(err) + : scr_stream_mkerr("ERR_STREAM_DESTROYED", "Cannot call pipe after a stream was destroyed"); + scr_stream_release(scr_stream_destroy(s, e)); + scr_error_release(e); +} + +static void scr_pipe_finish_impl(ScrStreamPipeCtx *ctx, ScrError *err, bool final) { + if (ctx->final_done) final = false; + if (err && (!ctx->error || scr_pipe_err_is(ctx->error, "ERR_STREAM_PREMATURE_CLOSE", "AbortError"))) { + if (ctx->error) scr_error_release(ctx->error); + ctx->error = scr_error_retain(err); + } + if (!ctx->error && !final) return; + if (!ctx->streams) return; + scr_stream_pipectx_retain_v(ctx); + while (ctx->destroy_head < ctx->n) { + size_t i = ctx->destroy_head++; + if (!ctx->destroyer[i] || ctx->finished[i]) continue; + ctx->finished[i] = true; + scr_pipe_destroy(ctx->streams[i], ctx->error); + if (!ctx->streams) break; + } + if (ctx->dispose) { + ScrDyn *d = ctx->dispose; + ctx->dispose = NULL; + ScrDyn *r = scr_dyn_call(d, NULL, 0, "dispose"); + if (r) scr_dyn_release(r); + scr_dyn_release(d); + } + if (final && ctx->streams) { + ctx->final_done = true; + if (!ctx->error) { + for (size_t i = 0; i < ctx->last_cleanups; i++) { + ScrClosure *c = ctx->last_cleanup[i]; + ((void (*)(ScrClosure *))c->fn)(c); + } + if (ctx->last_error_l) { + ScrStr *n = scr_str_new("error", 5); + scr_emitter_release(scr_emitter_off((ScrEmitter *)ctx->streams[ctx->n - 1], n, ctx->last_error_l)); + scr_str_release(n); } } - } - if (ctx->closed == ctx->watched && ctx->cb != NULL) { - scr_stream_pipectx_dispose(ctx); - ScrClosure *ucb = ctx->cb; + /* process.nextTick(callback, error, value) */ + ScrClosure *job = scr_closure_new((void *)&scr_pipe_callback_tick, ctx->error ? 4 : 3); + job->caps[0] = scr_box_new_obj(&scr_stream_pipectx_retain_v, &scr_stream_pipectx_release_v, NULL); + scr_box_set_ref(job->caps[0], scr_stream_pipectx_retain_v(ctx)); + job->caps[1] = scr_box_new_obj(&scr_closure_retain_v, &scr_closure_release_v, &scr_closure_trace_v); + scr_box_set_ref(job->caps[1], ctx->cb); /* moves */ ctx->cb = NULL; - ctx->inv(ucb, ctx->streams[ctx->n - 1], ctx->err); - scr_closure_release(ucb); + job->caps[2] = scr_box_new_obj(&scr_stream_retain_v, &scr_stream_release_v, &scr_stream_trace); + scr_box_set_ref(job->caps[2], scr_stream_retain(ctx->streams[ctx->n - 1])); + if (ctx->error) { + job->caps[3] = scr_box_new_obj(&scr_error_retain_v, &scr_error_release_v, NULL); + scr_box_set_ref(job->caps[3], scr_error_retain(ctx->error)); + } + scr_next_tick(job); + scr_stream_pipectx_drop(ctx); } scr_stream_pipectx_release_v(ctx); } +/* finish(err): finishImpl(err, --finishCount === 0) */ +static void scr_pipe_finish_inv(ScrClosure *cb, ScrStream *s, ScrError *err) { + (void)s; + ScrStreamPipeCtx *ctx = scr_pipe_of(cb, NULL); + scr_pipe_finish_impl(ctx, err, --ctx->finish_count == 0); + scr_stream_pipectx_release_v(ctx); +} + +static void scr_pipe_finish_l(ScrClosure *cb) { scr_pipe_finish_inv(cb, NULL, NULL); } + +static void scr_pipe_finish_err_l(ScrClosure *cb, void *err) { scr_pipe_finish_inv(cb, NULL, err); } + +/* destroyer: stream.on('close') marks finished; eos marks it !err. */ +static void scr_pipe_destroyer_close_l(ScrClosure *cb) { + size_t i; + ScrStreamPipeCtx *ctx = scr_pipe_of(cb, &i); + ctx->finished[i] = true; + scr_stream_pipectx_release_v(ctx); +} + +static void scr_pipe_destroyer_eos(ScrClosure *cb, ScrStream *s, ScrError *err) { + (void)s; + size_t i; + ScrStreamPipeCtx *ctx = scr_pipe_of(cb, &i); + ctx->finished[i] = err == NULL; + scr_stream_pipectx_release_v(ctx); +} + +/* onError: errors after pipe completed (not AbortError / premature close) */ +static void scr_pipe_error_l(ScrClosure *cb, void *errp) { + ScrError *err = errp; + ScrStreamPipeCtx *ctx = scr_pipe_of(cb, NULL); + if (err && !scr_pipe_err_is(err, "ERR_STREAM_PREMATURE_CLOSE", "AbortError")) { + scr_pipe_finish_impl(ctx, err, false); + } + scr_stream_pipectx_release_v(ctx); +} + +/* pipe(): the destination closed before the source ended */ +static void scr_pipe_dst_close_l(ScrClosure *cb) { + size_t i; + ScrStreamPipeCtx *ctx = scr_pipe_of(cb, &i); + if (!ctx->ended[i]) { + ScrError *e = scr_stream_mkerr("ERR_STREAM_PREMATURE_CLOSE", "Premature close"); + scr_pipe_finish_impl(ctx, e, false); + scr_error_release(e); + } + scr_stream_pipectx_release_v(ctx); +} + +/* pipe()'s endFn: ended = true; dst.end(). caps[1] the destination's + * index. */ +static void scr_pipe_end_fn(ScrClosure *cb) { + size_t i; + ScrStreamPipeCtx *ctx = scr_pipe_of(cb, &i); + ctx->ended[i] = true; + if (ctx->streams) { + ScrStream *r = scr_stream_end(ctx->streams[i], NULL, NULL, NULL); + if (r) scr_stream_release(r); + } + scr_stream_pipectx_release_v(ctx); +} + +/* pipe()'s eos(src, { readable: true, writable: false }): a premature + * close after the source already ended (no error) waits for 'end' or + * 'error' instead. caps[1] the source's index. */ +static void scr_pipe_src_eos(ScrClosure *cb, ScrStream *s, ScrError *err) { + size_t i; + ScrStreamPipeCtx *ctx = scr_pipe_of(cb, &i); + ScrStreamState *st = s->st; + if (err && scr_pipe_err_is(err, "ERR_STREAM_PREMATURE_CLOSE", NULL) && st->r.ended && + !st->errored && !st->error_emitted) { + scr_pipe_on(s, "end", scr_pipe_closure((void *)&scr_pipe_finish_l, ctx, i), scr_ee_inv_fixed0, true); + scr_pipe_on(s, "error", scr_pipe_closure((void *)&scr_pipe_finish_err_l, ctx, i), scr_ee_inv_fixed1, + true); + } else { + scr_pipe_finish_impl(ctx, err, --ctx->finish_count == 0); + } + scr_stream_pipectx_release_v(ctx); +} + +/* eos(stream, { readable, writable }, cb) with Node's listeners. */ +static ScrClosure *scr_pipe_eos(ScrStream *s, ScrClosure *cb, ScrStreamErrInv inv, bool readable, + bool writable) { + return scr_stream_eos_impl(s, cb, inv, NULL, false, readable, writable, true, NULL, NULL); +} + +static void scr_pipe_abort(ScrStreamPipeCtx *ctx) { + ScrError *e = scr_stream_abort_error(ctx->signal); + scr_pipe_finish_impl(ctx, e, false); + scr_error_release(e); +} + +static ScrDyn *scr_pipe_abort_thunk(ScrClosure *clo, ScrDyn *const *args, size_t argc) { + (void)args; + (void)argc; + ScrStreamPipeCtx *ctx = scr_pipe_of(clo, NULL); + if (ctx->signal) scr_pipe_abort(ctx); + scr_stream_pipectx_release_v(ctx); + return scr_dyn_retain(scr_dyn_undefined()); +} + +static void scr_pipe_abort_tick(ScrClosure *job) { + ScrStreamPipeCtx *ctx = scr_pipe_of(job, NULL); + if (ctx->signal) scr_pipe_abort(ctx); + scr_stream_pipectx_release_v(ctx); +} + +/* The context-free throws of pipelineImpl's loop. */ +static void scr_pipe_throw(const char *code, const char *msg, int kind) { + ScrStr *m = scr_str_new(msg, strlen(msg)); + ScrError *e = scr_error_new(kind, m); + scr_str_release(m); + scr_error_set_code(e, code); + scr_throw_obj(e, &scr_error_retain_v, &scr_error_release_v, scr_error_trace_arg()); +} + +static ScrStream *scr_stream_pipeline_ex(size_t n, ScrStream **streams, ScrClosure *cb, + ScrStreamErrInv inv, bool end_opt, ScrDyn *signal, + ScrAbortAddFn add) { + ScrStreamPipeCtx *ctx = calloc(1, sizeof *ctx); + if (!ctx) scr_stream_oom(); + ctx->rc = 1; + ctx->n = n; + ctx->streams = calloc(n, sizeof *ctx->streams); + ctx->destroyer = calloc(n, sizeof *ctx->destroyer); + ctx->finished = calloc(n, sizeof *ctx->finished); + ctx->ended = calloc(n, sizeof *ctx->ended); + if (!ctx->streams || !ctx->destroyer || !ctx->finished || !ctx->ended) scr_stream_oom(); + for (size_t i = 0; i < n; i++) ctx->streams[i] = scr_stream_retain(streams[i]); + ctx->cb = cb; + ctx->inv = inv; + static SCR_TL bool registered = false; + if (!registered) { + registered = true; + scr_atexit(scr_pipe_atexit); + } + ctx->live_next = scr_pipe_live; + if (scr_pipe_live) scr_pipe_live->live_prev = ctx; + scr_pipe_live = ctx; + bool aborted = false; + if (signal) { + ctx->signal = scr_dyn_retain(signal); + ScrDyn *a = scr_stream_signal_prop(signal, "aborted"); + aborted = a && scr_dyn_truthy(a); + if (a) scr_dyn_release(a); + ScrClosure *clo = scr_pipe_closure((void *)&scr_pipe_abort_thunk, ctx, 0); + ScrDyn *listener = scr_dyn_new_func(clo, &scr_pipe_abort_thunk, 0, "()", "abort"); + ctx->dispose = add(signal, listener); + scr_dyn_release(listener); + } + ScrStream *ret = NULL; + for (size_t i = 0; i < n; i++) { + ScrStream *s = streams[i]; + bool reading = i + 1 < n; + bool writing = i > 0; + bool end = reading || end_opt; + bool last = i + 1 == n; + if (reading && (streams[i + 1]->st->close_emitted || streams[i + 1]->st->destroyed)) { + scr_pipe_throw("ERR_STREAM_UNABLE_TO_PIPE", "Cannot pipe to a closed or destroyed stream", + SCR_ERR_ERROR); + scr_stream_pipectx_drop(ctx); + scr_stream_pipectx_release_v(ctx); + return NULL; + } + bool last_readable = last && scr_stream_prop(s, "readable"); + if (end) { + ctx->destroyer[i] = true; + scr_pipe_on(s, "close", scr_pipe_closure((void *)&scr_pipe_destroyer_close_l, ctx, i), + scr_ee_inv_fixed0, false); + ScrClosure *cleanup = scr_pipe_eos(s, scr_pipe_closure((void *)&scr_pipe_destroyer_eos, ctx, i), + &scr_pipe_destroyer_eos, reading, writing); + if (last_readable) ctx->last_cleanup[ctx->last_cleanups++] = cleanup; + else scr_closure_release(cleanup); + } + ScrClosure *on_error = scr_pipe_closure((void *)&scr_pipe_error_l, ctx, i); + scr_pipe_on(s, "error", scr_closure_retain(on_error), scr_ee_inv_fixed1, false); + if (last_readable) ctx->last_error_l = on_error; + else scr_closure_release(on_error); + if (i > 0) { + if (!ret->st->has_r) { + static const char msg[] = + "The \"val\" argument must be an instance of Readable, Iterable, AsyncIterable, " + "ReadableStream, or TransformStream. Received an instance of Writable"; + scr_pipe_throw("ERR_INVALID_ARG_TYPE", msg, SCR_ERR_TYPE); + scr_stream_pipectx_drop(ctx); + scr_stream_pipectx_release_v(ctx); + return NULL; + } + /* pipe(ret, s, finish, finishOnlyHandleError, { end }) */ + ctx->finish_count += 2; + scr_pipe_on(s, "close", scr_pipe_closure((void *)&scr_pipe_dst_close_l, ctx, i), scr_ee_inv_fixed0, + false); + scr_stream_release(scr_stream_pipe(ret, s, false)); + if (end) { + ScrClosure *end_fn = scr_pipe_closure((void *)&scr_pipe_end_fn, ctx, i); + if (ret->st->r.end_emitted) scr_next_tick(end_fn); + else scr_pipe_on(ret, "end", end_fn, scr_ee_inv_fixed0, true); + } else { + scr_pipe_finish_impl(ctx, NULL, --ctx->finish_count == 0); + } + scr_closure_release(scr_pipe_eos(ret, scr_pipe_closure((void *)&scr_pipe_src_eos, ctx, i - 1), + &scr_pipe_src_eos, true, false)); + ScrClosure *cleanup = scr_pipe_eos(s, scr_pipe_closure((void *)&scr_pipe_finish_inv, ctx, i), + &scr_pipe_finish_inv, false, true); + if (scr_stream_prop(s, "readable") && last) ctx->last_cleanup[ctx->last_cleanups++] = cleanup; + else scr_closure_release(cleanup); + } + ret = s; + } + if (aborted) scr_next_tick(scr_pipe_closure((void *)&scr_pipe_abort_tick, ctx, 0)); + scr_stream_pipectx_release_v(ctx); /* the listeners and the live list hold it */ + return scr_stream_retain(ret); +} + +ScrStream *scr_stream_pipeline(double n_d, ScrStream **streams /*borrowed*/, + ScrClosure *cb /*moves*/, ScrStreamErrInv inv) { + return scr_stream_pipeline_ex((size_t)n_d, streams, cb, inv, true, NULL, NULL); +} + /* The dyn-valued twins: the callback rides as a checked-dynamic value * (borrowed; the closure's cap box retains it). */ ScrClosure *scr_stream_finished_dyn(ScrStream *s, ScrDyn *cb) { @@ -2241,92 +3067,296 @@ ScrStream *scr_stream_pipeline_dyn(double n, ScrStream **streams, ScrDyn *cb) { return scr_stream_pipeline(n, streams, c, &scr_stream_finished_dyn_inv); } -static ScrError *scr_stream_abort_error(ScrDyn *signal); -static ScrDyn *scr_stream_signal_prop(ScrDyn *signal, const char *key); -static void *scr_stream_dyn_retain_v(void *p); -static void scr_stream_dyn_release_v(void *p); +/* ── stream.compose (compose.js over stream instances) ──────────────── + * The streams run as a pipeline; the answer is a Duplex whose writes go + * to the head and whose reads come from the tail. The context is shared + * by the Duplex's callbacks, the head/tail listeners, the tail's eos + * watcher and the pipeline callback. It holds the Duplex until the + * Duplex's destroy completes (or exit), which breaks the cycle through + * the Duplex's own callbacks. */ -/* pipeline's signal: an abort destroys every watched stage with - * new AbortError(undefined, { cause: signal.reason }) (finishImpl's - * destroyers); the callback follows the last close. caps[0] the context, - * caps[1] the signal. */ -static ScrDyn *scr_stream_pipeline_abort_thunk(ScrClosure *clo, ScrDyn *const *args, size_t argc) { - (void)args; - (void)argc; - ScrStreamPipeCtx *ctx = scr_box_get_ref(clo->caps[0]); /* +1 */ - ScrDyn *signal = scr_box_get_ref(clo->caps[1]); /* +1 */ - if (ctx->err == NULL && ctx->cb != NULL) { - ctx->err = scr_stream_abort_error(signal); - scr_stream_pipectx_dispose(ctx); - for (size_t i = 0; i < ctx->watched; i++) { - ScrStream *chain = scr_stream_destroy(ctx->streams[i], ctx->err); - scr_stream_release(chain); - } +enum { SCR_COMPOSE_RUNNING, SCR_COMPOSE_DONE, SCR_COMPOSE_WAIT }; + +typedef struct ScrCompose { + size_t rc; + ScrStream *head, *tail; /* +1 */ + ScrStream *d; /* +1 until teardown */ + ScrClosure *drain_l, *data_l, *end_l; /* +1, the listeners, or NULL */ + bool readable, writable; + bool ondrain, onfinish; + int onclose; /* compose.js's onclose: undefined, null, or d's destroy callback */ + struct ScrCompose *live_prev, *live_next; +} ScrCompose; + +static SCR_TL ScrCompose *scr_compose_live = NULL; + +static void *scr_compose_retain_v(void *p) { + ((ScrCompose *)p)->rc++; + return p; +} + +static void scr_compose_release_v(void *p) { + ScrCompose *c = p; + if (--c->rc > 0) return; + scr_stream_release(c->head); + scr_stream_release(c->tail); + free(c); +} + +static ScrClosure *scr_compose_closure(void *fn, ScrCompose *c) { + ScrClosure *cl = scr_closure_new(fn, 1); + cl->caps[0] = scr_box_new_obj(&scr_compose_retain_v, &scr_compose_release_v, NULL); + scr_box_set_ref(cl->caps[0], scr_compose_retain_v(c)); + return cl; +} + +static void scr_compose_off(ScrStream *s, const char *name, ScrClosure **l) { + if (!*l) return; + ScrStr *n = scr_str_new(name, strlen(name)); + scr_emitter_release(scr_emitter_off((ScrEmitter *)s, n, *l)); + scr_str_release(n); + scr_closure_release(*l); + *l = NULL; +} + +/* Drop the Duplex and the head/tail listeners. */ +static void scr_compose_teardown(ScrCompose *c) { + if (!c->d) return; + scr_compose_off(c->head, "drain", &c->drain_l); + scr_compose_off(c->tail, "data", &c->data_l); + scr_compose_off(c->tail, "end", &c->end_l); + if (c->live_prev) c->live_prev->live_next = c->live_next; + else scr_compose_live = c->live_next; + if (c->live_next) c->live_next->live_prev = c->live_prev; + c->live_prev = c->live_next = NULL; + ScrStream *d = c->d; + c->d = NULL; + scr_stream_release(d); + scr_compose_release_v(c); /* the live list's reference */ +} + +static void scr_compose_atexit(void) { + while (scr_compose_live) scr_compose_teardown(scr_compose_live); +} + +static void scr_compose_on(ScrStream *s, const char *name, ScrClosure **slot, void *fn, ScrCompose *c, + ScrEeInvoke inv) { + ScrStr *n = scr_str_new(name, strlen(name)); + *slot = scr_compose_closure(fn, c); + scr_emitter_release(scr_emitter_on((ScrEmitter *)s, n, scr_closure_retain(*slot), inv, false, false)); + scr_str_release(n); +} + +/* d's destroy callback: callback(err), then the context lets go. */ +static void scr_compose_destroyed(ScrCompose *c, ScrError *err) { + ScrStream *d = scr_stream_retain(c->d); + scr_compose_teardown(c); + scr_stream_destroy_done(d, err ? scr_error_retain(err) : NULL); + scr_stream_release(d); +} + +/* d._write: head.write(chunk) ? callback() : (ondrain = callback) */ +static void scr_compose_write_inv(ScrClosure *cb, ScrStream *s, ScrBytes *chunk) { + ScrCompose *c = scr_box_get_ref(cb->caps[0]); + if (scr_stream_write(c->head, chunk, NULL)) { + if (!scr_exc_pending()) scr_stream_write_done(s, NULL); + } else { + c->ondrain = true; } - scr_dyn_release(signal); - scr_stream_pipectx_release_v(ctx); - return scr_dyn_retain(scr_dyn_undefined()); + scr_compose_release_v(c); } -static ScrStream *scr_stream_pipeline_ex(size_t n, ScrStream **streams, ScrClosure *cb, - ScrStreamErrInv inv, bool end, ScrDyn *signal, - ScrAbortAddFn add); - -ScrStream *scr_stream_pipeline(double n_d, ScrStream **streams /*borrowed*/, - ScrClosure *cb /*moves*/, ScrStreamErrInv inv) { - return scr_stream_pipeline_ex((size_t)n_d, streams, cb, inv, true, NULL, NULL); +/* d._final: head.end(); onfinish = callback */ +static void scr_compose_final_inv(ScrClosure *cb, ScrStream *s) { + (void)s; + ScrCompose *c = scr_box_get_ref(cb->caps[0]); + ScrStream *r = scr_stream_end(c->head, NULL, NULL, NULL); + if (r) scr_stream_release(r); + c->onfinish = true; + scr_compose_release_v(c); } -static ScrStream *scr_stream_pipeline_ex(size_t n, ScrStream **streams, ScrClosure *cb, - ScrStreamErrInv inv, bool end, ScrDyn *signal, - ScrAbortAddFn add) { - ScrStreamPipeCtx *ctx = calloc(1, sizeof *ctx); - if (!ctx) scr_stream_oom(); - ctx->rc = 1; - ctx->n = n; - ctx->watched = end ? n : n - 1; - ctx->streams = calloc(n, sizeof *ctx->streams); - if (!ctx->streams) scr_stream_oom(); - for (size_t i = 0; i < n; i++) ctx->streams[i] = scr_stream_retain(streams[i]); - ctx->cb = cb; - ctx->inv = inv; - /* Chain the pipes first (Node's pipeline forwards EOF; end: false - * leaves the destination open). */ - for (size_t i = 0; i + 1 < n; i++) { - ScrStream *d = scr_stream_pipe(streams[i], streams[i + 1], end || i + 2 < n); +static void scr_compose_drain_l(ScrClosure *cb) { + ScrCompose *c = scr_box_get_ref(cb->caps[0]); + if (c->ondrain && c->d) { + c->ondrain = false; + scr_stream_write_done(c->d, NULL); + } + scr_compose_release_v(c); +} + +/* eos(tail): the pending final callback runs */ +static void scr_compose_tail_eos(ScrClosure *cb, ScrStream *s, ScrError *err) { + (void)s; + (void)err; + ScrCompose *c = scr_box_get_ref(cb->caps[0]); + if (c->onfinish && c->d) { + c->onfinish = false; + scr_stream_final_done(c->d, NULL); + } + scr_compose_release_v(c); +} + +/* d._read: tail.resume() */ +static void scr_compose_read_inv(ScrClosure *cb, ScrStream *s, double size) { + (void)s; + (void)size; + ScrCompose *c = scr_box_get_ref(cb->caps[0]); + if (c->d) scr_stream_release(scr_stream_resume(c->tail)); + scr_compose_release_v(c); +} + +/* tail.on('data'): d.push(chunk), pausing the tail when d is full. The + * chunk is the tail's Buffer, or its string when the tail is encoded. */ +static void scr_compose_data_l(ScrClosure *cb, ScrBytes *bytes, ScrStr *str) { + ScrCompose *c = scr_box_get_ref(cb->caps[0]); + ScrStream *d = c->d; + if (d) { + ScrStreamState *dst = d->st; + bool more; + if (dst->r.object_entries) { + int pushed = scr_stream_add_entry( + d, str ? (void *)scr_str_retain(str) : (void *)scr_bytes_retain(bytes), str != NULL); + if (pushed < 0) { + static const char m[] = "a composed stream whose tail mixes strings and Buffers is not supported yet"; + ScrStr *msg = scr_str_new(m, sizeof m - 1); + ScrError *e = scr_error_new(SCR_ERR_ERROR, msg); + scr_str_release(msg); + scr_stream_release(scr_stream_destroy(d, e)); + scr_error_release(e); + } + more = pushed > 0; + } else if (str) { + more = scr_stream_push_str(d, str); + } else { + more = scr_stream_push(d, bytes); + } + if (!more && !scr_exc_pending() && c->d) scr_stream_release(scr_stream_pause(c->tail)); + } + scr_compose_release_v(c); +} + +static void scr_compose_end_l(ScrClosure *cb) { + ScrCompose *c = scr_box_get_ref(cb->caps[0]); + if (c->d) scr_stream_push_null(c->d); + scr_compose_release_v(c); +} + +static ScrError *scr_compose_abort_error(void) { + ScrStr *message = scr_str_new("The operation was aborted", 25); + ScrError *e = scr_error_new(SCR_ERR_ERROR, message); + scr_str_release(message); + scr_str_release(e->name); + e->name = scr_str_new("AbortError", 10); + scr_error_set_code(e, "ABORT_ERR"); + e->name_present = true; + e->name_enumerable = true; + return e; +} + +/* d._destroy */ +static void scr_compose_destroy_inv(ScrClosure *cb, ScrStream *s, ScrError *err) { + (void)s; + ScrCompose *c = scr_box_get_ref(cb->caps[0]); + ScrError *e = err ? scr_error_retain(err) : NULL; + if (!e && c->onclose != SCR_COMPOSE_DONE) e = scr_compose_abort_error(); + c->ondrain = c->onfinish = false; + if (!c->tail->st->destroyed) scr_stream_release(scr_stream_destroy(c->tail, e)); + if (c->onclose == SCR_COMPOSE_DONE) { + if (c->d) scr_compose_destroyed(c, e); + } else { + c->onclose = SCR_COMPOSE_WAIT; + } + if (e) scr_error_release(e); + scr_compose_release_v(c); +} + +/* The pipeline callback (compose.js's onfinished). */ +static void scr_compose_finished(ScrClosure *cb, ScrStream *s, ScrError *err) { + (void)s; + ScrCompose *c = scr_box_get_ref(cb->caps[0]); + int onclose = c->onclose; + c->onclose = SCR_COMPOSE_DONE; + if (onclose == SCR_COMPOSE_WAIT) { + if (c->d) scr_compose_destroyed(c, err); + } else if (c->d && (err || (!c->readable && !c->writable))) { + ScrStream *d = scr_stream_retain(c->d); + scr_stream_release(scr_stream_destroy(d, err)); scr_stream_release(d); } - /* One watcher per stream (the destination is not watched under end: - * false): the terminal statuses drive error propagation and the final - * callback (after the LAST 'close'). */ - for (size_t i = 0; i < ctx->watched; i++) { - ScrClosure *w = scr_closure_new((void *)&scr_stream_pipeline_watch_inv, 1); - w->caps[0] = scr_box_new_obj(&scr_stream_pipectx_retain_v, &scr_stream_pipectx_release_v, NULL); - scr_box_set_ref(w->caps[0], scr_stream_pipectx_retain_v(ctx)); - scr_stream_fin_add(streams[i]->st, w, &scr_stream_pipeline_watch_inv); + scr_compose_release_v(c); +} + +static void scr_compose_arg_value(size_t i, const char *what, ScrStream *s) { + char m[160]; + int n = snprintf(m, sizeof m, "The argument 'streams[%zu]' must be %s. Received an instance of %s", + i, what, s->cls ? s->cls : "Stream"); + scr_throw_error_msg_code(SCR_ERR_TYPE, m, (size_t)n, "ERR_INVALID_ARG_VALUE"); +} + +ScrStream *scr_stream_compose(double n_d, ScrStream **streams) { + size_t n = (size_t)n_d; + if (n == 0) { + static const char msg[] = "The \"streams\" argument must be specified"; + scr_throw_error_msg_code(SCR_ERR_TYPE, msg, sizeof msg - 1, "ERR_MISSING_ARGS"); + return NULL; } - if (signal) { - ScrClosure *clo = scr_closure_new((void *)&scr_stream_pipeline_abort_thunk, 2); - clo->caps[0] = scr_box_new_obj(&scr_stream_pipectx_retain_v, &scr_stream_pipectx_release_v, NULL); - scr_box_set_ref(clo->caps[0], scr_stream_pipectx_retain_v(ctx)); - clo->caps[1] = scr_box_new_obj(&scr_stream_dyn_retain_v, &scr_stream_dyn_release_v, NULL); - scr_box_set_ref(clo->caps[1], scr_dyn_retain(signal)); - ScrDyn *listener = scr_dyn_new_func(clo, &scr_stream_pipeline_abort_thunk, 0, "()", "abort"); - ctx->dispose = add(signal, listener); - scr_dyn_release(listener); - if (ctx->dispose) { - static SCR_TL bool registered = false; - if (!registered) { - registered = true; - scr_atexit(scr_pipe_atexit); - } - ctx->live_next = scr_pipe_live; - if (scr_pipe_live) scr_pipe_live->live_prev = ctx; - scr_pipe_live = ctx; + if (n == 1) return scr_stream_retain(streams[0]); /* Duplex.from(duplex) */ + for (size_t i = 0; i < n; i++) { + if (i + 1 < n && !scr_stream_prop(streams[i], "readable")) { + scr_compose_arg_value(i, "readable", streams[i]); + return NULL; + } + if (i > 0 && !scr_stream_prop(streams[i], "writable")) { + scr_compose_arg_value(i, "writable", streams[i]); + return NULL; } } - scr_stream_pipectx_release_v(ctx); /* the boxes own it now */ - return scr_stream_retain(streams[n - 1]); + ScrCompose *c = calloc(1, sizeof *c); + if (!c) scr_stream_oom(); + c->rc = 1; + c->head = scr_stream_retain(streams[0]); + c->tail = scr_stream_retain(streams[n - 1]); + c->onclose = SCR_COMPOSE_RUNNING; + ScrStream *tail = scr_stream_pipeline_ex(n, streams, scr_compose_closure((void *)&scr_compose_finished, c), + &scr_compose_finished, true, NULL, NULL); + if (!tail) { + scr_compose_release_v(c); + return NULL; + } + scr_stream_release(tail); + c->writable = scr_stream_prop(c->head, "writable") != 0; + c->readable = scr_stream_prop(c->tail, "readable") != 0; + bool object_mode = c->tail->st->has_r && c->tail->st->r.object_entries; + ScrStream *d = scr_stream_new_duplex( + object_mode ? 16 : -1, -1, true, true, true, c->readable, c->writable, + c->readable ? scr_compose_closure((void *)&scr_compose_read_inv, c) : NULL, + c->readable ? &scr_compose_read_inv : NULL, + c->writable ? scr_compose_closure((void *)&scr_compose_write_inv, c) : NULL, + c->writable ? &scr_compose_write_inv : NULL, + c->writable ? scr_compose_closure((void *)&scr_compose_final_inv, c) : NULL, + c->writable ? &scr_compose_final_inv : NULL, + scr_compose_closure((void *)&scr_compose_destroy_inv, c), &scr_compose_destroy_inv); + if (object_mode) d->st->r.object_entries = true; + c->d = scr_stream_retain(d); + if (c->writable) { + scr_compose_on(c->head, "drain", &c->drain_l, (void *)&scr_compose_drain_l, c, scr_ee_inv_fixed0); + scr_closure_release(scr_stream_finished( + c->tail, scr_compose_closure((void *)&scr_compose_tail_eos, c), &scr_compose_tail_eos)); + } + if (c->readable) { + scr_compose_on(c->tail, "data", &c->data_l, (void *)&scr_compose_data_l, c, scr_ee_inv_fixed2); + scr_compose_on(c->tail, "end", &c->end_l, (void *)&scr_compose_end_l, c, scr_ee_inv_fixed0); + } + static SCR_TL bool registered = false; + if (!registered) { + registered = true; + scr_atexit(scr_compose_atexit); + } + c->live_next = scr_compose_live; + if (scr_compose_live) scr_compose_live->live_prev = c; + scr_compose_live = c; /* the reference c was created with */ + return d; } /* ── node:stream/promises ───────────────────────────────────────────── @@ -2368,7 +3398,8 @@ ScrPromise *scr_sp_finished(ScrStream *s) { ScrPromise *scr_sp_pipeline(double n, ScrStream **streams) { ScrPromise *p = scr_promise_new(); ScrStream *dst = scr_stream_pipeline(n, streams, scr_sp_watcher(p), &scr_sp_settle_inv); - scr_stream_release(dst); + if (dst) scr_stream_release(dst); + else scr_promise_reject_pending(p); return p; } @@ -2394,7 +3425,8 @@ ScrPromise *scr_sp_pipeline_opts(double n, ScrStream **streams, bool end, ScrDyn } ScrStream *dst = scr_stream_pipeline_ex((size_t)n, streams, scr_sp_watcher(p), &scr_sp_settle_inv, end, signal, add); - scr_stream_release(dst); + if (dst) scr_stream_release(dst); + else scr_promise_reject_pending(p); return p; } @@ -3346,7 +4378,7 @@ bool scr_stream_push_null(ScrStream *s) { ScrStr *tail = scr_strdec_end(st->r.enc, st->r.dec_pending); st->r.dec_pending = 0; if (scr_str_utf16_len(tail) > 0) { - scr_stream_rbuf_push(st, tail, false); + scr_stream_rbuf_push(st, tail, true, false); if (st->r.need_readable) scr_stream_emit_readable_nt(s); } else { scr_str_release(tail); @@ -3374,7 +4406,7 @@ void scr_stream_unshift(ScrStream *s, ScrBytes *chunk) { if (st->r.head_off > 0) { /* materialize the part-read head so the front slot is whole */ void *rest = scr_stream_rbuf_take(st, scr_stream_entry_len(st, st->r.buf[0]) - st->r.head_off); - scr_stream_rbuf_push(st, rest, true); + scr_stream_rbuf_push(st, rest, st->r.encoded, true); } scr_stream_add_chunk(s, chunk, true); } @@ -3434,28 +4466,44 @@ ScrStream *scr_stream_on_dyn(ScrStream *s, ScrStr *name, ScrDyn *cb, bool once, ScrDyn *scr_stream_read_dyn(ScrStream *s, ScrDyn *size) { double n = size && size->kind != SCR_DYN_UNDEF ? scr_dyn_number_coerce(size) : -1; if (scr_exc_pending()) return scr_dyn_undefined(); - void *chunk = s->st->has_r ? scr_stream_read_n(s, n) : NULL; + bool is_str = false; + void *chunk = s->st->has_r ? scr_stream_read_n_kind(s, n, &is_str) : NULL; if (!chunk) return scr_dyn_new_null(); - ScrDyn *result = s->st->r.encoded ? scr_dyn_new_str(chunk) : scr_dyn_new_buffer(chunk); - scr_stream_entry_release(s->st, chunk); + ScrDyn *result = is_str ? scr_dyn_new_str(chunk) : scr_dyn_new_buffer(chunk); + if (is_str) scr_str_release(chunk); + else scr_bytes_release(chunk); return result; } +/* read() answers STRINGS once encoded: the static result type here is + * `Buffer | null`, so the honest answer is a fence, not a mistyped + * payload (SEMANTICS.md; 'data' listeners carry strings). */ +static void scr_stream_read_str_fence(void) { + static const char msg[] = + "read() on a stream with an encoding set is not supported yet " + "(consume 'data' events, which deliver strings)"; + scr_throw_error_msg(SCR_ERR_ERROR, msg, sizeof msg - 1); +} + ScrBytes *scr_stream_read(ScrStream *s, double size) { /* a destroyed readable still DRAINS its buffer (Node: destroy leaves * buffered chunks readable; only _read refills are blocked) */ if (!s->st->has_r) return NULL; if (s->st->r.encoded) { - /* read() answers STRINGS once encoded — the static result type here - * is `Buffer | null`, so the honest answer is a fence, not a - * mistyped payload (SEMANTICS.md; 'data' listeners carry strings). */ - static const char msg[] = - "read() on a stream with an encoding set is not supported yet " - "(consume 'data' events, which deliver strings)"; - scr_throw_error_msg(SCR_ERR_ERROR, msg, sizeof msg - 1); + scr_stream_read_str_fence(); return NULL; } - return scr_stream_read_n(s, size); + bool is_str; + void *chunk = scr_stream_read_n_kind(s, size, &is_str); + if (chunk && is_str) { + /* the read itself made the stream hold strings (a lazy Readable.from + * pull's first string, or setEncoding() inside _read): the same + * fence, never a string in the Buffer result */ + scr_str_release(chunk); + scr_stream_read_str_fence(); + return NULL; + } + return chunk; } /* setEncoding(enc): flips the readable buffer into string mode — Node @@ -3469,11 +4517,13 @@ ScrStream *scr_stream_set_encoding(ScrStream *s, ScrStr *enc) { /* switching encodings mid-stream: fold the old decoder's pending * tail into the buffer first (rare; Node re-creates the decoder) */ ScrStr *tail = scr_strdec_end(st->r.enc, st->r.dec_pending); - if (scr_str_utf16_len(tail) > 0) scr_stream_rbuf_push(st, tail, false); + if (scr_str_utf16_len(tail) > 0) scr_stream_rbuf_push(st, tail, true, false); else scr_str_release(tail); scr_str_release(st->r.enc); st->r.enc = scr_str_retain(enc); st->r.dec_pending = 0; + st->r.decoding = true; + st->r.object_kind_set = true; return scr_stream_retain(s); } st->r.enc = scr_str_retain(enc); @@ -3501,12 +4551,29 @@ ScrStream *scr_stream_set_encoding(ScrStream *s, ScrStr *enc) { content = joined; } } + if (st->r.ended) { + /* Node 24: an ended stream's decoder flushes its partial tail */ + ScrStr *tail = scr_strdec_end(st->r.enc, st->r.dec_pending); + st->r.dec_pending = 0; + if (content == NULL) { + content = tail; + } else { + ScrStr *joined = scr_str_concat(content, tail); + scr_str_release(content); + scr_str_release(tail); + content = joined; + } + } st->r.n = 0; st->r.head_off = 0; st->r.length = 0; - st->r.encoded = true; /* AFTER the byte-entry walk above */ + /* AFTER the byte-entry walk above: from here every entry is a string, + * and Buffer entries pushed later decode (scr_stream_entry_convert) */ + st->r.encoded = true; + st->r.decoding = true; + st->r.object_kind_set = true; if (content != NULL) { - if (scr_str_utf16_len(content) > 0) scr_stream_rbuf_push(st, content, false); + if (scr_str_utf16_len(content) > 0) scr_stream_rbuf_push(st, content, true, false); else scr_str_release(content); } return scr_stream_retain(s); @@ -3685,8 +4752,18 @@ void scr_stream_write_done(ScrStream *s, ScrError *err /*moves*/) { ScrClosure *ucb = st->w.inflight_cb; st->w.inflight_cb = NULL; if (err) { - if (ucb) scr_st_tick(s, SCR_ST_WCB_ERR, NULL, ucb); - scr_stream_error_or_destroy(s, err); + /* Node's onwrite: errored is set now; onwriteError (the callback, + * then errorOrDestroy) runs on a tick after a synchronous write. */ + if (!st->errored) st->errored = scr_error_retain(err); + if (st->w.wsync) { + scr_st_tick(s, SCR_ST_WRITE_ERR, err, ucb); + return; + } + if (ucb) { + ((void (*)(ScrClosure *))ucb->fn)(ucb); + scr_closure_release(ucb); + } + if (!scr_exc_pending()) scr_stream_error_or_destroy(s, err); scr_error_release(err); return; } @@ -3707,6 +4784,7 @@ void scr_stream_write_done(ScrStream *s, ScrError *err /*moves*/) { void scr_stream_final_done(ScrStream *s, ScrError *err /*moves*/) { ScrStreamState *st = s->st; + st->w.final_pending = false; if (err) { scr_stream_error_or_destroy(s, err); scr_error_release(err); @@ -3765,6 +4843,7 @@ void scr_stream_transform_done(ScrStream *s, ScrError *err /*moves*/, ScrBytes * void scr_stream_flush_done(ScrStream *s, ScrError *err /*moves*/, ScrBytes *data /*moves*/, ScrStr *data_str /*moves*/) { ScrStreamState *st = s->st; + st->flush_pending = false; if (err) { if (data) scr_bytes_release(data); if (data_str) scr_str_release(data_str); @@ -4034,6 +5113,7 @@ static void scr_stream_run_tick(ScrStreamTick *t) { if (st->next_err_consumed && !scr_emitter_has((ScrEmitter *)s, "error")) { /* the for-await's rejection consumed it (Node's iterator * registers its own 'error' handler — no unhandled crash) */ + scr_stream_settle_next(s); break; } if (st->fin.n > 0 && !scr_emitter_has((ScrEmitter *)s, "error")) { @@ -4043,6 +5123,7 @@ static void scr_stream_run_tick(ScrStreamTick *t) { break; } scr_stream_emit_error(s, t->err); + if (!scr_exc_pending()) scr_stream_settle_next(s); } break; } @@ -4081,6 +5162,16 @@ static void scr_stream_run_tick(ScrStreamTick *t) { } break; } + case SCR_ST_WRITE_ERR: { + ScrClosure *cb = t->cb; + t->cb = NULL; + if (cb) { + ((void (*)(ScrClosure *))cb->fn)(cb); + scr_closure_release(cb); + } + if (!scr_exc_pending()) scr_stream_error_or_destroy(s, t->err); + break; + } case SCR_ST_FIN: { scr_stream_notify_finished(s); break; @@ -4089,7 +5180,7 @@ static void scr_stream_run_tick(ScrStreamTick *t) { if (st->r.next_waiter && st->r.length == 0 && (st->r.ended || st->destroyed)) { ScrPromise *w = st->r.next_waiter; st->r.next_waiter = NULL; - scr_stream_next_fulfill(st, w, NULL); + scr_stream_next_fulfill(st, w, NULL, false); } else if (st->r.next_waiter) { scr_stream_settle_next(s); /* content raced in: deliver it */ } diff --git a/packages/runtime/src/scr_stream.h b/packages/runtime/src/scr_stream.h index b2f9a7aff..b5a16013d 100644 --- a/packages/runtime/src/scr_stream.h +++ b/packages/runtime/src/scr_stream.h @@ -160,16 +160,22 @@ ScrClosure *scr_stream_finished(ScrStream *s, ScrClosure *cb, ScrStreamErrInv in * the slot): success calls with NO arguments, errors box through the * boundary encoding. cb's cap boxes the dyn callable. */ void scr_stream_finished_dyn_inv(ScrClosure *cb, ScrStream *s, ScrError *err); -/* stream.pipeline(s1..sn, cb) — the callback form over stream arguments: - * chains pipes (end: true), destroys every other stream with the first - * error (Node's destroyer, pipeline order), and calls cb once after the - * LAST 'close' with that error (or NULL). Streams borrowed; cb MOVES; - * answers the destination +1 (pipeline's return value). */ +/* stream.pipeline(s1..sn, cb) — the callback form over stream arguments + * (pipeline.js): pipes each pair, destroys the unfinished streams with + * the first error, and calls cb on a nextTick once every stream's eos + * watcher finished, with that error (or NULL). Streams borrowed; cb + * MOVES; answers the destination +1 (pipeline's return value), or NULL + * with ERR_STREAM_UNABLE_TO_PIPE / ERR_INVALID_ARG_TYPE pending. */ ScrStream *scr_stream_pipeline(double n, ScrStream **streams, ScrClosure *cb, ScrStreamErrInv inv); /* The dyn-valued twins (the callback as a checked-dynamic value — * mustCall wrappers; borrowed). */ ScrClosure *scr_stream_finished_dyn(ScrStream *s, struct ScrDyn *cb); ScrStream *scr_stream_pipeline_dyn(double n, ScrStream **streams, struct ScrDyn *cb); +/* stream.compose(s1..sn): a pipeline over the streams and a Duplex that + * writes to the first and reads from the last (compose.js). One stream + * answers itself (Duplex.from of a duplex). +1, or NULL with + * ERR_MISSING_ARGS / ERR_INVALID_ARG_VALUE pending. Streams borrowed. */ +ScrStream *scr_stream_compose(double n, ScrStream **streams); /* Emitted subclass RC/trace delegates for the state block (NULL-safe — * an exception before super(options) leaves st unset). The registry @@ -247,10 +253,23 @@ ScrPromise *scr_sc_text(ScrStream *s); ScrPromise *scr_sc_json(ScrStream *s); ScrPromise *scr_sc_buffer(ScrStream *s); ScrPromise *scr_sc_array_buffer(ScrStream *s); -/* Readable.from(array): +1 fully-seeded object-entry stream (one WHOLE - * chunk per element — strings or Buffers per the flag; hwm 1, already - * EOF'd). Borrows arr. */ -ScrStream *scr_stream_from_arr(ScrArr *arr, bool strings); +/* Readable.from over a typed source: arr holds the string or Buffer + * elements (one element under SCR_FROM_SINGLE, from.js's string/Buffer + * case). Elements push lazily, one _read at a time, as whole entries + * unless SCR_FROM_BYTE_MODE ({ objectMode: false }). hwm applies under + * SCR_FROM_HAS_HWM. +1, or NULL with ERR_INVALID_ARG_VALUE pending. + * Borrows arr. */ +enum { + SCR_FROM_STRINGS = 1, + SCR_FROM_SINGLE = 2, + SCR_FROM_BYTE_MODE = 4, + SCR_FROM_HAS_HWM = 8, +}; +ScrStream *scr_stream_from_arr(ScrArr *arr, double flags, double hwm); +/* Readable.from(iterable, options) in the JS lane: strings, Buffers and + * arrays of them; options objectMode and highWaterMark. +1, or NULL with + * the error pending (ERR_INVALID_ARG_TYPE for a non-iterable). Borrows. */ +ScrStream *scr_stream_from_dyn(ScrDyn *src, ScrDyn *opts); ScrStream *scr_stream_pause(ScrStream *s); /* recv +1 */ ScrStream *scr_stream_resume(ScrStream *s); /* recv +1 */ bool scr_stream_is_paused(ScrStream *s); @@ -279,6 +298,34 @@ ScrStream *scr_stream_iterator_close(ScrStream *s); * NULL with ERR_INVALID_ARG_TYPE pending. */ ScrStream *scr_stream_add_abort_signal(ScrStream *s, ScrDyn *signal, bool from_option, ScrAbortAddFn add); +/* Object-mode streams for the stream operators: Readable.from's stream + * (highWaterMark 1) around native callbacks (+1; callbacks move), and its + * push: whole string or Buffer entries, the first push fixing the kind. + * Answers push()'s result, -1 for another kind of value, -2 for a kind + * that differs from the first push's. */ +ScrStream *scr_stream_new_object_readable(ScrClosure *read, ScrStreamReadInv read_inv, + ScrClosure *destroy, ScrStreamErrInv destroy_inv); +int scr_stream_push_object(ScrStream *s, const ScrDyn *v); +/* The Readable operators (scr_stream_ops.c, the streamOps feature): the + * callbacks, counts and options are checked-dynamic values (borrowed). + * The stream-returning ones answer the +1 object-mode output stream, or + * NULL with Node's validation error pending; the promise-returning ones + * answer a +1 promise and reject instead of throwing. toArray settles a + * dyn array, find and reduce a dyn value — reduce's `kind` picks the + * payload instead: 0 dyn, 1 number, 2 boolean, 3 string — some and every + * a boolean, forEach nothing. */ +ScrStream *scr_stream_op_map(ScrStream *s, ScrDyn *fn, ScrDyn *options); +ScrStream *scr_stream_op_filter(ScrStream *s, ScrDyn *fn, ScrDyn *options); +ScrStream *scr_stream_op_flat_map(ScrStream *s, ScrDyn *fn, ScrDyn *options); +ScrStream *scr_stream_op_drop(ScrStream *s, ScrDyn *number, ScrDyn *options); +ScrStream *scr_stream_op_take(ScrStream *s, ScrDyn *number, ScrDyn *options); +ScrPromise *scr_stream_op_to_array(ScrStream *s, ScrDyn *options); +ScrPromise *scr_stream_op_for_each(ScrStream *s, ScrDyn *fn, ScrDyn *options); +ScrPromise *scr_stream_op_some(ScrStream *s, ScrDyn *fn, ScrDyn *options); +ScrPromise *scr_stream_op_every(ScrStream *s, ScrDyn *fn, ScrDyn *options); +ScrPromise *scr_stream_op_find(ScrStream *s, ScrDyn *fn, ScrDyn *options); +ScrPromise *scr_stream_op_reduce(ScrStream *s, ScrDyn *fn, ScrDyn *init, bool has_init, + ScrDyn *options, double kind); /* finished(s, options[, cb]) — Node's eos with its listeners and state * predicates: readable/writable are -1 (the default), 0 or 1; on_error is * options.error !== false; signal is NULL or borrowed (ERR_INVALID_ARG_TYPE diff --git a/packages/runtime/src/scr_stream_ops.c b/packages/runtime/src/scr_stream_ops.c new file mode 100644 index 000000000..1f64d5fa7 --- /dev/null +++ b/packages/runtime/src/scr_stream_ops.c @@ -0,0 +1,2057 @@ +/* node:stream's Readable operators (lib/internal/streams/operators.js): + * map, filter, forEach, toArray, some, find, every, flatMap, drop, take + * and reduce, over streams whose chunks are strings or Buffers. Linked + * only when a program calls one (the streamOps feature). + * + * Node writes the operators as async functions and async generators, and + * wraps the stream-returning ones in Readable.from. Their observable + * ordering (when callbacks run relative to the consumer, when promises + * settle) is the order of those functions' awaits, so this unit follows + * the JavaScript source step for step: every `await x` is one op_await + * (a promise job after x settles, the same hop a compiled await takes), + * the generators follow the async generator protocol (request queue, + * yield awaiting its operand, return and throw requests), and the + * Readable.from(generator) wrapper is from.js's nextAsync/close. Each + * step runs on its own fiber through scr_async_spawn_after, so the unit + * works on every target, wasm32 included. + * + * The user's callbacks are checked-dynamic function values called with + * one argument (the chunk); callbacks that declare the options argument + * ({ signal }) are refused, because the AbortSignal Node passes there has + * no representation here. options.signal itself is polled where Node + * polls it (signal.aborted), so no listener is needed. Stream-returning + * operators produce object-mode Readables whose entries are strings or + * Buffers; any other value is refused when it would be pushed. */ +#include "scr_runtime.h" + +#include +#include +#include +#include + +static void op_oom(void) { + fputs("scriptc: out of memory\n", stderr); + abort(); +} + +static void *op_alloc(size_t n) { + void *p = calloc(1, n); + if (!p) op_oom(); + return p; +} + +/* ── refcounted step owners ────────────────────────────────────────── */ + +typedef struct OpObj OpObj; +struct OpObj { + size_t rc; + void (*drop)(OpObj *self); /* frees the object's own fields and itself */ +}; + +static void *op_ref(void *o) { + ((OpObj *)o)->rc++; + return o; +} + +static void op_unref(void *o) { + OpObj *b = o; + if (--b->rc == 0) b->drop(b); +} + +/* ── sentinels ─────────────────────────────────────────────────────── */ + +/* kEmpty (a callback result that yields nothing), kEof (the pump's end + * marker) and the generators' done marker: process-lived objects + * compared by identity, never visible to the program. */ +static SCR_TL ScrDyn *op_k_empty = NULL; +static SCR_TL ScrDyn *op_k_eof = NULL; +static SCR_TL ScrDyn *op_k_done = NULL; + +static void op_sentinels_free(void) { + scr_dyn_release(op_k_empty); + scr_dyn_release(op_k_eof); + scr_dyn_release(op_k_done); + op_k_empty = op_k_eof = op_k_done = NULL; +} + +static void op_sentinels_init(void) { + if (op_k_empty) return; + op_k_empty = scr_dyn_new_obj(); + op_k_eof = scr_dyn_new_obj(); + op_k_done = scr_dyn_new_obj(); + scr_atexit(op_sentinels_free); +} + +/* ── awaits ────────────────────────────────────────────────────────── */ + +/* A continuation receives the awaited value (+1), or NULL with the + * rejection pending in the exception cell. It must leave no exception + * pending when it returns. */ +typedef void (*OpCont)(void *self, ScrDyn *value); + +typedef struct { + void *self; + OpCont k; + ScrPromise *p; +} OpAwait; + +static void op_await_entry(ScrFiber *fiber, void *arg) { + (void)fiber; + OpAwait a = *(OpAwait *)arg; + free(arg); + ScrDyn *v = scr_await_dyn(a.p); + scr_promise_release(a.p); + a.k(a.self, v); + if (scr_exc_pending()) scr_exc_clear(); + op_unref(a.self); +} + +/* `await p`: k runs in the promise job after p settles. Borrows p. */ +static void op_await(ScrPromise *p, OpCont k, void *self) { + OpAwait *a = op_alloc(sizeof *a); + a->self = op_ref(self); + a->k = k; + a->p = scr_promise_retain(p); + ScrPromise *fiber = scr_async_spawn_after(p, &op_await_entry, a); + scr_promise_release(fiber); +} + +static ScrPromise *op_resolved(ScrDyn *v /*borrowed*/) { + ScrPromise *p = scr_promise_new(); + scr_promise_fulfill_ref(p, scr_dyn_retain(v), scr_dyn_retain_v, scr_dyn_release_v, NULL); + return p; +} + +/* `await v` over a value: a promise is awaited, anything else is awaited + * as an already resolved promise (one job). Borrows v. */ +static void op_await_value(ScrDyn *v, OpCont k, void *self) { + if (v->kind == SCR_DYN_PROMISE) { + op_await(v->v.promise, k, self); + return; + } + ScrPromise *p = op_resolved(v); + op_await(p, k, self); + scr_promise_release(p); +} + +static void op_fulfill(ScrPromise *p, ScrDyn *v /*borrowed*/) { + scr_promise_fulfill_ref(p, scr_dyn_retain(v), scr_dyn_retain_v, scr_dyn_release_v, NULL); +} + +/* Rejects p with a saved exception (borrowed). */ +static void op_reject(ScrPromise *p, const ScrCaught *c) { + scr_rethrow(c); + scr_promise_reject_pending(p); +} + +/* PromiseReject(err) for a saved exception: +1 dyn promise. */ +static ScrDyn *op_rejected_dyn(const ScrCaught *c) { + ScrPromise *p = scr_promise_new(); + op_reject(p, c); + ScrDyn *d = scr_dyn_new_promise(p); + scr_promise_release(p); + return d; +} + +/* ── errors ────────────────────────────────────────────────────────── */ + +static ScrCaught *op_take_error(ScrError *e /*moves*/) { + scr_throw_obj(e, &scr_error_retain_v, &scr_error_release_v, scr_error_trace_arg()); + return scr_exc_take(); +} + +/* new AbortError() or new AbortError(undefined, { cause }) */ +static ScrError *op_abort_error(const ScrDyn *cause /*NULL: no cause*/) { + ScrStr *message = scr_str_new("The operation was aborted", 25); + ScrError *e = scr_error_new(SCR_ERR_ERROR, message); + scr_str_release(message); + scr_str_release(e->name); + e->name = scr_str_new("AbortError", 10); + /* AbortError assigns code, then name: own enumerable properties */ + scr_error_set_code(e, "ABORT_ERR"); + e->name_present = true; + e->name_enumerable = true; + if (cause) scr_error_define_cause(e, (ScrDyn *)cause); + return e; +} + +static ScrError *op_coded_error(int kind, const char *code, const char *message) { + ScrStr *m = scr_str_new(message, strlen(message)); + ScrError *e = scr_error_new(kind, m); + scr_str_release(m); + scr_error_set_code(e, code); + return e; +} + +/* ERR_OUT_OF_RANGE: `The value of "" is out of range. It must be + * . Received ` (thrown; pending on return). */ +static void op_throw_out_of_range(const char *name, const char *range, double received) { + char num[48]; + scr_num_received(received, num); + char msg[256]; + int n = snprintf(msg, sizeof msg, "The value of \"%s\" is out of range. It must be %s. Received %s", + name, range, num); + scr_throw_error_msg_code(SCR_ERR_RANGE, msg, (size_t)n, "ERR_OUT_OF_RANGE"); +} + +/* The runtime refusal for a value outside the string/Buffer model. */ +static ScrError *op_unsupported(const char *what) { + char msg[256]; + int n = snprintf(msg, sizeof msg, "%s is not supported yet", what); + ScrStr *m = scr_str_new(msg, (size_t)n); + ScrError *e = scr_error_new(SCR_ERR_ERROR, m); + scr_str_release(m); + return e; +} + +/* ── options ───────────────────────────────────────────────────────── */ + +static ScrDyn *op_get(const ScrDyn *obj, const char *key) { + if (!obj) return NULL; + size_t n = strlen(key); + if (obj->kind == SCR_DYN_OBJ) { + ScrDyn *v = scr_dyn_obj_get(obj, key, n); + return v ? scr_dyn_retain(v) : NULL; + } + if (obj->kind == SCR_DYN_HANDLE) { + ScrStr *k = scr_str_new(key, n); + ScrDyn *v = scr_dyn_handle_key_get(obj, k); + scr_str_release(k); + return v; + } + return NULL; +} + +static bool op_nullish(const ScrDyn *v) { + return v == NULL || v->kind == SCR_DYN_UNDEF || v->kind == SCR_DYN_NULL; +} + +/* validateObject(options, 'options') when options != null: false with + * ERR_INVALID_ARG_TYPE pending. */ +static bool op_check_options(const ScrDyn *options) { + if (op_nullish(options)) return true; + if ((options->kind == SCR_DYN_OBJ || options->kind == SCR_DYN_HANDLE) && + !scr_dyn_is_callable(options)) { + return true; + } + scr_dyn_arg_type_fail("options", "of type object", options); + return false; +} + +/* options.signal when != null, validated (validateAbortSignal): +1, or + * NULL when absent. *ok false with ERR_INVALID_ARG_TYPE pending. */ +static ScrDyn *op_signal(const ScrDyn *options, bool *ok) { + *ok = true; + ScrDyn *s = op_get(options, "signal"); + if (op_nullish(s)) { + if (s) scr_dyn_release(s); + return NULL; + } + ScrDyn *aborted = NULL; + if (s->kind == SCR_DYN_OBJ || s->kind == SCR_DYN_HANDLE) aborted = op_get(s, "aborted"); + if (!aborted) { + scr_dyn_prop_type_fail("options.signal", "an instance of AbortSignal", s); + scr_dyn_release(s); + *ok = false; + return NULL; + } + scr_dyn_release(aborted); + return s; +} + +static bool op_aborted(const ScrDyn *signal) { + if (!signal) return false; + ScrDyn *a = op_get(signal, "aborted"); + bool r = a && scr_dyn_truthy(a); + if (a) scr_dyn_release(a); + return r; +} + +/* The callback argument: a function that does not read the options + * argument. `max` is how many parameters Node's call passes before the + * options. False with the error pending. */ +static bool op_check_fn(const ScrDyn *fn, const char *name, uint32_t max) { + if (!fn || !scr_dyn_is_callable(fn)) { + scr_dyn_arg_type_fail(name, "of type function", fn ? fn : scr_dyn_undefined()); + return false; + } + if (fn->kind == SCR_DYN_FUNC && fn->v.fn.arity > max) { + static const char m[] = + "stream operator callbacks that read the options argument ({ signal }) are not supported yet"; + scr_throw_error_msg(SCR_ERR_ERROR, m, sizeof m - 1); + return false; + } + return true; +} + +/* ── iteration sources ─────────────────────────────────────────────── */ + +typedef struct OpGen OpGen; + +/* `for await (const v of src)` over a Readable (its async iterator) or an + * operator generator. */ +typedef struct { + ScrStream *s; /* +1, or NULL */ + OpGen *g; /* +1, or NULL */ +} OpSrc; + +static ScrPromise *gen_next(OpGen *g); +static ScrPromise *gen_return(OpGen *g, ScrDyn *v); +static ScrPromise *gen_throw(OpGen *g, const ScrCaught *c); + +/* iterator.next(): +1 promise of the next value or the end. */ +static ScrPromise *src_next(OpSrc *src) { + return src->s ? scr_stream_next_chunk_dyn(src->s) : gen_next(src->g); +} + +static bool src_done(const OpSrc *src, const ScrDyn *v) { + return src->s ? v->kind == SCR_DYN_UNDEF : v == op_k_done; +} + +/* The Readable iterator's return(): the iterator generator resumes at its + * yield with a return completion (one await), its finally destroys the + * stream (destroyer: an AbortError when not finished), and the request + * settles. */ +typedef struct { + OpObj base; + ScrStream *s; + ScrPromise *p; +} OpStreamReturn; + +static void op_stream_return_drop(OpObj *o) { + OpStreamReturn *r = (OpStreamReturn *)o; + scr_stream_release(r->s); + scr_promise_release(r->p); + free(r); +} + +static void op_stream_return_k(void *self, ScrDyn *v) { + OpStreamReturn *r = self; + if (v) scr_dyn_release(v); + scr_stream_release(scr_stream_iterator_close(r->s)); + op_fulfill(r->p, op_k_done); +} + +static ScrPromise *src_return(OpSrc *src) { + if (src->g) return gen_return(src->g, scr_dyn_undefined()); + OpStreamReturn *r = op_alloc(sizeof *r); + r->base.rc = 1; + r->base.drop = &op_stream_return_drop; + r->s = scr_stream_retain(src->s); + r->p = scr_promise_new(); + ScrPromise *result = scr_promise_retain(r->p); + op_await_value(scr_dyn_undefined(), &op_stream_return_k, r); + op_unref(r); + return result; +} + +static void src_release(OpSrc *src) { + if (src->s) scr_stream_release(src->s); + if (src->g) op_unref(src->g); + src->s = NULL; + src->g = NULL; +} + +/* ── the async generator protocol ──────────────────────────────────── */ + +enum { GS_START, GS_SUSPENDED, GS_EXECUTING, GS_AWAIT_RETURN, GS_DONE }; +enum { GR_NEXT, GR_RETURN, GR_THROW }; + +typedef struct { + ScrPromise *p; + int kind; + ScrDyn *value; /* return value (+1) */ + ScrCaught *error; /* throw value (+1) */ +} OpGenReq; + +/* An operator generator: the protocol state plus the body's entry points. + * resume runs the body from its start or from the yield it is suspended + * at (normal completion); ret delivers a return completion at that yield + * (after the unwrap await) and thr a throw completion. */ +struct OpGen { + OpObj base; + int state; + OpGenReq *q; + size_t qh, qn, qcap; + void (*resume)(OpGen *g); + void (*ret)(OpGen *g, ScrDyn *v); + void (*thr)(OpGen *g, ScrCaught *c); + void (*body_drop)(OpGen *g); +}; + +static void gen_drop(OpObj *o) { + OpGen *g = (OpGen *)o; + for (size_t i = g->qh; i < g->qn; i++) { + scr_promise_release(g->q[i].p); + if (g->q[i].value) scr_dyn_release(g->q[i].value); + if (g->q[i].error) scr_caught_release(g->q[i].error); + } + free(g->q); + g->body_drop(g); +} + +static ScrPromise *gen_enqueue(OpGen *g, int kind, ScrDyn *value, const ScrCaught *error) { + if (g->qn == g->qcap) { + if (g->qh > 0) { + memmove(g->q, g->q + g->qh, (g->qn - g->qh) * sizeof *g->q); + g->qn -= g->qh; + g->qh = 0; + } + if (g->qn == g->qcap) { + g->qcap = g->qcap ? g->qcap * 2 : 4; + g->q = realloc(g->q, g->qcap * sizeof *g->q); + if (!g->q) op_oom(); + } + } + OpGenReq *r = &g->q[g->qn++]; + r->p = scr_promise_new(); + r->kind = kind; + r->value = value ? scr_dyn_retain(value) : NULL; + r->error = error ? scr_caught_retain((ScrCaught *)error) : NULL; + return scr_promise_retain(r->p); +} + +static bool gen_has_req(const OpGen *g) { return g->qh < g->qn; } + +static OpGenReq gen_shift(OpGen *g) { + OpGenReq r = g->q[g->qh++]; + if (g->qh == g->qn) g->qh = g->qn = 0; + return r; +} + +static void gen_req_release(OpGenReq *r) { + scr_promise_release(r->p); + if (r->value) scr_dyn_release(r->value); + if (r->error) scr_caught_release(r->error); +} + +static void gen_drain(OpGen *g); + +/* AsyncGeneratorAwaitReturn: await the value, then the request settles + * done (a rejection rejects it). */ +typedef struct { + OpObj base; + OpGen *g; + ScrPromise *p; +} OpAwaitReturn; + +static void op_await_return_drop(OpObj *o) { + OpAwaitReturn *a = (OpAwaitReturn *)o; + op_unref(a->g); + scr_promise_release(a->p); + free(a); +} + +static void op_await_return_k(void *self, ScrDyn *v) { + OpAwaitReturn *a = self; + OpGen *g = a->g; + if (v) { + scr_dyn_release(v); + op_fulfill(a->p, op_k_done); + } else { + scr_promise_reject_pending(a->p); + } + g->state = GS_DONE; + gen_drain(g); +} + +static void gen_await_return(OpGen *g, OpGenReq *r) { + g->state = GS_AWAIT_RETURN; + OpAwaitReturn *a = op_alloc(sizeof *a); + a->base.rc = 1; + a->base.drop = &op_await_return_drop; + a->g = op_ref(g); + a->p = scr_promise_retain(r->p); + op_await_value(r->value ? r->value : scr_dyn_undefined(), &op_await_return_k, a); + op_unref(a); +} + +/* The completed generator answers its queued requests: next settles done + * at once, throw rejects at once, return awaits its value first. */ +static void gen_drain(OpGen *g) { + while (g->state == GS_DONE && gen_has_req(g)) { + OpGenReq r = gen_shift(g); + if (r.kind == GR_RETURN) { + gen_await_return(g, &r); + gen_req_release(&r); + return; + } + if (r.kind == GR_THROW) { + op_reject(r.p, r.error); + } else { + op_fulfill(r.p, op_k_done); + } + gen_req_release(&r); + } +} + +/* The body finished: the first request settles (done, or rejected with + * the body's exception), then the queue drains. Borrows c. */ +static void gen_complete(OpGen *g, const ScrCaught *c) { + g->state = GS_DONE; + if (gen_has_req(g)) { + OpGenReq r = gen_shift(g); + if (c) op_reject(r.p, c); + else op_fulfill(r.p, op_k_done); + gen_req_release(&r); + } + gen_drain(g); +} + +/* Delivers the first queued request to a body suspended at a yield (or + * already running its continuation): next resumes it, return awaits the + * value and then delivers the return completion, throw delivers the + * throw completion. */ +typedef struct { + OpObj base; + OpGen *g; +} OpUnwrapReturn; + +static void op_unwrap_return_drop(OpObj *o) { + OpUnwrapReturn *u = (OpUnwrapReturn *)o; + op_unref(u->g); + free(u); +} + +static void op_unwrap_return_k(void *self, ScrDyn *v) { + OpGen *g = ((OpUnwrapReturn *)self)->g; + if (!v) { + ScrCaught *c = scr_exc_take(); + g->thr(g, c); + scr_caught_release(c); + return; + } + g->ret(g, v); + scr_dyn_release(v); +} + +static void gen_deliver(OpGen *g) { + OpGenReq *r = &g->q[g->qh]; + g->state = GS_EXECUTING; + if (r->kind == GR_NEXT) { + g->resume(g); + } else if (r->kind == GR_THROW) { + g->thr(g, r->error); + } else { + OpUnwrapReturn *u = op_alloc(sizeof *u); + u->base.rc = 1; + u->base.drop = &op_unwrap_return_drop; + u->g = op_ref(g); + op_await_value(r->value ? r->value : scr_dyn_undefined(), &op_unwrap_return_k, u); + op_unref(u); + } +} + +static ScrPromise *gen_request(OpGen *g, int kind, ScrDyn *value, const ScrCaught *error) { + ScrPromise *p = gen_enqueue(g, kind, value, error); + if (g->state == GS_START) { + if (kind == GR_NEXT) { + gen_deliver(g); + } else { + /* A generator that never started completes without running. */ + g->state = GS_DONE; + gen_drain(g); + } + } else if (g->state == GS_SUSPENDED) { + gen_deliver(g); + } else if (g->state == GS_DONE) { + gen_drain(g); + } + return p; +} + +static ScrPromise *gen_next(OpGen *g) { return gen_request(g, GR_NEXT, NULL, NULL); } +static ScrPromise *gen_return(OpGen *g, ScrDyn *v) { return gen_request(g, GR_RETURN, v, NULL); } +static ScrPromise *gen_throw(OpGen *g, const ScrCaught *c) { return gen_request(g, GR_THROW, NULL, c); } + +/* The body yields an awaited value: the request settles with it, and the + * body continues at once when another request is queued (the generator + * never suspends), or suspends until the next one. Borrows v. */ +static void gen_yield_now(OpGen *g, ScrDyn *v) { + OpGenReq r = gen_shift(g); + op_fulfill(r.p, v); + gen_req_release(&r); + if (gen_has_req(g)) { + gen_deliver(g); + } else { + g->state = GS_SUSPENDED; + } +} + +/* `yield v` in an async generator awaits v first. */ +static void op_yield_k(void *self, ScrDyn *v) { + OpGen *g = self; + if (!v) { + ScrCaught *c = scr_exc_take(); + g->thr(g, c); + scr_caught_release(c); + return; + } + gen_yield_now(g, v); + scr_dyn_release(v); +} + +static void gen_yield(OpGen *g, ScrDyn *v /*borrowed*/) { op_await_value(v, &op_yield_k, g); } + +static void gen_init(OpGen *g, void (*body_drop)(OpGen *)) { + g->base.rc = 1; + g->base.drop = &gen_drop; + g->state = GS_START; + g->body_drop = body_drop; +} + +/* ── map (and filter, forEach, every through it) ───────────────────── */ + +/* How each source value turns into the queued result: map calls fn + * directly; filter, forEach and every go through Node's async wrappers + * (filterFn, forEachFn, and every's negating predicate inside filterFn). */ +enum { MODE_MAP, MODE_FILTER, MODE_FOREACH, MODE_EVERY }; + +typedef struct { + OpGen g; + OpSrc src; + ScrDyn *fn; /* +1 */ + ScrDyn *signal; /* +1 or NULL */ + int mode; + double concurrency, hwm; + ScrDyn **queue; /* +1 entries: dyn promises or kEof */ + size_t qh, qn, qcap; + double cnt; + bool done; + ScrPromise *next_p; /* the consumer's `await new Promise(r => next = r)` */ + ScrPromise *resume_p; /* the pump's `await new Promise(r => resume = r)` */ + bool started; +} OpMap; + +static void op_map_body_drop(OpGen *g) { + OpMap *m = (OpMap *)g; + src_release(&m->src); + scr_dyn_release(m->fn); + if (m->signal) scr_dyn_release(m->signal); + for (size_t i = m->qh; i < m->qn; i++) scr_dyn_release(m->queue[i]); + free(m->queue); + if (m->next_p) scr_promise_release(m->next_p); + if (m->resume_p) scr_promise_release(m->resume_p); + free(m); +} + +static void map_push(OpMap *m, ScrDyn *v /*moves*/) { + if (m->qn == m->qcap) { + if (m->qh > 0) { + memmove(m->queue, m->queue + m->qh, (m->qn - m->qh) * sizeof *m->queue); + m->qn -= m->qh; + m->qh = 0; + } + if (m->qn == m->qcap) { + m->qcap = m->qcap ? m->qcap * 2 : 4; + m->queue = realloc(m->queue, m->qcap * sizeof *m->queue); + if (!m->queue) op_oom(); + } + } + m->queue[m->qn++] = v; +} + +static size_t map_qlen(const OpMap *m) { return m->qn - m->qh; } + +/* resolve() of the pending resume/next promise */ +static void map_wake(ScrPromise **slot) { + if (!*slot) return; + ScrPromise *p = *slot; + *slot = NULL; + op_fulfill(p, scr_dyn_undefined()); + scr_promise_release(p); +} + +static void map_maybe_resume(OpMap *m) { + if (m->resume_p && !m->done && m->cnt < m->concurrency && (double)map_qlen(m) < m->hwm) { + map_wake(&m->resume_p); + } +} + +static void map_after_item(OpMap *m) { + m->cnt -= 1; + map_maybe_resume(m); +} + +/* PromisePrototypeThen(val, afterItemProcessed, onCatch) */ +static void map_item_settled_k(void *self, ScrDyn *v) { + OpMap *m = self; + if (v) { + scr_dyn_release(v); + } else { + scr_exc_clear(); + m->done = true; + } + map_after_item(m); +} + +/* ── the async wrappers around the user's callback ── */ + +/* An async function's promise settled from an awaited inner value. */ +typedef struct { + OpObj base; + ScrPromise *p; /* the wrapper's promise */ + ScrDyn *chunk; /* the source value (filter returns it) */ + int mode; + int stage; /* MODE_EVERY: 0 the negating arrow, 1 filterFn */ +} OpWrap; + +static void op_wrap_drop(OpObj *o) { + OpWrap *w = (OpWrap *)o; + scr_promise_release(w->p); + if (w->chunk) scr_dyn_release(w->chunk); + free(w); +} + +static OpWrap *op_wrap_new(int mode, ScrDyn *chunk, int stage) { + OpWrap *w = op_alloc(sizeof *w); + w->base.rc = 1; + w->base.drop = &op_wrap_drop; + w->p = scr_promise_new(); + w->chunk = chunk ? scr_dyn_retain(chunk) : NULL; + w->mode = mode; + w->stage = stage; + return w; +} + +static void op_wrap_k(void *self, ScrDyn *v) { + OpWrap *w = self; + if (!v) { + scr_promise_reject_pending(w->p); + return; + } + ScrDyn *out; + if (w->mode == MODE_FOREACH) { + out = scr_dyn_retain(op_k_empty); + } else if (w->mode == MODE_EVERY && w->stage == 0) { + out = scr_dyn_new_bool(!scr_dyn_truthy(v)); + } else { + out = scr_dyn_retain(scr_dyn_truthy(v) ? w->chunk : op_k_empty); + } + scr_dyn_release(v); + op_fulfill(w->p, out); + scr_dyn_release(out); +} + +/* Calls fn(chunk): +1 result, or NULL with the exception pending. */ +static ScrDyn *op_call(ScrDyn *fn, ScrDyn *chunk) { + ScrDyn *args[1] = {chunk}; + return scr_dyn_call(fn, args, 1, "fn"); +} + +/* An async wrapper body: `await fn(chunk)` then its transform. The + * wrapper's promise rejects at once when fn throws synchronously. +1. */ +static ScrPromise *op_wrap_call(OpMap *m, ScrDyn *chunk, int mode, int stage) { + OpWrap *w = op_wrap_new(mode, chunk, stage); + ScrPromise *result = scr_promise_retain(w->p); + ScrDyn *r = op_call(m->fn, chunk); + if (!r) { + scr_promise_reject_pending(w->p); + } else { + op_await_value(r, &op_wrap_k, w); + scr_dyn_release(r); + } + op_unref(w); + return result; +} + +/* every's filterFn: `await predicate(value)` where predicate is the + * negating async arrow, then filterFn's own test. */ +static ScrPromise *op_every_call(OpMap *m, ScrDyn *chunk) { + ScrPromise *inner = op_wrap_call(m, chunk, MODE_EVERY, 0); + OpWrap *w = op_wrap_new(MODE_EVERY, chunk, 1); + ScrPromise *result = scr_promise_retain(w->p); + op_await(inner, &op_wrap_k, w); + scr_promise_release(inner); + op_unref(w); + return result; +} + +/* val = fn(val, signalOpt), then PromiseResolve(val) — or + * PromiseReject(err) when the call throws. +1 dyn promise. */ +static ScrDyn *map_item(OpMap *m, ScrDyn *chunk) { + ScrPromise *p; + if (m->mode == MODE_MAP) { + ScrDyn *r = op_call(m->fn, chunk); + if (!r) { + ScrCaught *c = scr_exc_take(); + ScrDyn *d = op_rejected_dyn(c); + scr_caught_release(c); + return d; + } + if (r->kind == SCR_DYN_PROMISE) return r; + p = op_resolved(r); + scr_dyn_release(r); + } else if (m->mode == MODE_EVERY) { + p = op_every_call(m, chunk); + } else { + p = op_wrap_call(m, chunk, m->mode, 0); + } + ScrDyn *d = scr_dyn_new_promise(p); + scr_promise_release(p); + return d; +} + +/* ── the pump: `for await (let val of stream)` filling the queue ── */ + +static void map_pump_next(OpMap *m); + +static void map_pump_finally(OpMap *m) { + m->done = true; + map_wake(&m->next_p); +} + +/* catch (err): a rejected entry the consumer will throw. Borrows c. */ +static void map_pump_catch(OpMap *m, const ScrCaught *c) { + ScrDyn *val = op_rejected_dyn(c); + op_await_value(val, &map_item_settled_k, m); + map_push(m, val); + map_pump_finally(m); +} + +/* After the loop's AsyncIteratorClose: a return (`if (done) return`) or + * the AbortError thrown in the body. */ +typedef struct { + OpObj base; + OpMap *m; + ScrCaught *thrown; /* +1, or NULL for the return */ +} OpPumpClose; + +static void op_pump_close_drop(OpObj *o) { + OpPumpClose *c = (OpPumpClose *)o; + op_unref(c->m); + if (c->thrown) scr_caught_release(c->thrown); + free(c); +} + +static void op_pump_close_k(void *self, ScrDyn *v) { + OpPumpClose *c = self; + if (v) scr_dyn_release(v); + else scr_exc_clear(); /* a throw completion keeps the original error */ + if (c->thrown) map_pump_catch(c->m, c->thrown); + else map_pump_finally(c->m); +} + +static void map_pump_close(OpMap *m, ScrCaught *thrown /*moves or NULL*/) { + OpPumpClose *c = op_alloc(sizeof *c); + c->base.rc = 1; + c->base.drop = &op_pump_close_drop; + c->m = op_ref(m); + c->thrown = thrown; + ScrPromise *p = src_return(&m->src); + op_await(p, &op_pump_close_k, c); + scr_promise_release(p); + op_unref(c); +} + +static void map_pump_resumed_k(void *self, ScrDyn *v) { + if (v) scr_dyn_release(v); + map_pump_next(self); +} + +static void map_pump_val_k(void *self, ScrDyn *v) { + OpMap *m = self; + if (!v) { + ScrCaught *c = scr_exc_take(); + map_pump_catch(m, c); + scr_caught_release(c); + return; + } + if (src_done(&m->src, v)) { + scr_dyn_release(v); + map_push(m, scr_dyn_retain(op_k_eof)); + map_pump_finally(m); + return; + } + if (m->done) { + scr_dyn_release(v); + map_pump_close(m, NULL); + return; + } + if (op_aborted(m->signal)) { + scr_dyn_release(v); + map_pump_close(m, op_take_error(op_abort_error(NULL))); + return; + } + ScrDyn *val = map_item(m, v); + scr_dyn_release(v); + m->cnt += 1; + op_await_value(val, &map_item_settled_k, m); + map_push(m, val); + map_wake(&m->next_p); + if (!m->done && ((double)map_qlen(m) >= m->hwm || m->cnt >= m->concurrency)) { + m->resume_p = scr_promise_new(); + op_await(m->resume_p, &map_pump_resumed_k, m); + return; + } + map_pump_next(m); +} + +static void map_pump_next(OpMap *m) { + ScrPromise *p = src_next(&m->src); + op_await(p, &map_pump_val_k, m); + scr_promise_release(p); +} + +/* ── the generator body: awaiting queue[0] and yielding ── */ + +static void map_consume(OpMap *m); + +/* finally { done = true; if (resume) { resume(); resume = null; } } */ +static void map_finally(OpMap *m) { + m->done = true; + map_wake(&m->resume_p); +} + +static void map_cons_next_k(void *self, ScrDyn *v) { + if (v) scr_dyn_release(v); + map_consume(self); +} + +static void map_cons_val_k(void *self, ScrDyn *v) { + OpMap *m = self; + OpGen *g = &m->g; + if (!v) { + ScrCaught *c = scr_exc_take(); + map_finally(m); + gen_complete(g, c); + scr_caught_release(c); + return; + } + if (v == op_k_eof) { + scr_dyn_release(v); + map_finally(m); + gen_complete(g, NULL); + return; + } + if (op_aborted(m->signal)) { + scr_dyn_release(v); + ScrCaught *c = op_take_error(op_abort_error(NULL)); + map_finally(m); + gen_complete(g, c); + scr_caught_release(c); + return; + } + if (v != op_k_empty) { + gen_yield(g, v); + scr_dyn_release(v); + return; + } + scr_dyn_release(v); + scr_dyn_release(m->queue[m->qh++]); + map_maybe_resume(m); + map_consume(m); +} + +static void map_consume(OpMap *m) { + if (map_qlen(m) > 0) { + op_await_value(m->queue[m->qh], &map_cons_val_k, m); + return; + } + m->next_p = scr_promise_new(); + op_await(m->next_p, &map_cons_next_k, m); +} + +/* Resumed after a yield, or started: the first resumption starts the pump + * (pump() runs before the consumer loop), a later one shifts the entry + * that was yielded. */ +static void map_resume(OpGen *g) { + OpMap *m = (OpMap *)g; + if (!m->started) { + m->started = true; + map_pump_next(m); + } else { + scr_dyn_release(m->queue[m->qh++]); + map_maybe_resume(m); + } + map_consume(m); +} + +/* A return or throw delivered at the yield: the finally block, then the + * generator completes. */ +static void map_ret(OpGen *g, ScrDyn *v) { + (void)v; + map_finally((OpMap *)g); + gen_complete(g, NULL); +} + +static void map_thr(OpGen *g, ScrCaught *c) { + map_finally((OpMap *)g); + gen_complete(g, c); +} + +static OpMap *map_new(ScrStream *s, ScrDyn *fn, int mode, double concurrency, double hwm, + ScrDyn *signal) { + OpMap *m = op_alloc(sizeof *m); + gen_init(&m->g, &op_map_body_drop); + m->g.resume = &map_resume; + m->g.ret = &map_ret; + m->g.thr = &map_thr; + m->src.s = scr_stream_retain(s); + m->fn = scr_dyn_retain(fn); + m->signal = signal ? scr_dyn_retain(signal) : NULL; + m->mode = mode; + m->concurrency = concurrency; + m->hwm = hwm; + return m; +} + +/* ── drop and take ─────────────────────────────────────────────────── */ + +/* async function* drop() / take(): `for await (const val of this)` with + * the signal polled before the loop and at every value; take returns once + * its count is spent, closing the source (which destroys it). */ +typedef struct { + OpGen g; + OpSrc src; + ScrDyn *signal; /* +1 or NULL */ + double number; + bool take; + bool started; +} OpSlice; + +static void op_slice_body_drop(OpGen *g) { + OpSlice *t = (OpSlice *)g; + src_release(&t->src); + if (t->signal) scr_dyn_release(t->signal); + free(t); +} + +static void slice_next(OpSlice *t); + +/* AsyncIteratorClose(source), then the generator completes (with the + * thrown exception when one is given). */ +typedef struct { + OpObj base; + OpGen *g; + ScrCaught *thrown; /* +1 or NULL */ +} OpGenClose; + +static void op_gen_close_drop(OpObj *o) { + OpGenClose *c = (OpGenClose *)o; + op_unref(c->g); + if (c->thrown) scr_caught_release(c->thrown); + free(c); +} + +static void op_gen_close_k(void *self, ScrDyn *v) { + OpGenClose *c = self; + if (v) { + scr_dyn_release(v); + gen_complete(c->g, c->thrown); + return; + } + /* A rejected return() replaces a normal completion; a throw keeps its + * own exception. */ + ScrCaught *err = scr_exc_take(); + gen_complete(c->g, c->thrown ? c->thrown : err); + scr_caught_release(err); +} + +static void gen_close_source(OpGen *g, OpSrc *src, ScrCaught *thrown /*moves or NULL*/) { + OpGenClose *c = op_alloc(sizeof *c); + c->base.rc = 1; + c->base.drop = &op_gen_close_drop; + c->g = op_ref(g); + c->thrown = thrown; + ScrPromise *p = src_return(src); + op_await(p, &op_gen_close_k, c); + scr_promise_release(p); + op_unref(c); +} + +static void slice_val_k(void *self, ScrDyn *v) { + OpSlice *t = self; + if (!v) { + ScrCaught *c = scr_exc_take(); + gen_complete(&t->g, c); + scr_caught_release(c); + return; + } + if (src_done(&t->src, v)) { + scr_dyn_release(v); + gen_complete(&t->g, NULL); + return; + } + if (op_aborted(t->signal)) { + scr_dyn_release(v); + gen_close_source(&t->g, &t->src, op_take_error(op_abort_error(NULL))); + return; + } + bool yield; + if (t->take) { + yield = t->number-- > 0; + } else { + yield = t->number-- <= 0; + } + if (yield) { + gen_yield(&t->g, v); + scr_dyn_release(v); + return; + } + scr_dyn_release(v); + if (t->take && t->number <= 0) { + gen_close_source(&t->g, &t->src, NULL); + return; + } + slice_next(t); +} + +static void slice_next(OpSlice *t) { + ScrPromise *p = src_next(&t->src); + op_await(p, &slice_val_k, t); + scr_promise_release(p); +} + +static void slice_resume(OpGen *g) { + OpSlice *t = (OpSlice *)g; + if (!t->started) { + t->started = true; + if (op_aborted(t->signal)) { + ScrCaught *c = op_take_error(op_abort_error(NULL)); + gen_complete(g, c); + scr_caught_release(c); + return; + } + } else if (t->take && t->number <= 0) { + /* `if (number <= 0) return;` after the yield */ + gen_close_source(g, &t->src, NULL); + return; + } + slice_next(t); +} + +static void slice_ret(OpGen *g, ScrDyn *v) { + (void)v; + gen_close_source(g, &((OpSlice *)g)->src, NULL); +} + +static void slice_thr(OpGen *g, ScrCaught *c) { + gen_close_source(g, &((OpSlice *)g)->src, scr_caught_retain(c)); +} + +/* ── flatMap ───────────────────────────────────────────────────────── */ + +/* async function* flatMap() { for await (const val of values) yield* val; } + * over map's generator. yield* over an array (or a string's code points) + * goes through the async-from-sync iterator: each step resolves after the + * element's PromiseResolve job, and yield* awaits that result before + * yielding without a further await. */ +typedef struct { + OpGen g; + OpSrc values; /* the map generator */ + ScrDyn *inner; /* +1: the array or string being delegated to, or NULL */ + size_t idx; /* element index, or the string's byte offset */ + ScrPromise *step; /* the async-from-sync iterator result being awaited */ + bool started; +} OpFlat; + +static void op_flat_body_drop(OpGen *g) { + OpFlat *f = (OpFlat *)g; + src_release(&f->values); + if (f->inner) scr_dyn_release(f->inner); + if (f->step) scr_promise_release(f->step); + free(f); +} + +static void flat_next_value(OpFlat *f); + +/* The next element (+1), or NULL when the delegate is exhausted. */ +static ScrDyn *flat_take_element(OpFlat *f) { + ScrDyn *in = f->inner; + if (in->kind == SCR_DYN_ARR) { + if (f->idx >= in->v.arr.len) return NULL; + ScrDyn *e = in->v.arr.items[f->idx++]; + return e ? scr_dyn_retain(e) : scr_dyn_retain(scr_dyn_undefined()); + } + /* a string iterates by code point */ + const ScrStr *s = in->v.str; + if (f->idx >= s->len) return NULL; + size_t i = f->idx; + unsigned char c = (unsigned char)s->data[i]; + size_t n = c < 0x80 ? 1 : c < 0xE0 ? 2 : c < 0xF0 ? 3 : 4; + if (i + n > s->len) n = s->len - i; + f->idx += n; + ScrStr *piece = scr_str_new(s->data + i, n); + ScrDyn *d = scr_dyn_new_str(piece); + scr_str_release(piece); + return d; +} + +/* Await(innerResult): the iterator result is ready one job later. */ +static void flat_step_k(void *self, ScrDyn *v) { + OpFlat *f = self; + if (f->step) { + scr_promise_release(f->step); + f->step = NULL; + } + if (v == op_k_done || (v && v == op_k_eof)) { + scr_dyn_release(v); + scr_dyn_release(f->inner); + f->inner = NULL; + flat_next_value(f); + return; + } + gen_yield_now(&f->g, v); + scr_dyn_release(v); +} + +/* The async-from-sync continuation: PromiseResolve(value) then the result + * promise settles; yield* awaits that promise. */ +typedef struct { + OpObj base; + ScrPromise *p; + bool done; +} OpFlatWrap; + +static void op_flat_wrap_drop(OpObj *o) { + OpFlatWrap *w = (OpFlatWrap *)o; + scr_promise_release(w->p); + free(w); +} + +static void op_flat_wrap_k(void *self, ScrDyn *v) { + OpFlatWrap *w = self; + if (!v) { + scr_promise_reject_pending(w->p); + return; + } + op_fulfill(w->p, w->done ? op_k_done : v); + scr_dyn_release(v); +} + +static void flat_delegate_step(OpFlat *f) { + ScrDyn *e = flat_take_element(f); + OpFlatWrap *w = op_alloc(sizeof *w); + w->base.rc = 1; + w->base.drop = &op_flat_wrap_drop; + w->p = scr_promise_new(); + w->done = e == NULL; + f->step = scr_promise_retain(w->p); + op_await_value(e ? e : scr_dyn_undefined(), &op_flat_wrap_k, w); + if (e) scr_dyn_release(e); + op_await(w->p, &flat_step_k, f); + op_unref(w); +} + +static void flat_val_k(void *self, ScrDyn *v) { + OpFlat *f = self; + if (!v) { + ScrCaught *c = scr_exc_take(); + gen_complete(&f->g, c); + scr_caught_release(c); + return; + } + if (src_done(&f->values, v)) { + scr_dyn_release(v); + gen_complete(&f->g, NULL); + return; + } + if (v->kind != SCR_DYN_ARR && v->kind != SCR_DYN_STR) { + scr_dyn_release(v); + ScrCaught *c = op_take_error( + op_unsupported("flatMap callback results other than arrays or strings")); + gen_close_source(&f->g, &f->values, c); + return; + } + f->inner = v; + f->idx = 0; + flat_delegate_step(f); +} + +static void flat_next_value(OpFlat *f) { + ScrPromise *p = src_next(&f->values); + op_await(p, &flat_val_k, f); + scr_promise_release(p); +} + +static void flat_resume(OpGen *g) { + OpFlat *f = (OpFlat *)g; + if (!f->started) { + f->started = true; + flat_next_value(f); + return; + } + flat_delegate_step(f); +} + +/* A return during delegation: the async-from-sync iterator's return() + * resolves at once (arrays have no return method), yield* awaits it and + * then the return value, and the return completion leaves the for await, + * closing map's generator. */ +typedef struct { + OpObj base; + OpFlat *f; + int stage; +} OpFlatRet; + +static void op_flat_ret_drop(OpObj *o) { + OpFlatRet *r = (OpFlatRet *)o; + op_unref(r->f); + free(r); +} + +static void op_flat_ret_k(void *self, ScrDyn *v) { + OpFlatRet *r = self; + if (v) scr_dyn_release(v); + else scr_exc_clear(); + if (r->stage == 0) { + r->stage = 1; + op_await_value(scr_dyn_undefined(), &op_flat_ret_k, r); + return; + } + if (r->f->inner) { + scr_dyn_release(r->f->inner); + r->f->inner = NULL; + } + gen_close_source(&r->f->g, &r->f->values, NULL); +} + +static void flat_ret(OpGen *g, ScrDyn *v) { + (void)v; + OpFlatRet *r = op_alloc(sizeof *r); + r->base.rc = 1; + r->base.drop = &op_flat_ret_drop; + r->f = op_ref((OpFlat *)g); + op_await_value(scr_dyn_undefined(), &op_flat_ret_k, r); + op_unref(r); +} + +/* A throw during delegation: the array iterator has no throw method, so + * yield* rejects with Node's TypeError, which leaves the for await and + * closes map's generator. */ +static void flat_thr(OpGen *g, ScrCaught *c) { + (void)c; + OpFlat *f = (OpFlat *)g; + if (f->inner) { + scr_dyn_release(f->inner); + f->inner = NULL; + } + ScrError *e = op_coded_error(SCR_ERR_TYPE, NULL, "The iterator does not provide a 'throw' method."); + gen_close_source(g, &f->values, op_take_error(e)); +} + +/* ── Readable.from(generator) ──────────────────────────────────────── */ + +/* The operator's output stream: from.js's nextAsync pulls the generator + * while push() accepts, and _destroy runs close(error) (the generator's + * throw, then return) before the destroy callback, on a nextTick. */ +typedef struct { + OpObj base; + OpGen *g; /* +1 */ + ScrStream *out; /* borrowed: the stream owns this object */ + bool reading; +} OpFrom; + +static void op_from_drop(OpObj *o) { + OpFrom *f = (OpFrom *)o; + op_unref(f->g); + free(f); +} + +/* One pull: the stream is held for the await. */ +typedef struct { + OpObj base; + OpFrom *from; + ScrStream *out; /* +1 */ +} OpFromStep; + +static void op_from_step_drop(OpObj *o) { + OpFromStep *st = (OpFromStep *)o; + op_unref(st->from); + scr_stream_release(st->out); + free(st); +} + +static void from_next(OpFrom *f, ScrStream *out); + +/* An operator error as a stream error: Error values as themselves, any + * other thrown value refused. +1. */ +static ScrError *op_caught_error(const ScrCaught *c) { + ScrDyn *d = scr_caught_to_dyn(c); + ScrError *e = d ? scr_errdyn_err_of(d) : NULL; + if (d) scr_dyn_release(d); + if (e) return e; + return op_unsupported("a stream operator rejection that is not an Error object"); +} + +static void op_from_destroy_with(ScrStream *out, ScrError *e /*moves*/) { + scr_stream_release(scr_stream_destroy(out, e)); + scr_error_release(e); +} + +static void op_from_step_k(void *self, ScrDyn *v) { + OpFromStep *st = self; + OpFrom *f = st->from; + ScrStream *out = st->out; + if (!v) { + ScrCaught *c = scr_exc_take(); + op_from_destroy_with(out, op_caught_error(c)); + scr_caught_release(c); + return; + } + if (v == op_k_done) { + scr_dyn_release(v); + scr_stream_push_null(out); + return; + } + if (v->kind == SCR_DYN_NULL) { + scr_dyn_release(v); + f->reading = false; + op_from_destroy_with(out, op_coded_error(SCR_ERR_TYPE, "ERR_STREAM_NULL_VALUES", + "May not write null values to stream")); + return; + } + int pushed = scr_stream_push_object(out, v); + scr_dyn_release(v); + if (pushed == -2) { + op_from_destroy_with(out, op_unsupported( + "a stream operator output that mixes strings and Buffers")); + return; + } + if (pushed < 0) { + op_from_destroy_with(out, op_unsupported( + "a stream operator value that is not a string or Buffer (object-mode streams of other values)")); + return; + } + if (pushed) { + from_next(f, out); + return; + } + f->reading = false; +} + +static void from_next(OpFrom *f, ScrStream *out) { + OpFromStep *st = op_alloc(sizeof *st); + st->base.rc = 1; + st->base.drop = &op_from_step_drop; + st->from = op_ref(f); + st->out = scr_stream_retain(out); + ScrPromise *p = gen_next(f->g); + op_await(p, &op_from_step_k, st); + scr_promise_release(p); + op_unref(st); +} + +static OpFrom *op_from_of(ScrClosure *cb) { return scr_box_get_ref(cb->caps[0]); /* +1 */ } + +/* readable._read */ +static void op_from_read_inv(ScrClosure *cb, ScrStream *s, double size) { + (void)size; + OpFrom *f = op_from_of(cb); + if (!f->reading) { + f->reading = true; + from_next(f, s); + } + op_unref(f); +} + +/* close(error): `await iterator.throw(error)` (when there is an error), + * `await value`, then `await iterator.return()` and `await value`; the + * destroy callback follows on a nextTick from the promise's reaction. */ +typedef struct { + OpObj base; + OpFrom *from; + ScrStream *out; /* +1 */ + ScrError *error; /* +1, the destroy error, or NULL */ + ScrError *fail; /* +1, close()'s rejection */ + int stage; +} OpFromClose; + +static void op_from_close_drop(OpObj *o) { + OpFromClose *c = (OpFromClose *)o; + op_unref(c->from); + scr_stream_release(c->out); + if (c->error) scr_error_release(c->error); + if (c->fail) scr_error_release(c->fail); + free(c); +} + +static void op_from_cb_tick(ScrClosure *job) { + OpFromClose *c = scr_box_get_ref(job->caps[0]); + ScrError *e = c->fail ? c->fail : c->error; + scr_stream_destroy_done(c->out, e ? scr_error_retain(e) : NULL); + op_unref(c); +} + +static void op_from_close_settled(OpFromClose *c) { + /* PromisePrototypeThen(close(error), ...): the reaction job, then + * process.nextTick(cb, ...) */ + ScrClosure *job = scr_closure_new((void *)&op_from_cb_tick, 1); + job->caps[0] = scr_box_new_obj(&op_ref, &op_unref, NULL); + scr_box_set_ref(job->caps[0], op_ref(c)); + scr_next_tick(job); +} + +static void op_from_close_k(void *self, ScrDyn *v); + +static void op_from_close_step(OpFromClose *c, ScrPromise *p) { + op_await(p, &op_from_close_k, c); + scr_promise_release(p); +} + +static void op_from_close_k(void *self, ScrDyn *v) { + OpFromClose *c = self; + if (!v) { + ScrCaught *err = scr_exc_take(); + c->fail = op_caught_error(err); + scr_caught_release(err); + c->stage = 9; + op_await_value(scr_dyn_undefined(), &op_from_close_k, c); + return; + } + bool done = v == op_k_done; + scr_dyn_release(v); + switch (c->stage) { + case 0: /* await iterator.throw(error) */ + c->stage = done ? 2 : 1; + op_await_value(scr_dyn_undefined(), &op_from_close_k, c); /* await value */ + return; + case 1: /* not done: iterator.return() */ + case 3: + c->stage = 4; + op_from_close_step(c, gen_return(c->from->g, scr_dyn_undefined())); + return; + case 2: /* done after the throw: close() returns */ + case 5: + case 9: + op_from_close_settled(c); + return; + case 4: /* await iterator.return() */ + c->stage = 5; + op_await_value(scr_dyn_undefined(), &op_from_close_k, c); /* await value */ + return; + default: + op_from_close_settled(c); + } +} + +static void op_from_destroy_inv(ScrClosure *cb, ScrStream *s, ScrError *err) { + OpFrom *f = op_from_of(cb); + OpFromClose *c = op_alloc(sizeof *c); + c->base.rc = 1; + c->base.drop = &op_from_close_drop; + c->from = f; /* the +1 from op_from_of moves here */ + c->out = scr_stream_retain(s); + c->error = err ? scr_error_retain(err) : NULL; + if (err) { + ScrCaught *thrown = op_take_error(scr_error_retain(err)); + c->stage = 0; + op_from_close_step(c, gen_throw(f->g, thrown)); + scr_caught_release(thrown); + } else { + c->stage = 4; + op_from_close_step(c, gen_return(f->g, scr_dyn_undefined())); + } + op_unref(c); +} + +static ScrClosure *op_from_closure(void *fn, OpFrom *f) { + ScrClosure *c = scr_closure_new(fn, 1); + c->caps[0] = scr_box_new_obj(&op_ref, &op_unref, NULL); + scr_box_set_ref(c->caps[0], op_ref(f)); + return c; +} + +/* Readable.from(generator): +1 object-mode stream (hwm 1). Moves g. */ +static ScrStream *op_from_gen(OpGen *g) { + OpFrom *f = op_alloc(sizeof *f); + f->base.rc = 1; + f->base.drop = &op_from_drop; + f->g = g; + ScrStream *out = scr_stream_new_object_readable(op_from_closure((void *)&op_from_read_inv, f), + &op_from_read_inv, + op_from_closure((void *)&op_from_destroy_inv, f), + &op_from_destroy_inv); + f->out = out; + op_unref(f); + return out; +} + +/* ── the promise operators ─────────────────────────────────────────── */ + +enum { PK_DYN, PK_F64, PK_BOOL, PK_STR, PK_VOID }; + +/* An async operator function: its promise and the loop state. */ +typedef struct { + OpObj base; + ScrPromise *p; + OpSrc src; + ScrDyn *fn; /* +1 or NULL */ + ScrDyn *signal; /* +1 or NULL */ + ScrDyn *acc; /* toArray's array / reduce's accumulator (+1) */ + int op; + int kind; /* the promise payload (PK_*) */ + bool has_init, got_any, negate; +} OpFn; + +enum { FN_TO_ARRAY, FN_REDUCE, FN_SOME, FN_FIND, FN_EVERY, FN_FOR_EACH }; + +static void op_fn_drop(OpObj *o) { + OpFn *a = (OpFn *)o; + scr_promise_release(a->p); + src_release(&a->src); + if (a->fn) scr_dyn_release(a->fn); + if (a->signal) scr_dyn_release(a->signal); + if (a->acc) scr_dyn_release(a->acc); + free(a); +} + +static OpFn *op_fn_new(int op, int kind) { + OpFn *a = op_alloc(sizeof *a); + a->base.rc = 1; + a->base.drop = &op_fn_drop; + a->p = scr_promise_new(); + a->op = op; + a->kind = kind; + return a; +} + +/* Settles the operator's promise with v converted to its payload kind (a + * value of the wrong kind rejects with the checked conversion's error). */ +static void op_fn_resolve(OpFn *a, ScrDyn *v /*borrowed*/) { + switch (a->kind) { + case PK_VOID: + scr_promise_fulfill_void(a->p); + return; + case PK_BOOL: + scr_promise_fulfill_bool(a->p, scr_dyn_truthy(v)); + return; + case PK_F64: + if (v->kind != SCR_DYN_NUM) break; + scr_promise_fulfill_f64(a->p, v->v.num); + return; + case PK_STR: + if (v->kind != SCR_DYN_STR) break; + scr_promise_fulfill_str(a->p, scr_str_retain(v->v.str)); + return; + default: + op_fulfill(a->p, v); + return; + } + static const char m[] = "reduce result does not match its declared type"; + scr_throw_error_msg(SCR_ERR_TYPE, m, sizeof m - 1); + scr_promise_reject_pending(a->p); +} + +static void op_fn_reject(OpFn *a, const ScrCaught *c) { op_reject(a->p, c); } + +static void op_fn_next(OpFn *a); + +/* After the loop's AsyncIteratorClose: the throw that left the loop, or + * the early return's result. */ +typedef struct { + OpObj base; + OpFn *a; + ScrCaught *thrown; /* +1 or NULL */ + ScrDyn *result; /* +1 or NULL */ +} OpFnClose; + +static void op_fn_close_drop(OpObj *o) { + OpFnClose *c = (OpFnClose *)o; + op_unref(c->a); + if (c->thrown) scr_caught_release(c->thrown); + if (c->result) scr_dyn_release(c->result); + free(c); +} + +static void op_fn_finish(OpFn *a, ScrDyn *result); + +static void op_fn_close_k(void *self, ScrDyn *v) { + OpFnClose *c = self; + if (v) { + scr_dyn_release(v); + } else if (!c->thrown) { + ScrCaught *err = scr_exc_take(); + op_fn_reject(c->a, err); + scr_caught_release(err); + return; + } else { + scr_exc_clear(); + } + if (c->thrown) op_fn_reject(c->a, c->thrown); + else op_fn_finish(c->a, c->result); +} + +static void op_fn_close(OpFn *a, ScrCaught *thrown /*moves*/, ScrDyn *result /*moves*/) { + OpFnClose *c = op_alloc(sizeof *c); + c->base.rc = 1; + c->base.drop = &op_fn_close_drop; + c->a = op_ref(a); + c->thrown = thrown; + c->result = result; + ScrPromise *p = src_return(&a->src); + op_await(p, &op_fn_close_k, c); + scr_promise_release(p); + op_unref(c); +} + +/* every: `!(await some(...))` — one more job after some's result. */ +static void op_every_k(void *self, ScrDyn *v) { + OpFn *a = self; + if (!v) { + scr_promise_reject_pending(a->p); + return; + } + scr_promise_fulfill_bool(a->p, !scr_dyn_truthy(v)); + scr_dyn_release(v); +} + +static void op_fn_finish(OpFn *a, ScrDyn *result) { + op_fn_resolve(a, result ? result : scr_dyn_undefined()); +} + +static void op_reduce_value_k(void *self, ScrDyn *v) { + OpFn *a = self; + if (!v) { + /* the rejection leaves the for await: AsyncIteratorClose */ + op_fn_close(a, scr_exc_take(), NULL); + return; + } + scr_dyn_release(a->acc); + a->acc = v; + op_fn_next(a); +} + +static void op_fn_val_k(void *self, ScrDyn *v) { + OpFn *a = self; + if (!v) { + ScrCaught *c = scr_exc_take(); + op_fn_reject(a, c); + scr_caught_release(c); + return; + } + if (src_done(&a->src, v)) { + scr_dyn_release(v); + switch (a->op) { + case FN_TO_ARRAY: + case FN_REDUCE: + if (a->op == FN_REDUCE && !a->got_any && !a->has_init) { + ScrError *e = op_coded_error(SCR_ERR_TYPE, "ERR_MISSING_ARGS", + "Reduce of an empty stream requires an initial value"); + ScrCaught *c = op_take_error(e); + op_fn_reject(a, c); + scr_caught_release(c); + return; + } + op_fn_finish(a, a->acc); + return; + case FN_SOME: + case FN_EVERY: { + ScrDyn *f = scr_dyn_new_bool(false); + op_fn_finish(a, f); + scr_dyn_release(f); + return; + } + default: + op_fn_finish(a, NULL); + return; + } + } + switch (a->op) { + case FN_TO_ARRAY: + if (op_aborted(a->signal)) { + ScrDyn *reason = op_get(a->signal, "reason"); + ScrError *e = op_abort_error(reason ? reason : scr_dyn_undefined()); + if (reason) scr_dyn_release(reason); + scr_dyn_release(v); + op_fn_close(a, op_take_error(e), NULL); + return; + } + scr_dyn_arr_push(a->acc, v); /* moves v */ + op_fn_next(a); + return; + case FN_REDUCE: { + a->got_any = true; + if (op_aborted(a->signal)) { + scr_dyn_release(v); + op_fn_close(a, op_take_error(op_abort_error(NULL)), NULL); + return; + } + if (!a->has_init) { + scr_dyn_release(a->acc); + a->acc = v; + a->has_init = true; + op_fn_next(a); + return; + } + ScrDyn *args[2] = {a->acc, v}; + ScrDyn *r = scr_dyn_call(a->fn, args, 2, "reducer"); + scr_dyn_release(v); + if (!r) { + op_fn_close(a, scr_exc_take(), NULL); + return; + } + op_await_value(r, &op_reduce_value_k, a); + scr_dyn_release(r); + return; + } + case FN_SOME: + case FN_EVERY: { + scr_dyn_release(v); + op_fn_close(a, NULL, scr_dyn_new_bool(true)); + return; + } + case FN_FIND: + op_fn_close(a, NULL, v); + return; + default: /* forEach never receives a value */ + scr_dyn_release(v); + op_fn_next(a); + return; + } +} + +static void op_fn_next(OpFn *a) { + ScrPromise *p = src_next(&a->src); + op_await(p, &op_fn_val_k, a); + scr_promise_release(p); +} + +/* ── validation and entry points ───────────────────────────────────── */ + +/* map()'s option reading: concurrency, highWaterMark (MathFloor, then + * validateInteger). False with the error pending. */ +static bool op_map_options(const ScrDyn *options, double *concurrency, double *hwm) { + *concurrency = 1; + ScrDyn *c = op_get(options, "concurrency"); + if (!op_nullish(c)) { + double n = scr_dyn_number_coerce(c); + if (scr_exc_pending()) { + scr_dyn_release(c); + return false; + } + *concurrency = floor(n); + } + if (c) scr_dyn_release(c); + *hwm = *concurrency - 1; + ScrDyn *h = op_get(options, "highWaterMark"); + if (!op_nullish(h)) { + double n = scr_dyn_number_coerce(h); + if (scr_exc_pending()) { + scr_dyn_release(h); + return false; + } + *hwm = floor(n); + } + if (h) scr_dyn_release(h); + const double max = 9007199254740991.0; + if (!isfinite(*concurrency) || floor(*concurrency) != *concurrency) { + op_throw_out_of_range("options.concurrency", "an integer", *concurrency); + return false; + } + if (*concurrency < 1 || *concurrency > max) { + op_throw_out_of_range("options.concurrency", ">= 1 && <= 9007199254740991", *concurrency); + return false; + } + if (!isfinite(*hwm) || floor(*hwm) != *hwm) { + op_throw_out_of_range("options.highWaterMark", "an integer", *hwm); + return false; + } + if (*hwm < 0 || *hwm > max) { + op_throw_out_of_range("options.highWaterMark", ">= 0 && <= 9007199254740991", *hwm); + return false; + } + *hwm += *concurrency; + return true; +} + +/* map(fn, options) with fn already checked: +1 generator, or NULL with + * the error pending. */ +static OpMap *op_map_make(ScrStream *s, ScrDyn *fn, ScrDyn *options, int mode) { + op_sentinels_init(); + if (!op_check_options(options)) return NULL; + bool ok; + ScrDyn *signal = op_signal(options, &ok); + if (!ok) return NULL; + double concurrency, hwm; + if (!op_map_options(options, &concurrency, &hwm)) { + if (signal) scr_dyn_release(signal); + return NULL; + } + OpMap *m = map_new(s, fn, mode, concurrency, hwm, signal); + if (signal) scr_dyn_release(signal); + return m; +} + +ScrStream *scr_stream_op_map(ScrStream *s, ScrDyn *fn, ScrDyn *options) { + if (!op_check_fn(fn, "fn", 1)) return NULL; + OpMap *m = op_map_make(s, fn, options, MODE_MAP); + return m ? op_from_gen(&m->g) : NULL; +} + +ScrStream *scr_stream_op_filter(ScrStream *s, ScrDyn *fn, ScrDyn *options) { + if (!op_check_fn(fn, "fn", 1)) return NULL; + OpMap *m = op_map_make(s, fn, options, MODE_FILTER); + return m ? op_from_gen(&m->g) : NULL; +} + +ScrStream *scr_stream_op_flat_map(ScrStream *s, ScrDyn *fn, ScrDyn *options) { + if (!op_check_fn(fn, "fn", 1)) return NULL; + OpMap *m = op_map_make(s, fn, options, MODE_MAP); + if (!m) return NULL; + OpFlat *f = op_alloc(sizeof *f); + gen_init(&f->g, &op_flat_body_drop); + f->g.resume = &flat_resume; + f->g.ret = &flat_ret; + f->g.thr = &flat_thr; + f->values.g = &m->g; /* moves */ + return op_from_gen(&f->g); +} + +/* drop(number, options) / take(number, options) */ +static ScrStream *op_slice(ScrStream *s, ScrDyn *number, ScrDyn *options, bool take) { + op_sentinels_init(); + if (!op_check_options(options)) return NULL; + bool ok; + ScrDyn *signal = op_signal(options, &ok); + if (!ok) return NULL; + /* toIntegerOrInfinity: Number(number), NaN → 0, negative → out of range */ + double n = scr_dyn_number_constructor(number ? number : scr_dyn_undefined()); + if (scr_exc_pending()) { + if (signal) scr_dyn_release(signal); + return NULL; + } + if (isnan(n)) n = 0; + if (n < 0) { + op_throw_out_of_range("number", ">= 0", n); + if (signal) scr_dyn_release(signal); + return NULL; + } + OpSlice *t = op_alloc(sizeof *t); + gen_init(&t->g, &op_slice_body_drop); + t->g.resume = &slice_resume; + t->g.ret = &slice_ret; + t->g.thr = &slice_thr; + t->src.s = scr_stream_retain(s); + t->signal = signal; + t->number = n; + t->take = take; + return op_from_gen(&t->g); +} + +ScrStream *scr_stream_op_drop(ScrStream *s, ScrDyn *number, ScrDyn *options) { + return op_slice(s, number, options, false); +} + +ScrStream *scr_stream_op_take(ScrStream *s, ScrDyn *number, ScrDyn *options) { + return op_slice(s, number, options, true); +} + +/* An async operator whose synchronous prefix threw: the promise rejects. */ +static ScrPromise *op_fn_reject_now(OpFn *a) { + scr_promise_reject_pending(a->p); + ScrPromise *p = scr_promise_retain(a->p); + op_unref(a); + return p; +} + +static ScrPromise *op_fn_start(OpFn *a) { + ScrPromise *p = scr_promise_retain(a->p); + op_fn_next(a); + op_unref(a); + return p; +} + +ScrPromise *scr_stream_op_to_array(ScrStream *s, ScrDyn *options) { + op_sentinels_init(); + OpFn *a = op_fn_new(FN_TO_ARRAY, PK_DYN); + if (!op_check_options(options)) return op_fn_reject_now(a); + bool ok; + a->signal = op_signal(options, &ok); + if (!ok) return op_fn_reject_now(a); + a->acc = scr_dyn_new_arr(); + a->src.s = scr_stream_retain(s); + return op_fn_start(a); +} + +/* reduce's pre-aborted path: destroy with the AbortError, wait for the + * stream to finish, then reject with that error. */ +static void op_reduce_aborted_k(void *self, ScrDyn *v) { + OpFn *a = self; + if (v) scr_dyn_release(v); + else scr_exc_clear(); + ScrError *e = scr_stream_errored(a->src.s); + if (!e) e = op_abort_error(NULL); + ScrCaught *c = op_take_error(e); + op_fn_reject(a, c); + scr_caught_release(c); +} + +static void op_noop_listener(ScrClosure *cb, void *err) { + (void)cb; + (void)err; +} + +ScrPromise *scr_stream_op_reduce(ScrStream *s, ScrDyn *fn, ScrDyn *init, bool has_init, + ScrDyn *options, double kind) { + op_sentinels_init(); + OpFn *a = op_fn_new(FN_REDUCE, (int)kind); + if (!op_check_fn(fn, "reducer", 2)) return op_fn_reject_now(a); + if (!op_check_options(options)) return op_fn_reject_now(a); + bool ok; + a->signal = op_signal(options, &ok); + if (!ok) return op_fn_reject_now(a); + a->fn = scr_dyn_retain(fn); + a->has_init = has_init; + a->acc = scr_dyn_retain(init && has_init ? init : scr_dyn_undefined()); + a->src.s = scr_stream_retain(s); + if (op_aborted(a->signal)) { + ScrDyn *reason = op_get(a->signal, "reason"); + ScrError *e = op_abort_error(reason ? reason : scr_dyn_undefined()); + if (reason) scr_dyn_release(reason); + /* this.once('error', () => {}) */ + ScrStr *name = scr_str_new("error", 5); + scr_emitter_release(scr_emitter_on((ScrEmitter *)s, name, + scr_closure_new((void *)&op_noop_listener, 0), + scr_ee_inv_fixed1, true, false)); + scr_str_release(name); + scr_stream_release(scr_stream_destroy(s, e)); + scr_error_release(e); + ScrPromise *fin = scr_sp_finished(s); + ScrPromise *p = scr_promise_retain(a->p); + op_await(fin, &op_reduce_aborted_k, a); + scr_promise_release(fin); + op_unref(a); + return p; + } + return op_fn_start(a); +} + +/* some/find/every/forEach: the loop over map's generator (filter's for + * some, find and every). */ +static ScrPromise *op_gen_fn(ScrStream *s, ScrDyn *fn, ScrDyn *options, int op, int mode, int kind) { + op_sentinels_init(); + OpFn *a = op_fn_new(op, kind); + if (!op_check_fn(fn, "fn", 1)) return op_fn_reject_now(a); + OpMap *m = op_map_make(s, fn, options, mode); + if (!m) return op_fn_reject_now(a); + a->src.g = &m->g; /* moves */ + return op_fn_start(a); +} + +ScrPromise *scr_stream_op_some(ScrStream *s, ScrDyn *fn, ScrDyn *options) { + return op_gen_fn(s, fn, options, FN_SOME, MODE_FILTER, PK_BOOL); +} + +ScrPromise *scr_stream_op_find(ScrStream *s, ScrDyn *fn, ScrDyn *options) { + return op_gen_fn(s, fn, options, FN_FIND, MODE_FILTER, PK_DYN); +} + +ScrPromise *scr_stream_op_for_each(ScrStream *s, ScrDyn *fn, ScrDyn *options) { + return op_gen_fn(s, fn, options, FN_FOR_EACH, MODE_FOREACH, PK_VOID); +} + +/* every(fn, options) = !(await some.call(this, async (...args) => + * !(await fn(...args)), options)) */ +ScrPromise *scr_stream_op_every(ScrStream *s, ScrDyn *fn, ScrDyn *options) { + op_sentinels_init(); + OpFn *outer = op_fn_new(FN_EVERY, PK_BOOL); + if (!op_check_fn(fn, "fn", 1)) return op_fn_reject_now(outer); + ScrPromise *some = op_gen_fn(s, fn, options, FN_EVERY, MODE_EVERY, PK_DYN); + ScrPromise *p = scr_promise_retain(outer->p); + op_await(some, &op_every_k, outer); + scr_promise_release(some); + op_unref(outer); + return p; +} diff --git a/tests/corpus/5335-stream-operators-values.ts b/tests/corpus/5335-stream-operators-values.ts new file mode 100644 index 000000000..be56f32c6 --- /dev/null +++ b/tests/corpus/5335-stream-operators-values.ts @@ -0,0 +1,21 @@ +import { Readable } from "node:stream"; + +const bytes = (): Readable => new Readable({ read() { this.push(Buffer.from("ab")); this.push(Buffer.from("cd")); this.push(null); } }); +console.log(await bytes().toArray()); +console.log(await bytes().map((b: Buffer) => b.toString("hex")).toArray()); +console.log(await bytes().filter((b: Buffer) => b[0] === 0x63).toArray()); +console.log(await bytes().drop(1).toArray(), await bytes().take(1).toArray()); +console.log(await bytes().reduce((n: number, b: Buffer) => n + b.length, 0)); +console.log(await bytes().find((b: Buffer) => b.length === 2), await bytes().every((b: Buffer) => b.length === 2)); +console.log(await Readable.from(["a", "b"]).flatMap((c: string) => [c, c + c]).toArray()); +console.log(await Readable.from(["a", "b"]).map(async (c: string) => c + "!", { concurrency: 2 }).toArray()); +try { await Readable.from(["x"]).reduce((a: string, b: string) => a + b); } catch (e) { console.log((e as Error & { code?: string }).code, (e as Error).message); } +try { Readable.from(["a"]).drop(-1); } catch (e) { console.log((e as Error & { code?: string }).code, (e as Error).message); } +try { Readable.from(["a"]).map((x: string) => x, { concurrency: 0 }); } catch (e) { console.log((e as Error & { code?: string }).code, (e as Error).message); } +const ac = new AbortController(); +ac.abort(); +try { await Readable.from(["a"]).toArray({ signal: ac.signal }); } catch (e) { console.log("toArray aborted", (e as Error).name); } +try { await Readable.from(["a"]).map((x: string) => x, { signal: ac.signal }).toArray(); } catch (e) { console.log("map aborted", (e as Error).name); } +try { await Readable.from(["a"]).reduce((a: string, b: string) => a + b, "", { signal: ac.signal }); } catch (e) { console.log("reduce aborted", (e as Error).name); } +const m = Readable.from(["a"]).map((x: string) => x); +console.log(m.readableObjectMode, m.readableHighWaterMark); diff --git a/tests/corpus/5336-stream-operators-promise-timing.ts b/tests/corpus/5336-stream-operators-promise-timing.ts new file mode 100644 index 000000000..016583ba8 --- /dev/null +++ b/tests/corpus/5336-stream-operators-promise-timing.ts @@ -0,0 +1,14 @@ +import { Readable } from "node:stream"; +const log: string[] = []; +const p = Readable.from(["a", "b"]).toArray().then((a: unknown) => log.push(`toArray ${String(a)}`)); +const q = Readable.from(["x", "y"]).some((c: string) => { log.push(`some ${c}`); return c === "x"; }).then((v: boolean) => log.push(`some -> ${v}`)); +const r = Readable.from(["1", "2"]).every((c: string) => { log.push(`every ${c}`); return c !== "2"; }).then((v: boolean) => log.push(`every -> ${v}`)); +const s = Readable.from(["k", "l"]).find((c: string) => c === "l").then((v: unknown) => log.push(`find -> ${String(v)}`)); +const t = Readable.from(["m", "n"]).forEach((c: string) => { log.push(`each ${c}`); }).then(() => log.push("forEach done")); +const u = Readable.from(["p", "q"]).reduce((acc: string, c: string) => acc + c, ">").then((v: string) => log.push(`reduce -> ${v}`)); +let i = 0; +const hop = (): void => { if (i++ < 40) { log.push(`hop ${i}`); Promise.resolve().then(hop); } }; +Promise.resolve().then(hop); +await Promise.all([p, q, r, s, t, u]); +await new Promise((res) => setTimeout(res, 5)); +console.log(log.join("\n")); diff --git a/tests/corpus/5337-stream-operators-concurrency.ts b/tests/corpus/5337-stream-operators-concurrency.ts new file mode 100644 index 000000000..ffbefb1ae --- /dev/null +++ b/tests/corpus/5337-stream-operators-concurrency.ts @@ -0,0 +1,44 @@ +import { Readable } from "node:stream"; + +// map output interleaved with microtasks, ticks and immediates +await new Promise((done) => { + const log: string[] = []; + const src = Readable.from(["a", "b", "c"]); + const out = src.map((c: string) => { + log.push(`map ${c}`); + return c.toUpperCase(); + }); + out.on("data", (c: string) => log.push(`data ${c}`)); + out.on("end", () => { + log.push("end"); + console.log(log.join("\n")); + done(); + }); + Promise.resolve() + .then(() => log.push("micro1")) + .then(() => log.push("micro2")) + .then(() => log.push("micro3")); + process.nextTick(() => log.push("tick")); + setImmediate(() => log.push("immediate")); +}); + +// concurrency 2 keeps output order; take() destroys its source. "a" is +// released only after "c" ends and the others resume on setImmediate, so +// the order does not depend on timer precision. +const log: string[] = []; +const gate = Promise.withResolvers(); +const out = Readable.from(["a", "b", "c", "d"]).map(async (c: string) => { + log.push(`start ${c}`); + if (c === "a") await gate.promise; + else await new Promise((r) => setImmediate(r)); + log.push(`end ${c}`); + if (c === "c") gate.resolve(); + return c + c; +}, { concurrency: 2 }); +for await (const v of out) log.push(`got ${String(v)}`); +console.log(log.join("\n")); +const taken = Readable.from(["1", "2", "3", "4"]); +const firstTwo = await taken.map((c: string) => { log.length = 0; return c; }).take(2).toArray(); +console.log(firstTwo, taken.destroyed); +const dropped = await Readable.from(["1", "2", "3"]).drop(1.5).toArray(); +console.log(dropped); diff --git a/tests/corpus/5338-stream-operators-checks.cjs b/tests/corpus/5338-stream-operators-checks.cjs new file mode 100644 index 000000000..2ef58c72a --- /dev/null +++ b/tests/corpus/5338-stream-operators-checks.cjs @@ -0,0 +1,38 @@ +'use strict'; +const { Readable } = require('node:stream'); +const r = () => Readable.from(['a', 'b', 'c']); +function syncCheck(label, f) { + try { f(); console.log(label, 'ok'); } catch (e) { console.log(label, e.name, e.code, e.message); } +} +async function asyncCheck(label, p) { + console.log(label); + try { await p; } catch (e) { console.log(' rejected', e.name, e.code, e.message); } +} +async function main() { + syncCheck('map(5)', () => r().map(5)); + syncCheck('filter(null)', () => r().filter(null)); + syncCheck('map options 7', () => r().map((x) => x, 7)); + syncCheck('map options []', () => r().map((x) => x, [])); + syncCheck('map signal {}', () => r().map((x) => x, { signal: {} })); + syncCheck('map concurrency NaN', () => r().map((x) => x, { concurrency: 'two' })); + syncCheck('map concurrency 1.5', () => r().map((x) => x, { concurrency: 1.5 })); + syncCheck('map highWaterMark -1', () => r().map((x) => x, { highWaterMark: -1 })); + syncCheck('take(-0.5)', () => r().take(-0.5)); + syncCheck('drop("x")', () => r().drop('x')); + syncCheck('take options null', () => r().take(1, null)); + await asyncCheck('forEach(1)', r().forEach(1)); + await asyncCheck('toArray(3)', r().toArray(3)); + await asyncCheck('reduce(null)', r().reduce(null)); + await asyncCheck('some({})', r().some({})); + await asyncCheck('every signal 1', r().every((x) => x, { signal: 1 })); + try { await Readable.from([]).reduce((a, b) => a + b); } catch (e) { console.log(e.name, e.code, e.message); } + console.log(await Readable.from(['x']).reduce((a, b) => a + b)); + console.log(await Readable.from(['1', '2']).reduce((a, b) => a + Number(b), 0)); + console.log(await r().map((c) => c + c).filter((c) => c !== 'bb').toArray()); + console.log(await r().drop(0).take(Infinity).toArray(), await r().take(0).toArray()); + const thrower = r().map((c) => { if (c === 'b') throw new Error('bad ' + c); return c; }); + thrower.on('error', (e) => console.log('stream error', e.message)); + try { for await (const v of thrower) console.log('v', v); } catch (e) { console.log('loop threw', e.message); } + try { await r().forEach(async (c) => { if (c === 'c') throw new Error('async bad'); }); } catch (e) { console.log('forEach rejected', e.message); } +} +main(); diff --git a/tests/corpus/5339-stream-for-await-hops.ts b/tests/corpus/5339-stream-for-await-hops.ts new file mode 100644 index 000000000..0a9c99544 --- /dev/null +++ b/tests/corpus/5339-stream-for-await-hops.ts @@ -0,0 +1,17 @@ +import { Readable } from "node:stream"; +const log: string[] = []; +let i = 0; +const hop = (): void => { if (i++ < 12) { log.push(`hop ${i}`); Promise.resolve().then(hop); } }; +async function run(): Promise { + for await (const c of Readable.from(["a", "b"])) log.push(`got ${String(c)}`); + log.push("loop done"); +} +Promise.resolve().then(hop); +const p = run(); +await p; +await new Promise((r) => setTimeout(r, 5)); +console.log(log.join(" | ")); + +// pushes made inside the iterator's own _read arrive as one chunk +const r = new Readable({ read() { this.push(Buffer.from("ab")); this.push(Buffer.from("cd")); this.push(null); } }); +for await (const c of r) console.log("chunk", c); diff --git a/tests/corpus/5340-readable-from-options.ts b/tests/corpus/5340-readable-from-options.ts new file mode 100644 index 000000000..2a9df7ec8 --- /dev/null +++ b/tests/corpus/5340-readable-from-options.ts @@ -0,0 +1,48 @@ +import { Readable } from "node:stream"; + +const a = Readable.from(["x", "y"]); +console.log("array", a.readableLength, a.readableHighWaterMark, a.readableObjectMode); +const s = Readable.from("hello"); +console.log("string", s.readableLength, s.readableHighWaterMark, s.readableObjectMode); +const b = Readable.from(Buffer.from("hi")); +console.log("buffer", b.readableHighWaterMark, b.readableObjectMode); +const bytes = Readable.from(["ab", "cd"], { objectMode: false }); +console.log("objectMode false", bytes.readableObjectMode, bytes.readableHighWaterMark); +bytes.on("data", (c: Buffer) => console.log("bytes data", c)); +const hw = Readable.from(["1", "2", "3"], { highWaterMark: 2 }); +console.log("hwm 2", hw.readableHighWaterMark); +hw.on("data", (c: string) => console.log("hw data", c, hw.readableLength)); +hw.on("close", () => console.log("hw close")); +for (const h of [-1, 1.5, NaN, Infinity]) { + try { + Readable.from(["a"], { highWaterMark: h }); + } catch (err) { + console.log((err as NodeJS.ErrnoException).code, (err as Error).message); + } +} +const lazy = Readable.from([Buffer.from("p"), Buffer.from("q")]); +lazy.on("readable", () => { + let c: Buffer | null; + while ((c = lazy.read()) !== null) console.log("read", String(c), lazy.readableLength); +}); +lazy.on("end", () => console.log("lazy end")); +lazy.on("close", () => console.log("lazy close")); +const sb = Readable.from(Buffer.from("xyz"), { objectMode: false }); +sb.on("data", (c: Buffer) => console.log("single bytes", c, sb.readableHighWaterMark)); +const d = Readable.from(["d1", "d2"]); +d.on("close", () => console.log("destroyed close")); +d.destroy(); +process.nextTick(() => console.log("tick 1")); +queueMicrotask(() => console.log("micro 1")); +const ord = Readable.from(["o1", "o2", "o3"]); +ord.on("data", (c: string) => { + console.log("ord", c); + process.nextTick(() => console.log("ord tick", c)); +}); +ord.on("end", () => console.log("ord end")); +ord.on("close", () => console.log("ord close")); +// an `any` source takes the checked-dynamic form, options included +const anySrc: any = JSON.parse('["a","b"]'); +const fromAny = Readable.from(anySrc, { objectMode: false }); +console.log("any source", fromAny.readableObjectMode, fromAny.readableHighWaterMark); +fromAny.on("data", (c: Buffer) => console.log("any data", c)); diff --git a/tests/corpus/5341-readable-from-dynamic.cjs b/tests/corpus/5341-readable-from-dynamic.cjs new file mode 100644 index 000000000..70b5919b5 --- /dev/null +++ b/tests/corpus/5341-readable-from-dynamic.cjs @@ -0,0 +1,34 @@ +// @no-node-shims +'use strict'; +const { Readable } = require('node:stream'); +const e = Readable.from([]); +e.on('end', () => console.log('empty end')); +e.resume(); +for (const bad of [42, null, undefined, { a: 1 }, true, Promise.resolve(1)]) { + try { Readable.from(bad); console.log('accepted', bad); } catch (err) { console.log(err.code, err.message); } +} +const n = Readable.from(['ok', null, 'later']); +n.on('data', (c) => console.log('null-stream data', c)); +n.on('error', (err) => console.log('null-stream error', err.name, err.code, err.message)); +n.on('close', () => console.log('null-stream close')); +const bm = Readable.from(['ok', 7], { objectMode: false }); +bm.on('data', (c) => console.log('bm data', c)); +bm.on('error', (err) => console.log('bm error', err.name, err.code, err.message)); +const dynSrc = JSON.parse('["j1","j2"]'); +const j = Readable.from(dynSrc, { highWaterMark: 4 }); +console.log('dyn', j.readableHighWaterMark, j.readableObjectMode, j.readableLength); +j.on('data', (c) => console.log('dyn data', typeof c, c)); +const und = Readable.from(['u'], { highWaterMark: undefined }); +console.log('undef hwm', und.readableHighWaterMark); +const und2 = Readable.from(['u'], { objectMode: false, highWaterMark: null }); +console.log('null hwm bytes', und2.readableHighWaterMark, und2.readableObjectMode); +try { Readable.from(['x'], { highWaterMark: '2' }); } catch (err) { console.log(err.code, err.message); } +const one = Readable.from('single'); +one.on('data', (c) => console.log('single', c)); +const opt = Readable.from(Buffer.from('bb'), JSON.parse('{"objectMode":false}')); +opt.on('data', (c) => console.log('opt', c, opt.readableObjectMode)); +(async () => { + for await (const c of Readable.from(['f1', 'f2'])) console.log('for await', c); + const arr = await Readable.from(['t1', 't2']).toArray(); + console.log('toArray', arr); +})(); diff --git a/tests/corpus/5342-stream-compose.ts b/tests/corpus/5342-stream-compose.ts new file mode 100644 index 000000000..25959634b --- /dev/null +++ b/tests/corpus/5342-stream-compose.ts @@ -0,0 +1,72 @@ +import { compose, PassThrough, Readable, Transform, Writable } from "node:stream"; + +const upper = new Transform({ + transform(chunk: Buffer, _enc, cb) { + cb(null, chunk.toString().toUpperCase()); + }, +}); +const exclaim = new Transform({ + transform(chunk: Buffer, _enc, cb) { + cb(null, chunk.toString() + "!"); + }, +}); +const d = compose(upper, exclaim); +console.log("duplex", d.readable, d.writable, d.readableObjectMode); +d.on("data", (c: Buffer) => console.log("data", c.toString())); +d.on("finish", () => console.log("finish")); +d.on("end", () => console.log("end")); +d.on("close", () => console.log("close")); +d.write("a"); +d.write("b"); +d.end("c"); + +const p = new PassThrough(); +console.log("single", compose(p) === p); + +try { + compose(); +} catch (err) { + console.log((err as Error).name, (err as NodeJS.ErrnoException).code, (err as Error).message); +} + +// A readable head: the composed stream is read-only. +const src = Readable.from(["x", "y"]); +const pt = new PassThrough(); +const r = compose(src, pt); +console.log("read-only", r.readable, r.writable); +r.on("data", (c: Buffer) => console.log("r data", c.toString())); +r.on("end", () => console.log("r end")); +r.on("close", () => console.log("r close")); + +// A writable tail: the composed stream is write-only. +const seen: string[] = []; +const sink = new Writable({ + write(chunk: Buffer, _enc, cb) { + seen.push(chunk.toString()); + cb(); + }, +}); +const w = compose(new PassThrough(), sink); +console.log("write-only", w.readable, w.writable); +w.on("finish", () => console.log("w finish", seen.join(","))); +w.on("close", () => console.log("w close")); +w.end("z"); + +// Errors: a failing stage destroys the composed stream. +const bad = new Transform({ + transform(_chunk: Buffer, _enc, cb) { + cb(new Error("boom")); + }, +}); +const e = compose(new PassThrough(), bad); +e.on("error", (err: Error) => console.log("e error", err.message)); +e.on("close", () => console.log("e close")); +e.write("q"); + +// destroy() before the pipeline ends: AbortError. +const k = compose(new PassThrough(), new PassThrough()); +k.on("error", (err: Error) => console.log("k error", err.name, (err as NodeJS.ErrnoException).code)); +k.on("close", () => console.log("k close")); +k.destroy(); + +process.nextTick(() => console.log("tick")); diff --git a/tests/corpus/5343-stream-compose-edges.ts b/tests/corpus/5343-stream-compose-edges.ts new file mode 100644 index 000000000..488012a02 --- /dev/null +++ b/tests/corpus/5343-stream-compose-edges.ts @@ -0,0 +1,51 @@ +import { compose, PassThrough, Readable, Transform, Writable } from "node:stream"; + +// object-mode tail: Readable.from strings through compose +const objs = compose(Readable.from(["a", "b", "c"]), new PassThrough({ objectMode: false })); +console.log("objs", objs.readableObjectMode, objs.readableHighWaterMark); +objs.on("data", (c: Buffer) => console.log("objs data", c.toString())); +objs.on("end", () => console.log("objs end")); + +// an operator stream as the tail: object mode +const mapped = compose(new PassThrough(), new PassThrough()).map((c: Buffer) => c.toString() + "?"); +mapped.on("data", (c: string) => console.log("mapped", c)); + +// backpressure: a slow sink makes the composed writable buffer +const slow = new Writable({ + highWaterMark: 2, + write(chunk: Buffer, _enc, cb) { + console.log("slow write", chunk.toString()); + setTimeout(cb, 1); + }, +}); +const bp = compose(new PassThrough({ highWaterMark: 2 }), slow); +console.log("bp write 1", bp.write("aa")); +console.log("bp write 2", bp.write("bb")); +console.log("bp write 3", bp.write("cc")); +bp.on("drain", () => console.log("bp drain")); +bp.on("finish", () => console.log("bp finish")); +bp.end(); + +// a destroyed stream in the chain +const gone = new PassThrough(); +gone.destroy(); +try { + compose(gone, new PassThrough()); +} catch (err) { + console.log((err as NodeJS.ErrnoException).code, (err as Error).message.slice(0, 44)); +} + +// upper-casing round trip with a for-await reader +const up = compose( + new Transform({ + transform(c: Buffer, _e, cb) { + cb(null, c.toString().toUpperCase()); + }, + }), + new PassThrough(), +); +(async () => { + up.end("hello"); + for await (const c of up) console.log("for await", String(c)); + console.log("done"); +})(); diff --git a/tests/corpus/5344-stream-pipeline-timing.ts b/tests/corpus/5344-stream-pipeline-timing.ts new file mode 100644 index 000000000..163130db3 --- /dev/null +++ b/tests/corpus/5344-stream-pipeline-timing.ts @@ -0,0 +1,71 @@ +import { PassThrough, pipeline, Transform, Writable } from "node:stream"; + +const a = new PassThrough(); +const b = new PassThrough(); +b.resume(); +b.on("close", () => { + console.log("b close"); + process.nextTick(() => console.log("tick after b close")); + queueMicrotask(() => console.log("micro after b close")); +}); +pipeline(a, b, (err?: Error | null) => console.log("cb", err ? err.message : "ok")); +a.end("x"); + +const c = new PassThrough(); +const d = new PassThrough(); +d.on("close", () => { + console.log("d close"); + process.nextTick(() => console.log("tick after d close")); +}); +c.on("close", () => { + console.log("c close"); + process.nextTick(() => console.log("tick after c close")); +}); +pipeline(c, d, (err?: Error | null) => console.log("cb2", err ? err.message : "ok")); +d.destroy(new Error("bad")); + +// a synchronous write error reaches 'error' one tick later + +let k = 0; +const note = (s: string) => console.log(k++, s); +const src = new PassThrough(); +const failing = new Transform({ + transform(chunk: Buffer, _enc, cb) { + note("transform " + chunk.toString()); + cb(new Error("boom")); + }, +}); +failing.on("error", (err: Error) => note("failing error " + err.message)); +src.on("resume", () => note("src resume")); +src.pipe(failing); +src.write("q"); +note("after write"); +let u = 0; +const step = () => { + if (u++ < 3) { + note("step " + u); + process.nextTick(step); + } +}; +process.nextTick(step); + +// 'finish' waits for a pending _final callback, even when end() runs again +const slowFinal = new Writable({ + write(_c, _e, cb) { + cb(); + }, + final(cb) { + console.log("final called"); + setTimeout(() => { + console.log("final done"); + cb(); + }, 5); + }, +}); +slowFinal.on("finish", () => console.log("slow final finish")); +slowFinal.write("a"); +slowFinal.end(); +setTimeout(() => { + slowFinal.end(); + console.log("second end"); +}, 1); diff --git a/tests/corpus/5710-map-number-keys-hashed.ts b/tests/corpus/5710-map-number-keys-hashed.ts new file mode 100644 index 000000000..df67ef240 --- /dev/null +++ b/tests/corpus/5710-map-number-keys-hashed.ts @@ -0,0 +1,104 @@ +// Number-keyed Map and Set lookups on hashed storage: small maps that +// scan their entries, bucket tables, keys that need SameValueZero +// normalization, deletes and clears between reads, and maps that move +// between the direct index and hashing. + +const special = [0, -0, NaN, Infinity, -Infinity, 0.5, -1, 2 ** 53, 1e300, Number.MIN_VALUE, Number.MAX_VALUE]; + +function describe(m: Map, keys: number[]): string { + return keys.map((k) => `${Object.is(k, -0) ? "-0" : k}:${m.has(k) ? m.get(k) : "-"}`).join(" "); +} + +// Small maps keep no bucket table. +const small = new Map(); +small.set(1e300, "big"); +small.set(-0, "zero"); +small.set(NaN, "nan"); +console.log("small", small.size, describe(small, special)); +console.log("small keys", [...small.keys()].map((k) => (Object.is(k, -0) ? "-0" : String(k))).join(",")); +small.delete(0); +console.log("small after delete", small.size, describe(small, special)); +small.set(0, "again"); +console.log("small re-add", describe(small, [0, -0, NaN, 1e300])); + +// Bucket tables of sparse ids, with hits, misses and tombstones. +const table = new Map(); +for (let i = 0; i < 3000; i++) table.set(100000 + i * 7919, i); +let hits = 0, + misses = 0, + sum = 0; +for (let i = 0; i < 6000; i++) { + const k = 100000 + i * 7919; + const v = table.get(k); + if (v === undefined) misses++; + else { + hits++; + sum += v; + } + if (table.has(k + 1)) hits += 1000000; +} +console.log("table", table.size, hits, misses, sum); +for (let i = 0; i < 3000; i += 3) table.delete(100000 + i * 7919); +let live = 0; +for (let i = 0; i < 3000; i++) if (table.has(100000 + i * 7919)) live++; +console.log("table after deletes", table.size, live, table.get(100000), table.get(100000 + 7919)); +for (const k of special) table.set(k, -1); +console.log("table specials", table.size, special.map((k) => table.get(k)).join(","), table.get(0.25)); +table.clear(); +console.log("table cleared", table.size, table.get(100000 + 7919), table.has(NaN)); + +// A read, a write of the same key, then the read again. +const m = new Map(); +for (let i = 0; i < 64; i++) m.set(i * 1000003, i); +const key = 5 * 1000003; +const before = m.get(key); +m.delete(key); +const afterDelete = m.get(key); +const hadIt = m.has(key); +m.set(key, 99); +const afterSet = m.get(key); +for (let i = 64; i < 400; i++) m.set(i * 1000003, i); // grows the bucket table +const afterGrowth = m.get(key); +console.log("reread", before, afterDelete, hadIt, afterSet, afterGrowth, m.size); + +// Direct index to hashing and back. +const ids = new Map(); +for (let i = 0; i < 40; i++) ids.set(i, `id${i}`); +console.log("dense", ids.get(0), ids.get(39), ids.get(40), ids.has(-1), ids.get(-0)); +ids.set(-5, "neg"); +ids.set(1.5, "half"); +console.log("hashed", ids.size, ids.get(0), ids.get(39), ids.get(-5), ids.get(1.5), ids.get(2.5)); +ids.delete(-5); +ids.delete(1.5); +for (let i = 40; i < 200; i++) ids.set(i, `id${i}`); +console.log("regrown", ids.size, ids.get(0), ids.get(150), ids.get(-5), ids.get(1.5)); + +// Sets and boolean and reference values. +const seen = new Set(); +for (let i = 0; i < 500; i++) seen.add((i * 2654435761) % 4294967296); +let found = 0; +for (let i = 0; i < 1000; i++) if (seen.has((i * 2654435761) % 4294967296)) found++; +console.log("set", seen.size, found, seen.has(NaN), seen.has(-0)); +seen.add(-0); +seen.add(NaN); +console.log("set specials", seen.has(0), seen.has(NaN), seen.size); + +const flags = new Map(); +for (let i = 0; i < 100; i++) flags.set(i * 37 + 1e6, i % 3 === 0); +let truthy = 0; +for (let i = 0; i < 100; i++) if (flags.get(i * 37 + 1e6)) truthy++; +console.log("flags", truthy, flags.get(1e6), flags.get(1e6 + 37), flags.get(5)); + +interface Node { + id: number; + label: string; +} +const nodes = new Map(); +for (let i = 0; i < 300; i++) nodes.set(50000 + i * 13, { id: i, label: `n${i}` }); +let labels = ""; +for (let i = 0; i < 300; i += 37) { + const node = nodes.get(50000 + i * 13); + labels += node === undefined ? "?" : node.label; + if (nodes.has(50000 + i * 13 + 1)) labels += "!"; +} +console.log("nodes", nodes.size, labels, nodes.get(49999)?.label ?? "none"); diff --git a/tests/corpus/5712-map-number-overwrite-at-growth.ts b/tests/corpus/5712-map-number-overwrite-at-growth.ts new file mode 100644 index 000000000..acbdb06e7 --- /dev/null +++ b/tests/corpus/5712-map-number-overwrite-at-growth.ts @@ -0,0 +1,61 @@ +// Number-key Map.set and Set.add of present keys while the map is at its +// bucket-table growth boundary (the next new key grows the index): +// overwrites keep the entry, its position and the size, and release the +// value they replace. + +interface Box { + label: string; +} + +function sparse(i: number): number { + return 1000 + i * 7919; +} + +for (const size of [8, 64, 1024]) { + const nums = new Map(); + const boxes = new Map(); + const names = new Map(); + const seen = new Set(); + for (let i = 0; i < size; i++) { + nums.set(sparse(i), i); + boxes.set(sparse(i), { label: `b${i}` }); + names.set(sparse(i), `n${i}`); + seen.add(sparse(i)); + } + for (let round = 0; round < 3; round++) { + for (let i = size - 1; i >= 0; i--) { + nums.set(sparse(i), i * 10 + round); + boxes.set(sparse(i), { label: `b${i}.${round}` }); + names.set(sparse(i), `n${i}.${round}`.repeat(2)); + seen.add(sparse(i)); + } + } + let sum = 0; + for (const v of nums.values()) sum += v; + const firstKeys = [...nums.keys()].slice(0, 3).join(","); + console.log( + size, + nums.size, + boxes.size, + names.size, + seen.size, + sum, + firstKeys, + boxes.get(sparse(size - 1))?.label, + names.get(sparse(0)), + [...seen].slice(0, 2).join(","), + ); + // One more key grows the index; existing entries are unchanged. + nums.set(-1, -1); + boxes.set(-1, { label: "new" }); + console.log(size, "grown", nums.size, nums.get(sparse(size >> 1)), nums.get(-1), boxes.get(-1)?.label, boxes.get(sparse(1))?.label); +} + +// At the boundary after deletes: a delete frees no bucket, so re-adding +// the deleted key compacts instead of growing. +const churn = new Map(); +for (let i = 0; i < 64; i++) churn.set(sparse(i), i); +churn.delete(sparse(10)); +churn.set(sparse(11), 111); +churn.set(sparse(10), 110); +console.log("churn", churn.size, churn.get(sparse(10)), churn.get(sparse(11)), [...churn.keys()].indexOf(sparse(10))); diff --git a/tests/corpus/stream-dynamic-entry-kinds.cjs b/tests/corpus/stream-dynamic-entry-kinds.cjs new file mode 100644 index 000000000..82d70b16a --- /dev/null +++ b/tests/corpus/stream-dynamic-entry-kinds.cjs @@ -0,0 +1,28 @@ +// @no-node-shims +// Readable.from over parsed JSON strings: the first lazy pull makes the +// stream hold strings, which reach 'data', for await, an operator and a +// pipe as strings; a mixed source decodes once setEncoding() is set. +'use strict'; +const { Readable, Writable } = require('node:stream'); +async function main() { + const fromJson = () => Readable.from(JSON.parse('["j1", "j2", "j3"]')); + const data = []; + const r = fromJson(); + r.on('data', (c) => data.push(typeof c + ':' + c)); + await new Promise((resolve) => r.on('end', resolve)); + console.log('data', data.join(' ')); + for await (const c of fromJson()) console.log('for await', typeof c, c); + console.log('toArray', await fromJson().map((c) => c.toUpperCase()).toArray()); + const hex = []; + const sink = new Writable({ write(chunk, enc, cb) { hex.push(chunk.toString('hex')); cb(); } }); + fromJson().pipe(sink); + await new Promise((resolve) => sink.on('finish', resolve)); + console.log('piped', hex.join(',')); + const mixed = Readable.from(['s1', Buffer.from('b1'), 's2']); + mixed.setEncoding('utf8'); + for await (const c of mixed) console.log('mixed', typeof c, c); + const switched = fromJson(); + switched.setEncoding('latin1'); + console.log('switched', await switched.toArray()); +} +main(); diff --git a/tests/corpus/stream-dynamic-entry-kinds.ts b/tests/corpus/stream-dynamic-entry-kinds.ts new file mode 100644 index 000000000..a891df8a2 --- /dev/null +++ b/tests/corpus/stream-dynamic-entry-kinds.ts @@ -0,0 +1,18 @@ +// A Readable.from stream over parsed JSON strings read through an any +// receiver: read() answers the strings, as in Node, and setEncoding() +// before the first read keeps a Buffer and string source as strings. +import { Readable } from "node:stream"; + +const source: any = JSON.parse('["t1", "t2"]'); +const r: any = Readable.from(source); +r.once("readable", () => { + let c: any; + while ((c = r.read()) !== null) console.log("read", typeof c, c, Buffer.isBuffer(c)); +}); +r.on("end", () => console.log("end")); + +const values: any = JSON.parse('["s1"]'); +values.unshift(Buffer.from("b1")); +const mixed = Readable.from(values); +mixed.setEncoding("utf8"); +mixed.on("data", (c: string) => console.log("mixed", typeof c, c)); diff --git a/tests/corpus/stream-encoded-from-entries.cjs b/tests/corpus/stream-encoded-from-entries.cjs new file mode 100644 index 000000000..64c763e4d --- /dev/null +++ b/tests/corpus/stream-encoded-from-entries.cjs @@ -0,0 +1,29 @@ +// @no-node-shims +// setEncoding on Readable.from streams from CommonJS: Buffer, string and +// Uint8Array entries, a decoder set after the first chunk, and pipe. +'use strict'; +const { Readable, Writable } = require('node:stream'); +async function main() { + const mixed = Readable.from([Buffer.from('b1'), 's2', new Uint8Array([0x75, 0x33])]); + mixed.setEncoding('utf8'); + for await (const c of mixed) console.log('mixed', typeof c, c); + const late = Readable.from([Buffer.from('q1'), Buffer.from('q2')], { objectMode: false }); + const rest = []; + await new Promise((resolve) => { + late.once('data', (first) => { + console.log('late first', Buffer.isBuffer(first), String(first)); + late.setEncoding('hex'); + late.on('data', (c) => rest.push(typeof c + ':' + c)); + late.on('end', () => resolve(undefined)); + }); + }); + console.log('late rest', rest.join(' ')); + const seen = []; + const w = new Writable({ write(chunk, enc, cb) { seen.push(chunk.toString('hex')); cb(); } }); + const src = Readable.from([Buffer.from('w1'), Buffer.from('w2')]); + src.setEncoding('utf8'); + src.pipe(w); + await new Promise((resolve) => w.on('finish', resolve)); + console.log('piped', seen.join(',')); +} +main(); diff --git a/tests/corpus/stream-encoded-from-entries.ts b/tests/corpus/stream-encoded-from-entries.ts new file mode 100644 index 000000000..d96c0a505 --- /dev/null +++ b/tests/corpus/stream-encoded-from-entries.ts @@ -0,0 +1,71 @@ +// setEncoding on Readable.from streams: Buffer and Uint8Array entries +// decode through the stream's decoder whether they were buffered before +// setEncoding or pulled after it, in object and byte mode, and reach +// for-await, pipe, pipeline, an operator's output and a composed stream +// as strings. +import { PassThrough, Readable, Writable, compose, pipeline } from "node:stream"; + +const log: string[] = []; +const sink = (label: string): Writable => + new Writable({ + write(chunk: Buffer, _enc, cb) { + log.push(`${label} wrote ${chunk.toString("hex")}`); + cb(); + }, + }); +const drain = async (label: string, r: Readable): Promise => { + for await (const c of r) log.push(`${label} got ${typeof c} ${String(c)}`); +}; + +// setEncoding before the first lazy read +const before = Readable.from([Buffer.from("ab"), Buffer.from("c\\u00e9")]); +before.setEncoding("utf8"); +await drain("before", before); + +// setEncoding after the first lazy read (byte mode: Node re-counts an +// object-mode buffer's length in characters) +const after = Readable.from([Buffer.from("xy"), Buffer.from("zw"), Buffer.from("!")], { + objectMode: false, +}); +await new Promise((resolve) => { + after.once("data", (c: Buffer) => { + log.push(`after first ${c.toString()}`); + after.pause(); + after.setEncoding("hex"); + after.on("data", (s: string) => log.push(`after rest ${s}`)); + after.on("end", () => resolve()); + after.resume(); + }); +}); + +// Uint8Array entries, a split multi-byte character across entries +const split = Readable.from([new Uint8Array([0xe2, 0x82]), new Uint8Array([0xac, 0x41])]); +split.setEncoding("utf8"); +await drain("split", split); + +// byte mode +const bytes = Readable.from([Buffer.from("by"), Buffer.from("te")], { objectMode: false }); +bytes.setEncoding("base64"); +await drain("bytes", bytes); + +// pipe and pipeline delivery of an encoded stream +const piped = Readable.from([Buffer.from("p1"), Buffer.from("p2")]); +piped.setEncoding("utf8"); +await new Promise((resolve) => piped.pipe(sink("pipe")).on("finish", () => resolve())); +const lined = Readable.from([Buffer.from("l1"), Buffer.from("l2")]); +lined.setEncoding("latin1"); +await new Promise((resolve) => + pipeline(lined, new PassThrough(), sink("pipeline"), () => resolve()), +); + +// an operator's output and a composed stream, encoded +const mapped = Readable.from([Buffer.from("m")]).map((b: Buffer) => Buffer.concat([b, b])); +mapped.setEncoding("utf8"); +await drain("mapped", mapped); +const composed = compose(Readable.from([Buffer.from("k1"), Buffer.from("k2")]), new PassThrough()); +composed.setEncoding("utf8"); +const parts: string[] = []; +composed.on("data", (c: string) => parts.push(typeof c + ":" + c)); +await new Promise((resolve) => composed.on("end", () => resolve())); +log.push(`composed ${parts.join("").replace(/string:/g, "")} ${parts.every((p) => p.startsWith("string:"))}`); +console.log(log.join("\n")); diff --git a/tests/harness/stream-entry-kinds.test.ts b/tests/harness/stream-entry-kinds.test.ts new file mode 100644 index 000000000..f7333b298 --- /dev/null +++ b/tests/harness/stream-entry-kinds.test.ts @@ -0,0 +1,158 @@ +/* Stream chunk kinds Node cannot oracle — scriptc-only. + * + * A typed read() answers `Buffer | null`. When the read itself makes the + * stream hold strings (the first string a lazy Readable.from pull over a + * dynamic array produces, or setEncoding() inside _read), Node returns + * the string, which the typed result cannot hold, so the read throws the + * encoding fence instead (caught here, so the program exits normally). + * These pins keep that refusal exact and, under + * SCRIPTC_SAN=1 (ASan and the RC audit, like every harness lane), prove + * no string reaches Buffer code. The Node-observable cases are corpus + * programs (stream-dynamic-entry-kinds.*). + */ +import { execFile } from "node:child_process"; +import { createHash } from "node:crypto"; +import { mkdirSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { promisify } from "node:util"; +import { describe, expect, test } from "vitest"; +import { compile } from "@scriptc/compiler"; + +const execFileAsync = promisify(execFile); +const repoRoot = join(import.meta.dirname, "../.."); +const cacheDir = join(repoRoot, "node_modules/.cache/scriptc-tests"); +const sanitize = process.env["SCRIPTC_SAN"] === "1"; + +interface RunResult { + stdout: string; + stderr: string; + exitCode: number; +} + +async function compileAndRun(name: string, source: string, ext = "ts"): Promise { + const key = createHash("sha256") + .update(source) + .update(sanitize ? "san" : "plain") + .digest("hex") + .slice(0, 16); + const outDir = join(cacheDir, `stream-kinds-${key}`); + mkdirSync(outDir, { recursive: true }); + const file = join(outDir, `${name}.${ext}`); + writeFileSync(file, source); + const result = await compile(file, { + outPath: join(outDir, name), + outDir, + sanitize, + backend: "llvm", + }); + if (!result.ok) { + throw new Error( + "stream program failed to compile:\n" + + result.diagnostics.map((d) => `${d.code}: ${d.message}`).join("\n"), + ); + } + try { + const { stdout, stderr } = await execFileAsync(result.binaryPath, [], { encoding: "utf8" }); + return { stdout, stderr: stripAsanFiberWarning(stderr), exitCode: 0 }; + } catch (err) { + const e = err as { code?: unknown; stdout?: string; stderr?: string }; + if (typeof e.code !== "number") throw err; + return { + stdout: e.stdout ?? "", + stderr: stripAsanFiberWarning(e.stderr ?? ""), + exitCode: e.code, + }; + } +} + +/** Linux ASan's once-per-process swapcontext warning (see errors.test.ts). */ +function stripAsanFiberWarning(s: string): string { + return s.replace( + /^==\d+==WARNING: ASan doesn't fully support makecontext\/swapcontext.*\n/gm, + "", + ); +} + +const READ_FENCE_MESSAGE = + "read() on a stream with an encoding set is not supported yet (consume 'data' events, which deliver strings)"; + +describe(`typed stream reads of string chunks (scriptc-only${sanitize ? ", sanitized" : ""})`, () => { + test("a typed read() whose lazy pull yields the first string throws the encoding fence", async () => { + const r = await compileAndRun( + "typed-read-json-string", + `import { Readable } from "node:stream"; +const r = Readable.from(JSON.parse('["j1", "j2"]')); +try { + const chunk = r.read(); + console.log("hex", chunk === null ? "null" : chunk.toString("hex")); +} catch (err) { + console.log("caught", (err as Error).message); +} +`, + ); + expect(r.exitCode).toBe(0); + expect(r.stdout).toBe(`caught ${READ_FENCE_MESSAGE}\n`); + }); + + test("a typed read() whose _read calls setEncoding() throws the encoding fence", async () => { + const r = await compileAndRun( + "typed-read-encoding-in-read", + `import { Readable } from "node:stream"; +const r = new Readable({ + read() { + this.setEncoding("utf8"); + this.push(Buffer.from("ab")); + this.push(null); + }, +}); +try { + const chunk = r.read(); + console.log("hex", chunk === null ? "null" : chunk.toString("hex")); +} catch (err) { + console.log("caught", (err as Error).message); +} +`, + ); + expect(r.exitCode).toBe(0); + expect(r.stdout).toBe(`caught ${READ_FENCE_MESSAGE}\n`); + }); + + test("the JavaScript lane's read() on a JSON source throws the same fence", async () => { + const r = await compileAndRun( + "js-read-json-string", + `'use strict'; +const { Readable } = require('node:stream'); +const r = Readable.from(JSON.parse('["j1"]')); +r.once('readable', () => { + try { + const chunk = r.read(); + console.log('got', chunk === null ? 'null' : chunk.length); + } catch (err) { + console.log('caught', err.message); + } +}); +`, + "cjs", + ); + expect(r.exitCode).toBe(0); + expect(r.stdout).toBe(`caught ${READ_FENCE_MESSAGE}\n`); + }); + + test("a dynamic source that mixes Buffers and strings without a decoder is destroyed", async () => { + const r = await compileAndRun( + "mixed-dynamic-source", + `import { Readable } from "node:stream"; +const values: any = JSON.parse('["s1"]'); +values.unshift(Buffer.from("b1")); +const r = Readable.from(values); +r.on("data", (c: any) => console.log("data", Buffer.isBuffer(c), String(c))); +r.on("error", (err: Error) => console.log("error", err.message)); +r.on("close", () => console.log("close")); +`, + ); + expect(r.exitCode).toBe(0); + expect(r.stdout).toBe( + "data true b1\nerror a Readable.from source that mixes strings and Buffers is not supported yet\nclose\n", + ); + }); +}); diff --git a/tests/harness/wasi-differential.test.ts b/tests/harness/wasi-differential.test.ts index 7a898b19d..94b096f7e 100644 --- a/tests/harness/wasi-differential.test.ts +++ b/tests/harness/wasi-differential.test.ts @@ -269,6 +269,8 @@ describe.skipIf(!zigOnPath())("wasm32-wasi differential", () => { "5317-map-iterate-while-mutating.ts", "5318-set-iterate-after-deletes.ts", "5319-map-iteration-values.ts", + // Hashed number-key lookups probe ScrMap through size_t-width fields. + "5710-map-number-keys-hashed.ts", // JSON writers read array slots as pointers. "5305-json-stringify-typed-arrays.ts", // Comparator sorts move 64-bit slots of 32-bit pointers.