From d426960f454e86bc8f53226e59b5a4db256f317c Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 22 Oct 2026 01:55:38 -0500 Subject: [PATCH 1/2] Make Cipheriv and Decipheriv Transforms and accept function stages anywhere in pipeline() Cipheriv and Decipheriv values are Transforms, as in Node: they can be written and ended, piped into, used as pipeline() and stream/promises pipeline stages, composed, read, iterated with for await, read by stream/consumers, passed to finished(), and listened to, and their stream properties read as in Node. Each chunk goes through the cipher's update(), and the end pushes final() or reports its error through 'error'; writing after final() throws out of write(). The stream unit builds a cipher's Transform on its first stream use from operations the cipher's handle provides (ScrDynHandleOps.transform), so programs that use ciphers without streams link as before. The cipher is a collector-headered object behind a traced handle: its Transform's steps hold it, and the pair is reclaimed as a cycle. Through untyped values, in a program that uses streams, a cipher's stream properties and its on, once, off, pipe, unpipe, read, pause, resume, setEncoding, write, end, destroy, cork and uncork members run on its Transform; other stream members throw a coded SC2020 error. A Transform holds a write's callback while its readable side is at the high-water mark and runs it at the next read, and runs it on a tick once the readable side has ended, as Transform._write does. write() on a PassThrough, a Transform, a Hash or a cipher used to return true however much the readable side buffered. zlib streams applied the rule already and keep their timing. pipeline() accepts function stages anywhere among its streams, so runs of async generator stages can sit on both sides of a stream. A stage can be named by a function declaration, or a const bound to a function expression, that nothing reassigns; its body lowers again at the pipeline with its source typed by the pipeline. The source can also be an array, a string, a synchronous generator object, or a synchronous generator function without parameters: an async generator steps the source's iterator and yields each element, and the next stage receives each element on the same tick as with Node's for await over the source. The runtime's function-stage entry points take a bitmask of the function positions and one (next, read, return) helper triple for each generator pumped into a stream, so RUNTIME_ABI_VERSION is now 25. In JavaScript, the chunks a stage receives from the previous stage's generator read as the strings or Buffers it yields, so their string and Buffer methods compile. Corpus 7055-7059 cover cipher streams in TypeScript and JavaScript, their errors, close order and backpressure, the pipeline stages and sources with their tick order, and Transform backpressure. --- .../limitations/builtin-apis/crypto.mdx | 5 +- .../builtin-apis/streams-and-readline.mdx | 8 +- .../src/generated/node-v24-compatibility.json | 24 +- .../generated/node-v24-internal.json | 45 ++ internal/compatibility/src/generate.mjs | 2 +- internal/compatibility/static-support.json | 6 +- packages/cli/test/native-link-info.test.ts | 4 +- .../ambient/scriptc-node-fallback.d.ts | 7 +- .../compiler/src/backend/llvm/lib-shared.ts | 1 + .../compiler/src/backend/llvm/lib-stream.ts | 29 +- packages/compiler/src/backend/runtime-abi.ts | 4 +- .../src/backend/runtime-pack-targets.test.ts | 2 +- .../compiler/src/backend/runtime-pack.test.ts | 2 +- .../src/frontend/lowering/binding-analysis.ts | 10 +- .../src/frontend/lowering/builtins/crypto.ts | 5 +- .../frontend/lowering/builtins/hash-stream.ts | 133 +++++- .../src/frontend/lowering/containers/bytes.ts | 11 +- .../lowering/containers/iterator-helpers.ts | 4 +- .../src/frontend/lowering/dyn-streams.ts | 2 +- .../lowering/expressions/property-access.ts | 3 + .../src/frontend/lowering/lower-stmts.ts | 10 +- .../src/frontend/lowering/lower-stream.ts | 424 ++++++++++++++---- .../compiler/src/frontend/lowering/lowerer.ts | 15 +- packages/compiler/src/ir/builtin-effects.ts | 1 + .../compiler/src/ir/builtin-signatures.ts | 35 +- packages/compiler/src/ir/runtime-features.ts | 1 + packages/compiler/src/ir/validate.ts | 33 +- .../test/native-codegen-integration.test.ts | 4 +- .../tests/corpus/7055-cipher-streams.ts.json | 6 + .../corpus/7056-cipher-stream-errors.ts.json | 6 + .../tests/corpus/7057-cipher-streams.mjs.json | 6 + .../7058-pipeline-function-stages.mjs.json | 6 + .../7059-transform-backpressure.ts.json | 6 + .../runtime-darwin-arm64/scripts/build.mjs | 2 +- .../runtime-pack-common/scripts/build.mjs | 2 +- packages/runtime/src/scr_cipher.c | 132 +++++- packages/runtime/src/scr_console.c | 2 +- packages/runtime/src/scr_core.h | 2 +- packages/runtime/src/scr_dyn.h | 29 ++ packages/runtime/src/scr_json.c | 3 + packages/runtime/src/scr_stream.c | 386 ++++++++++++---- packages/runtime/src/scr_stream.h | 64 +-- packages/runtime/src/scr_stream_dyn.c | 5 +- packages/runtime/src/scr_zlib_stream.c | 3 +- tests/corpus/7055-cipher-streams.ts | 182 ++++++++ tests/corpus/7056-cipher-stream-errors.ts | 111 +++++ tests/corpus/7057-cipher-streams.mjs | 85 ++++ .../corpus/7058-pipeline-function-stages.mjs | 180 ++++++++ tests/corpus/7059-transform-backpressure.ts | 90 ++++ .../corpus-costs/7055-cipher-streams.ts.json | 1 + .../7056-cipher-stream-errors.ts.json | 1 + .../corpus-costs/7057-cipher-streams.mjs.json | 1 + .../7058-pipeline-function-stages.mjs.json | 1 + .../7059-transform-backpressure.ts.json | 1 + .../self-hosting-native-toolchain.test.ts | 2 +- 55 files changed, 1803 insertions(+), 342 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7055-cipher-streams.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7056-cipher-stream-errors.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7057-cipher-streams.mjs.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7058-pipeline-function-stages.mjs.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7059-transform-backpressure.ts.json create mode 100644 tests/corpus/7055-cipher-streams.ts create mode 100644 tests/corpus/7056-cipher-stream-errors.ts create mode 100644 tests/corpus/7057-cipher-streams.mjs create mode 100644 tests/corpus/7058-pipeline-function-stages.mjs create mode 100644 tests/corpus/7059-transform-backpressure.ts create mode 100644 tests/harness/corpus-costs/7055-cipher-streams.ts.json create mode 100644 tests/harness/corpus-costs/7056-cipher-stream-errors.ts.json create mode 100644 tests/harness/corpus-costs/7057-cipher-streams.mjs.json create mode 100644 tests/harness/corpus-costs/7058-pipeline-function-stages.mjs.json create mode 100644 tests/harness/corpus-costs/7059-transform-backpressure.ts.json diff --git a/docs/content/limitations/builtin-apis/crypto.mdx b/docs/content/limitations/builtin-apis/crypto.mdx index 3c830c63d..9ca6bfff4 100644 --- a/docs/content/limitations/builtin-apis/crypto.mdx +++ b/docs/content/limitations/builtin-apis/crypto.mdx @@ -8,7 +8,10 @@ `randomFill`, `randomFillSync` (every view kind and ArrayBuffer), `randomInt`, `randomUUID`, and `randomUUIDv7` follow Node 24.22, including callback timing; a `randomInt` callback whose error parameter is typed `Error | null` receives `null` where Node passes `undefined`. Scrypt supports its default parameters, numeric options, aliases, and memory limit. Both KDFs accept strings, Buffers, numeric typed arrays, DataViews, and fixed-length ArrayBuffers. - `createCipheriv` and `createDecipheriv` support aes-128, aes-192, and aes-256 in CBC, CTR, and GCM modes and chacha20-poly1305, on native targets, through `update()`, `final()`, `setAAD()`, `setAutoPadding()`, `getAuthTag()`, and `setAuthTag()`; other ciphers, cipher streams, and wasm32-wasi builds are refused. + `createCipheriv` and `createDecipheriv` support aes-128, aes-192, and aes-256 in CBC, CTR, and GCM modes and chacha20-poly1305, on native targets, through `update()`, `final()`, `setAAD()`, `setAutoPadding()`, `getAuthTag()`, and `setAuthTag()`; other ciphers and wasm32-wasi builds are refused. + Cipheriv and Decipheriv values are also Transforms: they can be written, piped into, used as `pipeline()` stages, read, iterated with `for await`, and listened to, and their stream properties read as in Node; written chunks go through `update()`, and the end pushes `final()` or reports its error through `'error'`. + Through `unknown` and `any` slots and untyped JavaScript, in a program that uses streams, their stream properties and their `on`, `once`, `off`, `pipe`, `unpipe`, `read`, `pause`, `resume`, `setEncoding`, `write`, `end`, `destroy`, `cork`, and `uncork` members work; `write` and `end` take no encoding or callback there, and the other stream members throw a coded SC2020 error. + Hash, Hmac, Cipheriv, and Decipheriv streams count a written string's bytes in `writableLength`, where Node counts the string's length. Decrypting chacha20-poly1305 without an authentication tag fails in `final()`; Node 24.22 returns unverified plaintext. After `update()` fails, for example past the AES-GCM limit of 2^36 - 32 bytes per message, every later call on that cipher throws; Node still lets `final()` run. An output encoding that is a function or an instance of a class defining `toString` throws a coded SC2020 error. diff --git a/docs/content/limitations/builtin-apis/streams-and-readline.mdx b/docs/content/limitations/builtin-apis/streams-and-readline.mdx index a21f7a617..b7c2e062e 100644 --- a/docs/content/limitations/builtin-apis/streams-and-readline.mdx +++ b/docs/content/limitations/builtin-apis/streams-and-readline.mdx @@ -3,14 +3,16 @@ `readable.pipe(process.stdout)` and `readable.pipe(process.stderr)` compile and, as in Node, do not end the process stream. When an untyped value reaches `NodeJS.ReadableStream` or `NodeJS.WritableStream`, those two interfaces hold any value, so JavaScript stream objects (constructor-function streams, plain objects with `on` and `pipe`) pass through TypeScript functions typed with them and come back to JavaScript as the same objects. Member calls on such a value use the native stream when it is one and call the object's own members otherwise; `readable.pipe()` into a JavaScript destination follows Node's pipe protocol. - A `pipeline` stage and a `for await` source that arrive as untyped values must be native streams at run time; other values throw a not-supported error. + A `pipeline` stage and a `for await` source that arrive as untyped values must be native streams, or `Cipheriv` and `Decipheriv` values, at run time; other values throw a not-supported error. Native streams expose no `emit` to JavaScript, so a JavaScript stream whose own `pipe()` emits on its destination (the legacy `Stream` pipe) cannot pipe into one. `Readable.from` accepts arrays, Sets, Map and Set iterators, generators, async generators, and class iterables and async iterables whose values are strings or Buffers, and in JavaScript any iterable or async iterable of them; a stream that mixes strings and Buffers is unsupported. In JavaScript, a sync iterator value with a `then` method that is not a promise, and an object whose `[Symbol.asyncIterator]` or `[Symbol.iterator]` method is a generator method, are unsupported. Object-mode streams of other values (including `Readable.from` over a `Map`, whose values are entry arrays) are unsupported. - `pipeline` accepts one run of function stages: async generator functions between streams (`async function* (source)`, iterating `source` with `for await`), an async generator source function (`async function* ({ signal })`), and an async function destination whose value resolves the promise form and reaches the callback's second argument. + `pipeline` accepts function stages anywhere among its streams: async generator functions (`async function* (source)`, iterating `source` with `for await`), an async generator source function (`async function* ({ signal })`), and an async function destination whose value resolves the promise form and reaches the callback's second argument. + A stage is written inline, or named by a function declaration or a `const` bound to a function expression that nothing reassigns. + The source may also be an array, a string, a synchronous generator object, or a synchronous generator function without parameters; their elements reach the next stage on the same ticks as in Node. Stages may take the `{ signal }` argument. - An async generator function as the destination, using the PassThrough that `pipeline()` returns for a function destination, several runs of function stages, and function stages together with pipeline options are unsupported. + An async generator function as the destination, using the PassThrough that `pipeline()` returns for a function destination, and function stages or iterable sources together with pipeline options are unsupported. `Readable.from` accepts `objectMode` and `highWaterMark` options. The `Readable` operators (`map`, `filter`, `forEach`, `toArray`, `some`, `find`, `every`, `flatMap`, `drop`, `take`, and `reduce`) accept `concurrency`, `highWaterMark`, and `signal` options; their callbacks take only the chunk, `map` returns strings or Buffers, and `flatMap` returns strings or arrays of them. `readable.compose` is unsupported. diff --git a/docs/src/generated/node-v24-compatibility.json b/docs/src/generated/node-v24-compatibility.json index 66b0ba3dd..d77322eb5 100644 --- a/docs/src/generated/node-v24-compatibility.json +++ b/docs/src/generated/node-v24-compatibility.json @@ -12581,7 +12581,7 @@ }, "static": { "status": "partial", - "detail": "Matches Node's update() output lengths, padding, authentication tags, string encodings, and state errors for the supported ciphers; the stream.Transform interface is refused.", + "detail": "Matches Node's update() output lengths, padding, authentication tags, string encodings, and state errors for the supported ciphers, and its stream.Transform behavior: chunks written through update(), final() pushed at the end, and its errors reported through 'error'.", "verification": "test-backed" } }, @@ -12604,7 +12604,7 @@ }, "static": { "status": "partial", - "detail": "Matches Node's update() output lengths, padding, authentication tags, string encodings, and state errors for the supported ciphers; the stream.Transform interface is refused.", + "detail": "Matches Node's update() output lengths, padding, authentication tags, string encodings, and state errors for the supported ciphers, and its stream.Transform behavior: chunks written through update(), final() pushed at the end, and its errors reported through 'error'.", "verification": "test-backed" } }, @@ -12627,7 +12627,7 @@ }, "static": { "status": "partial", - "detail": "Matches Node's update() output lengths, padding, authentication tags, string encodings, and state errors for the supported ciphers; the stream.Transform interface is refused.", + "detail": "Matches Node's update() output lengths, padding, authentication tags, string encodings, and state errors for the supported ciphers, and its stream.Transform behavior: chunks written through update(), final() pushed at the end, and its errors reported through 'error'.", "verification": "test-backed" } }, @@ -12650,7 +12650,7 @@ }, "static": { "status": "partial", - "detail": "Matches Node's update() output lengths, padding, authentication tags, string encodings, and state errors for the supported ciphers; the stream.Transform interface is refused.", + "detail": "Matches Node's update() output lengths, padding, authentication tags, string encodings, and state errors for the supported ciphers, and its stream.Transform behavior: chunks written through update(), final() pushed at the end, and its errors reported through 'error'.", "verification": "test-backed" } }, @@ -12673,7 +12673,7 @@ }, "static": { "status": "partial", - "detail": "Matches Node's update() output lengths, padding, authentication tags, string encodings, and state errors for the supported ciphers; the stream.Transform interface is refused.", + "detail": "Matches Node's update() output lengths, padding, authentication tags, string encodings, and state errors for the supported ciphers, and its stream.Transform behavior: chunks written through update(), final() pushed at the end, and its errors reported through 'error'.", "verification": "test-backed" } }, @@ -12696,7 +12696,7 @@ }, "static": { "status": "partial", - "detail": "Matches Node's update() output lengths, padding, authentication tags, string encodings, and state errors for the supported ciphers; the stream.Transform interface is refused.", + "detail": "Matches Node's update() output lengths, padding, authentication tags, string encodings, and state errors for the supported ciphers, and its stream.Transform behavior: chunks written through update(), final() pushed at the end, and its errors reported through 'error'.", "verification": "test-backed" } }, @@ -12719,7 +12719,7 @@ }, "static": { "status": "partial", - "detail": "Matches Node's update() output lengths, padding, authentication tags, string encodings, and state errors for the supported ciphers; the stream.Transform interface is refused.", + "detail": "Matches Node's update() output lengths, padding, authentication tags, string encodings, and state errors for the supported ciphers, and its stream.Transform behavior: chunks written through update(), final() pushed at the end, and its errors reported through 'error'.", "verification": "test-backed" } }, @@ -12742,7 +12742,7 @@ }, "static": { "status": "partial", - "detail": "Matches Node's update() output lengths, padding, authentication tags, string encodings, and state errors for the supported ciphers; the stream.Transform interface is refused.", + "detail": "Matches Node's update() output lengths, padding, authentication tags, string encodings, and state errors for the supported ciphers, and its stream.Transform behavior: chunks written through update(), final() pushed at the end, and its errors reported through 'error'.", "verification": "test-backed" } }, @@ -12765,7 +12765,7 @@ }, "static": { "status": "partial", - "detail": "Matches Node's update() output lengths, padding, authentication tags, string encodings, and state errors for the supported ciphers; the stream.Transform interface is refused.", + "detail": "Matches Node's update() output lengths, padding, authentication tags, string encodings, and state errors for the supported ciphers, and its stream.Transform behavior: chunks written through update(), final() pushed at the end, and its errors reported through 'error'.", "verification": "test-backed" } }, @@ -12788,7 +12788,7 @@ }, "static": { "status": "partial", - "detail": "Matches Node's update() output lengths, padding, authentication tags, string encodings, and state errors for the supported ciphers; the stream.Transform interface is refused.", + "detail": "Matches Node's update() output lengths, padding, authentication tags, string encodings, and state errors for the supported ciphers, and its stream.Transform behavior: chunks written through update(), final() pushed at the end, and its errors reported through 'error'.", "verification": "test-backed" } }, @@ -12811,7 +12811,7 @@ }, "static": { "status": "partial", - "detail": "Matches Node's update() output lengths, padding, authentication tags, string encodings, and state errors for the supported ciphers; the stream.Transform interface is refused.", + "detail": "Matches Node's update() output lengths, padding, authentication tags, string encodings, and state errors for the supported ciphers, and its stream.Transform behavior: chunks written through update(), final() pushed at the end, and its errors reported through 'error'.", "verification": "test-backed" } }, @@ -12834,7 +12834,7 @@ }, "static": { "status": "partial", - "detail": "Matches Node's update() output lengths, padding, authentication tags, string encodings, and state errors for the supported ciphers; the stream.Transform interface is refused.", + "detail": "Matches Node's update() output lengths, padding, authentication tags, string encodings, and state errors for the supported ciphers, and its stream.Transform behavior: chunks written through update(), final() pushed at the end, and its errors reported through 'error'.", "verification": "test-backed" } }, diff --git a/internal/compatibility/generated/node-v24-internal.json b/internal/compatibility/generated/node-v24-internal.json index c6a7a102c..537b8f05a 100644 --- a/internal/compatibility/generated/node-v24-internal.json +++ b/internal/compatibility/generated/node-v24-internal.json @@ -19313,6 +19313,9 @@ "tests/corpus/6703-ecdh-private-keys.ts", "tests/corpus/6704-ecdh-set-public-key.cjs", "tests/corpus/6705-ecdh-handles.ts", + "tests/corpus/7055-cipher-streams.ts", + "tests/corpus/7056-cipher-stream-errors.ts", + "tests/corpus/7057-cipher-streams.mjs", "tests/corpus/node24-crypto-kdf-errors.ts", "tests/corpus/node24-crypto-kdf-options.ts", "tests/corpus/node24-crypto-kdf.ts", @@ -19700,6 +19703,9 @@ "tests/corpus/5257-cipher-vectors.ts", "tests/corpus/5258-cipher-handles.js", "tests/corpus/5259-cipher-chunked.ts", + "tests/corpus/7055-cipher-streams.ts", + "tests/corpus/7056-cipher-stream-errors.ts", + "tests/corpus/7057-cipher-streams.mjs", "tests/fixtures/node-types/cipher.ts" ] }, @@ -19736,6 +19742,9 @@ "tests/corpus/5257-cipher-vectors.ts", "tests/corpus/5258-cipher-handles.js", "tests/corpus/5259-cipher-chunked.ts", + "tests/corpus/7055-cipher-streams.ts", + "tests/corpus/7056-cipher-stream-errors.ts", + "tests/corpus/7057-cipher-streams.mjs", "tests/fixtures/node-types/cipher.ts" ] }, @@ -21734,6 +21743,9 @@ "tests/corpus/5257-cipher-vectors.ts", "tests/corpus/5258-cipher-handles.js", "tests/corpus/5259-cipher-chunked.ts", + "tests/corpus/7055-cipher-streams.ts", + "tests/corpus/7056-cipher-stream-errors.ts", + "tests/corpus/7057-cipher-streams.mjs", "tests/fixtures/node-types/cipher.ts" ] }, @@ -21770,6 +21782,9 @@ "tests/corpus/5257-cipher-vectors.ts", "tests/corpus/5258-cipher-handles.js", "tests/corpus/5259-cipher-chunked.ts", + "tests/corpus/7055-cipher-streams.ts", + "tests/corpus/7056-cipher-stream-errors.ts", + "tests/corpus/7057-cipher-streams.mjs", "tests/fixtures/node-types/cipher.ts" ] }, @@ -21806,6 +21821,9 @@ "tests/corpus/5257-cipher-vectors.ts", "tests/corpus/5258-cipher-handles.js", "tests/corpus/5259-cipher-chunked.ts", + "tests/corpus/7055-cipher-streams.ts", + "tests/corpus/7056-cipher-stream-errors.ts", + "tests/corpus/7057-cipher-streams.mjs", "tests/fixtures/node-types/cipher.ts" ] }, @@ -21842,6 +21860,9 @@ "tests/corpus/5257-cipher-vectors.ts", "tests/corpus/5258-cipher-handles.js", "tests/corpus/5259-cipher-chunked.ts", + "tests/corpus/7055-cipher-streams.ts", + "tests/corpus/7056-cipher-stream-errors.ts", + "tests/corpus/7057-cipher-streams.mjs", "tests/fixtures/node-types/cipher.ts" ] }, @@ -21878,6 +21899,9 @@ "tests/corpus/5257-cipher-vectors.ts", "tests/corpus/5258-cipher-handles.js", "tests/corpus/5259-cipher-chunked.ts", + "tests/corpus/7055-cipher-streams.ts", + "tests/corpus/7056-cipher-stream-errors.ts", + "tests/corpus/7057-cipher-streams.mjs", "tests/fixtures/node-types/cipher.ts" ] }, @@ -21914,6 +21938,9 @@ "tests/corpus/5257-cipher-vectors.ts", "tests/corpus/5258-cipher-handles.js", "tests/corpus/5259-cipher-chunked.ts", + "tests/corpus/7055-cipher-streams.ts", + "tests/corpus/7056-cipher-stream-errors.ts", + "tests/corpus/7057-cipher-streams.mjs", "tests/fixtures/node-types/cipher.ts" ] }, @@ -21950,6 +21977,9 @@ "tests/corpus/5257-cipher-vectors.ts", "tests/corpus/5258-cipher-handles.js", "tests/corpus/5259-cipher-chunked.ts", + "tests/corpus/7055-cipher-streams.ts", + "tests/corpus/7056-cipher-stream-errors.ts", + "tests/corpus/7057-cipher-streams.mjs", "tests/fixtures/node-types/cipher.ts" ] }, @@ -21986,6 +22016,9 @@ "tests/corpus/5257-cipher-vectors.ts", "tests/corpus/5258-cipher-handles.js", "tests/corpus/5259-cipher-chunked.ts", + "tests/corpus/7055-cipher-streams.ts", + "tests/corpus/7056-cipher-stream-errors.ts", + "tests/corpus/7057-cipher-streams.mjs", "tests/fixtures/node-types/cipher.ts" ] }, @@ -22022,6 +22055,9 @@ "tests/corpus/5257-cipher-vectors.ts", "tests/corpus/5258-cipher-handles.js", "tests/corpus/5259-cipher-chunked.ts", + "tests/corpus/7055-cipher-streams.ts", + "tests/corpus/7056-cipher-stream-errors.ts", + "tests/corpus/7057-cipher-streams.mjs", "tests/fixtures/node-types/cipher.ts" ] }, @@ -22058,6 +22094,9 @@ "tests/corpus/5257-cipher-vectors.ts", "tests/corpus/5258-cipher-handles.js", "tests/corpus/5259-cipher-chunked.ts", + "tests/corpus/7055-cipher-streams.ts", + "tests/corpus/7056-cipher-stream-errors.ts", + "tests/corpus/7057-cipher-streams.mjs", "tests/fixtures/node-types/cipher.ts" ] }, @@ -22094,6 +22133,9 @@ "tests/corpus/5257-cipher-vectors.ts", "tests/corpus/5258-cipher-handles.js", "tests/corpus/5259-cipher-chunked.ts", + "tests/corpus/7055-cipher-streams.ts", + "tests/corpus/7056-cipher-stream-errors.ts", + "tests/corpus/7057-cipher-streams.mjs", "tests/fixtures/node-types/cipher.ts" ] }, @@ -22130,6 +22172,9 @@ "tests/corpus/5257-cipher-vectors.ts", "tests/corpus/5258-cipher-handles.js", "tests/corpus/5259-cipher-chunked.ts", + "tests/corpus/7055-cipher-streams.ts", + "tests/corpus/7056-cipher-stream-errors.ts", + "tests/corpus/7057-cipher-streams.mjs", "tests/fixtures/node-types/cipher.ts" ] }, diff --git a/internal/compatibility/src/generate.mjs b/internal/compatibility/src/generate.mjs index 9bd3927b8..cf77c1f84 100644 --- a/internal/compatibility/src/generate.mjs +++ b/internal/compatibility/src/generate.mjs @@ -594,7 +594,7 @@ function publicDetail(tier) { return "Matches Node's key formats, point encodings, private-key range checks, key-pair checks and errors on the supported curves; ECDH.convertKey is supported, and property reads and writes are refused."; } if (/^compiler-dedicated:(?:Cipheriv|Decipheriv|cipher\.|decipher\.)/.test(source)) { - return "Matches Node's update() output lengths, padding, authentication tags, string encodings, and state errors for the supported ciphers; the stream.Transform interface is refused."; + return "Matches Node's update() output lengths, padding, authentication tags, string encodings, and state errors for the supported ciphers, and its stream.Transform behavior: chunks written through update(), final() pushed at the end, and its errors reported through 'error'."; } if (source.startsWith("compiler-dedicated:")) return "Implemented by a dedicated static compiler/runtime path."; diff --git a/internal/compatibility/static-support.json b/internal/compatibility/static-support.json index 23c7a159b..0e608148f 100644 --- a/internal/compatibility/static-support.json +++ b/internal/compatibility/static-support.json @@ -42,8 +42,8 @@ "node-builtin.child_process": ["tests/corpus/1361-spawn-events.ts", "tests/corpus/1565-spawn-pipe-streams.ts", "tests/corpus/2911-execfile-callback.ts", "tests/corpus/2939-child-shell-text.ts", "tests/corpus/2946-child-fork-ipc/main.ts", "tests/corpus/3127-spawn-runtime-options.ts", "tests/corpus/3130-child-generic-streams.ts", "tests/corpus/3131-child-rest-listeners.js", "tests/corpus/3132-child-generic-stream-completion.ts"], "node-builtin.cluster": ["tests/corpus/957-builtins-namespace.ts"], "node-builtin.console": ["tests/corpus/console-imports.cjs", "tests/corpus/2440-console-inspect-args.ts", "tests/corpus/console-native-value.js", "tests/harness/npm-static.test.ts"], - "node-builtin.crypto.createCipheriv": ["tests/corpus/5250-cipher-aes-gcm.ts", "tests/corpus/5251-cipher-aes-cbc.ts", "tests/corpus/5252-cipher-aes-ctr.ts", "tests/corpus/5253-cipher-chacha20-poly1305.ts", "tests/corpus/5254-cipher-errors.js", "tests/corpus/5255-cipher-encodings.cjs", "tests/corpus/5256-cipher-deprecation.cjs", "tests/corpus/5257-cipher-vectors.ts", "tests/corpus/5258-cipher-handles.js", "tests/corpus/5259-cipher-chunked.ts", "tests/fixtures/node-types/cipher.ts"], - "node-builtin.crypto.createDecipheriv": ["tests/corpus/5250-cipher-aes-gcm.ts", "tests/corpus/5251-cipher-aes-cbc.ts", "tests/corpus/5252-cipher-aes-ctr.ts", "tests/corpus/5253-cipher-chacha20-poly1305.ts", "tests/corpus/5254-cipher-errors.js", "tests/corpus/5255-cipher-encodings.cjs", "tests/corpus/5256-cipher-deprecation.cjs", "tests/corpus/5257-cipher-vectors.ts", "tests/corpus/5258-cipher-handles.js", "tests/corpus/5259-cipher-chunked.ts", "tests/fixtures/node-types/cipher.ts"], + "node-builtin.crypto.createCipheriv": ["tests/corpus/5250-cipher-aes-gcm.ts", "tests/corpus/5251-cipher-aes-cbc.ts", "tests/corpus/5252-cipher-aes-ctr.ts", "tests/corpus/5253-cipher-chacha20-poly1305.ts", "tests/corpus/5254-cipher-errors.js", "tests/corpus/5255-cipher-encodings.cjs", "tests/corpus/5256-cipher-deprecation.cjs", "tests/corpus/5257-cipher-vectors.ts", "tests/corpus/5258-cipher-handles.js", "tests/corpus/5259-cipher-chunked.ts", "tests/corpus/7055-cipher-streams.ts", "tests/corpus/7056-cipher-stream-errors.ts", "tests/corpus/7057-cipher-streams.mjs", "tests/fixtures/node-types/cipher.ts"], + "node-builtin.crypto.createDecipheriv": ["tests/corpus/5250-cipher-aes-gcm.ts", "tests/corpus/5251-cipher-aes-cbc.ts", "tests/corpus/5252-cipher-aes-ctr.ts", "tests/corpus/5253-cipher-chacha20-poly1305.ts", "tests/corpus/5254-cipher-errors.js", "tests/corpus/5255-cipher-encodings.cjs", "tests/corpus/5256-cipher-deprecation.cjs", "tests/corpus/5257-cipher-vectors.ts", "tests/corpus/5258-cipher-handles.js", "tests/corpus/5259-cipher-chunked.ts", "tests/corpus/7055-cipher-streams.ts", "tests/corpus/7056-cipher-stream-errors.ts", "tests/corpus/7057-cipher-streams.mjs", "tests/fixtures/node-types/cipher.ts"], "node-builtin.crypto.createECDH": ["tests/corpus/6700-ecdh-shared-secret.ts", "tests/corpus/6701-ecdh-formats.ts", "tests/corpus/6702-ecdh-errors.js", "tests/corpus/6703-ecdh-private-keys.ts", "tests/corpus/6704-ecdh-set-public-key.cjs", "tests/corpus/6705-ecdh-handles.ts", "tests/fixtures/node-types/ecdh.ts"], "node-builtin.crypto": ["tests/corpus/1358-crypto-random.ts", "tests/corpus/1534-crypto-hash-chain.ts", "tests/corpus/2556-crypto-introspection.ts", "tests/corpus/2890-crypto-static-utility.ts", "tests/harness/npm-static.test.ts", "tests/corpus/node24-crypto-sha2.ts", "tests/corpus/node24-crypto-kdf.ts", "tests/corpus/node24-crypto-kdf-errors.ts", "tests/corpus/node24-crypto-kdf-options.ts"], "node-builtin.dgram": ["tests/corpus/2597-dgram-send-ladders.cjs"], @@ -412,7 +412,7 @@ { "symbols": ["assert.partialDeepStrictEqual"], "status": "partial", "evidence": ["tests/corpus/5517-assert-partial-deep-equal.js", "tests/corpus/5518-assert-partial-deep-typed.ts", "tests/corpus/5519-assert-partial-deep-typed-edges.ts"] }, { "symbols": ["assert.deepEqual", "assert.notDeepEqual"], "status": "partial", "evidence": ["tests/corpus/5513-assert-deep-equal-loose.js", "tests/corpus/5514-assert-deep-equal-typed.ts", "tests/corpus/5516-assert-deep-equal-edges.js"] }, { "symbols": ["assert", "assert.ok", "assert.equal", "assert.notEqual", "assert.strictEqual", "assert.notStrictEqual", "assert.deepStrictEqual", "assert.notDeepStrictEqual", "assert.fail", "assert.match", "assert.doesNotMatch", "assert.throws", "assert.rejects", "assert.ifError"], "status": "partial", "evidence": ["tests/corpus/1600-assert-passing.ts", "tests/corpus/1604-assert-deep-structures.ts", "tests/corpus/1609-assert-async.ts", "tests/corpus/1721-assert-throws-regex-class.ts", "tests/corpus/2936-static-loose-equality.ts", "tests/corpus/5515-assert-deep-strict-typed-messages.ts"] }, - { "chapter": "crypto", "symbols": ["Cipheriv", "cipher.update", "cipher.final", "cipher.setAAD", "cipher.getAuthTag", "cipher.setAutoPadding", "Decipheriv", "decipher.update", "decipher.final", "decipher.setAAD", "decipher.setAuthTag", "decipher.setAutoPadding"], "status": "partial", "evidence": ["tests/corpus/5250-cipher-aes-gcm.ts", "tests/corpus/5251-cipher-aes-cbc.ts", "tests/corpus/5252-cipher-aes-ctr.ts", "tests/corpus/5253-cipher-chacha20-poly1305.ts", "tests/corpus/5254-cipher-errors.js", "tests/corpus/5255-cipher-encodings.cjs", "tests/corpus/5256-cipher-deprecation.cjs", "tests/corpus/5257-cipher-vectors.ts", "tests/corpus/5258-cipher-handles.js", "tests/corpus/5259-cipher-chunked.ts", "tests/fixtures/node-types/cipher.ts"] }, + { "chapter": "crypto", "symbols": ["Cipheriv", "cipher.update", "cipher.final", "cipher.setAAD", "cipher.getAuthTag", "cipher.setAutoPadding", "Decipheriv", "decipher.update", "decipher.final", "decipher.setAAD", "decipher.setAuthTag", "decipher.setAutoPadding"], "status": "partial", "evidence": ["tests/corpus/5250-cipher-aes-gcm.ts", "tests/corpus/5251-cipher-aes-cbc.ts", "tests/corpus/5252-cipher-aes-ctr.ts", "tests/corpus/5253-cipher-chacha20-poly1305.ts", "tests/corpus/5254-cipher-errors.js", "tests/corpus/5255-cipher-encodings.cjs", "tests/corpus/5256-cipher-deprecation.cjs", "tests/corpus/5257-cipher-vectors.ts", "tests/corpus/5258-cipher-handles.js", "tests/corpus/5259-cipher-chunked.ts", "tests/corpus/7055-cipher-streams.ts", "tests/corpus/7056-cipher-stream-errors.ts", "tests/corpus/7057-cipher-streams.mjs", "tests/fixtures/node-types/cipher.ts"] }, { "chapter": "crypto", "symbols": ["ECDH", "ecdh.computeSecret", "ecdh.generateKeys", "ecdh.getPrivateKey", "ecdh.getPublicKey", "ecdh.setPrivateKey", "ecdh.setPublicKey"], "status": "partial", "evidence": ["tests/corpus/6700-ecdh-shared-secret.ts", "tests/corpus/6701-ecdh-formats.ts", "tests/corpus/6702-ecdh-errors.js", "tests/corpus/6703-ecdh-private-keys.ts", "tests/corpus/6704-ecdh-set-public-key.cjs", "tests/corpus/6705-ecdh-handles.ts", "tests/fixtures/node-types/ecdh.ts"] }, { "chapter": "errors", "symbols": ["ERR_CRYPTO_ECDH_INVALID_FORMAT", "ERR_CRYPTO_ECDH_INVALID_PUBLIC_KEY"], "status": "partial", "evidence": ["tests/corpus/6702-ecdh-errors.js"] }, { "symbols": ["crypto.createHash", "Hash", "hash.update", "hash.digest", "hash.copy", "crypto.createHmac", "Hmac", "hmac.update", "hmac.digest", "crypto.hash", "crypto.pbkdf2", "crypto.pbkdf2Sync"], "status": "partial", "evidence": ["tests/corpus/5760-crypto-hash-stream.ts", "tests/corpus/5768-crypto-hash-unknown.ts", "tests/corpus/5769-crypto-hash-unknown.cjs", "tests/corpus/5761-crypto-hash-stream-states.ts", "tests/corpus/5762-crypto-hash-stream.cjs", "tests/corpus/2890-crypto-static-utility.ts", "tests/corpus/node24-crypto-sha2.ts", "tests/corpus/5435-crypto-hash-algorithms.ts", "tests/corpus/5436-crypto-hash-names.js", "tests/corpus/5437-crypto-hmac-algorithms.ts", "tests/corpus/5438-crypto-kdf-digests.ts", "tests/corpus/5440-crypto-hash-encodings.js", "tests/corpus/5441-crypto-hash-output-length.ts", "tests/harness/npm-static.test.ts"] }, diff --git a/packages/cli/test/native-link-info.test.ts b/packages/cli/test/native-link-info.test.ts index f5ff2642e..56ca46878 100644 --- a/packages/cli/test/native-link-info.test.ts +++ b/packages/cli/test/native-link-info.test.ts @@ -72,7 +72,7 @@ describe.runIf(supported)("macOS arm64 native link info", () => { minimum_os: "14.0", }, program: { object, entry_symbol: "main" }, - runtime_abi: { version: 24, marker: "scr_runtime_abi_v24" }, + runtime_abi: { version: 25, marker: "scr_runtime_abi_v25" }, runtime_pack: { kind: "precompiled", package: "@scriptc/runtime-darwin-arm64", @@ -165,6 +165,6 @@ describe.runIf(supported)("macOS arm64 native link info", () => { () => null, (failure: { stderr?: string }) => failure, ); - expect(error?.stderr).toContain("scr_runtime_abi_v24"); + expect(error?.stderr).toContain("scr_runtime_abi_v25"); }); }); diff --git a/packages/compiler/ambient/scriptc-node-fallback.d.ts b/packages/compiler/ambient/scriptc-node-fallback.d.ts index dbb3cd8fd..16d7b9845 100644 --- a/packages/compiler/ambient/scriptc-node-fallback.d.ts +++ b/packages/compiler/ambient/scriptc-node-fallback.d.ts @@ -2852,7 +2852,8 @@ declare module "crypto" { ): void; export type BinaryLike = string | ArrayBuffer | ArrayBufferView; /* Cipheriv/Decipheriv: aes-128/192/256 in cbc, ctr, and gcm modes and - * chacha20-poly1305. The stream.Transform members are not declared. */ + * chacha20-poly1305. Both are Transforms: written chunks go through + * update(), and the end pushes final(). */ export type CipherGCMTypes = "aes-128-gcm" | "aes-192-gcm" | "aes-256-gcm"; export type CipherChaCha20Poly1305Types = "chacha20-poly1305"; export type CipherKey = BinaryLike | KeyObject; @@ -2862,7 +2863,7 @@ declare module "crypto" { export interface CipherChaCha20Poly1305Options { authTagLength?: number; } - export interface Cipheriv { + export interface Cipheriv extends Transform { update(data: string | ArrayBufferView): Buffer; update(data: string, inputEncoding: BufferEncoding): Buffer; update(data: ArrayBufferView, inputEncoding: undefined, outputEncoding: BufferEncoding): string; @@ -2883,7 +2884,7 @@ declare module "crypto" { setAAD(buffer: ArrayBufferView, options?: { plaintextLength: number }): this; getAuthTag(): Buffer; } - export interface Decipheriv { + export interface Decipheriv extends Transform { update(data: ArrayBufferView): Buffer; update(data: string, inputEncoding: BufferEncoding): Buffer; update(data: ArrayBufferView, inputEncoding: undefined, outputEncoding: BufferEncoding): string; diff --git a/packages/compiler/src/backend/llvm/lib-shared.ts b/packages/compiler/src/backend/llvm/lib-shared.ts index 8bb8bc3aa..524d5939d 100644 --- a/packages/compiler/src/backend/llvm/lib-shared.ts +++ b/packages/compiler/src/backend/llvm/lib-shared.ts @@ -754,6 +754,7 @@ export const LIB_FN_SYMS: Record = { "procStream.write": "scr_proc_stream_write", "stream.stdioWritable": "scr_stream_stdio_writable", "stream.unboxDyn": "scr_stream_unbox_dyn", + "stream.handleStream": "scr_stream_handle", "stream.pipeDyn": "scr_stream_pipe_dyn", "stream.unpipeDyn": "scr_stream_unpipe_dyn", "process.kill": "scr_process_kill_named", diff --git a/packages/compiler/src/backend/llvm/lib-stream.ts b/packages/compiler/src/backend/llvm/lib-stream.ts index d3b4b1f5b..5d198d306 100644 --- a/packages/compiler/src/backend/llvm/lib-stream.ts +++ b/packages/compiler/src/backend/llvm/lib-stream.ts @@ -2,6 +2,7 @@ import { InternalCompilerError } from "../../errors.js"; import type { LlvmEmitterContext, LibCallExpr, LlValue } from "./expr-context.js"; import { f64Lit } from "./common.js"; +import { pipelinePumpedRuns } from "../../ir/builtin-signatures.js"; const STREAM_CANONICAL_CBS: Record = { readable: ["r", "d"], @@ -662,14 +663,14 @@ export function emitStreamLibCall(host: LlvmEmitterContext, e: LibCallExpr): LlV e.fn === "stream.pipelineStages" || e.fn === "stream.pipelineStagesDyn" ) { - // (n, at, k, flags, mask, s1..sn, stage1..stagek, (next, read, - // return)?, box?, (signal, abort)?[, cb]): the streams and the stage - // closures ride stack arrays; absent groups pass null. - const [nArg, atArg, kArg, flagsArg, maskArg] = e.args; + // (n, k, positions, flags, mask, s1..sn, stage1..stagek, helpers..., + // box?, (signal, abort)?[, cb]): the streams, the stage closures and + // the helpers ride stack arrays; absent groups pass null. + const [nArg, kArg, positionsArg, flagsArg, maskArg] = e.args; if ( nArg?.kind !== "numLit" || - atArg?.kind !== "numLit" || kArg?.kind !== "numLit" || + positionsArg?.kind !== "numLit" || flagsArg?.kind !== "numLit" || maskArg?.kind !== "numLit" ) @@ -677,6 +678,7 @@ export function emitStreamLibCall(host: LlvmEmitterContext, e: LibCallExpr): LlV const n = nArg.value; const k = kArg.value; const flags = flagsArg.value; + const helperCount = 3 * pipelinePumpedRuns(n + k, positionsArg.value); const args = e.args.map((a) => host.emitExpr(a)); const packed = (from: number, count: number): string => { if (count === 0) return "null"; @@ -691,20 +693,17 @@ export function emitStreamLibCall(host: LlvmEmitterContext, e: LibCallExpr): LlV }; const streams = packed(5, n); const stages = packed(5 + n, k); - let at = 5 + n + k; + const helpers = packed(5 + n + k, helperCount); + let at = 5 + n + k + helperCount; const take = (present: boolean): string => (present ? args[at++]!.name : "null"); - const hasHelpers = (flags & 8) !== 0; - const next = take(hasHelpers); - const read = take(hasHelpers); - const ret = take(hasHelpers); - const box = take((flags & 16) !== 0); + const box = take(((flags >> 1) & 3) === 2); const signal = take((flags & 32) !== 0); const abort = take((flags & 32) !== 0); const head = - `double ${f64Lit(n)}, ptr ${streams}, double ${f64Lit(atArg.value)}, double ${f64Lit(k)}, ` + - `ptr ${stages}, double ${f64Lit(flags)}, double ${f64Lit(maskArg.value)}, ` + - `ptr ${next}, ptr ${read}, ptr ${ret}, ptr ${box}, ptr ${signal}, ptr ${abort}`; - const params = "double, ptr, double, double, ptr, double, double, ptr, ptr, ptr, ptr, ptr, ptr"; + `double ${f64Lit(n)}, ptr ${streams}, double ${f64Lit(k)}, ptr ${stages}, ` + + `double ${f64Lit(positionsArg.value)}, double ${f64Lit(flags)}, double ${f64Lit(maskArg.value)}, ` + + `ptr ${helpers}, ptr ${box}, ptr ${signal}, ptr ${abort}`; + const params = "double, ptr, double, ptr, double, double, double, ptr, ptr, ptr, ptr"; const t = B.tmp(); if (e.fn === "sp.pipelineStages") { host.declare(`declare ptr @scr_sp_pipeline_stages(${params})`); diff --git a/packages/compiler/src/backend/runtime-abi.ts b/packages/compiler/src/backend/runtime-abi.ts index 5eb24482b..f02510338 100644 --- a/packages/compiler/src/backend/runtime-abi.ts +++ b/packages/compiler/src/backend/runtime-abi.ts @@ -2,8 +2,8 @@ * of the link contract: a program object keeps this symbol undefined and the * matching runtime defines it, so incompatible manual links fail before the * program can start. */ -export const RUNTIME_ABI_VERSION = 24 as const; -export const RUNTIME_ABI_MARKER = "scr_runtime_abi_v24" as const; +export const RUNTIME_ABI_VERSION = 25 as const; +export const RUNTIME_ABI_MARKER = "scr_runtime_abi_v25" as const; /** Object consumption is useful today, but the complete scr_* surface may * still change before 1.0. The versioned marker prevents accidental mixing; diff --git a/packages/compiler/src/backend/runtime-pack-targets.test.ts b/packages/compiler/src/backend/runtime-pack-targets.test.ts index fc7ed9efa..8bdbc8492 100644 --- a/packages/compiler/src/backend/runtime-pack-targets.test.ts +++ b/packages/compiler/src/backend/runtime-pack-targets.test.ts @@ -52,7 +52,7 @@ async function fixture(target: NativeTargetSpec, workers = false) { object_format: target.objectFormat, minimum_os: target.minimumOs, }, - runtime_abi: { version: 24, marker: "scr_runtime_abi_v24" }, + runtime_abi: { version: 25, marker: "scr_runtime_abi_v25" }, compiler: { command: "fixture", identity: "fixture", target: target.llvmTriple }, macros: { executable: [], excluded: ["SCR_LIB"], sanitizer: "external-toolchain-required" }, flavors: { diff --git a/packages/compiler/src/backend/runtime-pack.test.ts b/packages/compiler/src/backend/runtime-pack.test.ts index 5ffb4797d..a3ac9fe5d 100644 --- a/packages/compiler/src/backend/runtime-pack.test.ts +++ b/packages/compiler/src/backend/runtime-pack.test.ts @@ -124,7 +124,7 @@ async function fixture(target: NativeTargetSpec = MACOS_ARM64_TARGET) { object_format: target.objectFormat, minimum_os: target.minimumOs, }, - runtime_abi: { version: 24, marker: "scr_runtime_abi_v24" }, + runtime_abi: { version: 25, marker: "scr_runtime_abi_v25" }, compiler: { command: "clang", identity: "fixture clang", diff --git a/packages/compiler/src/frontend/lowering/binding-analysis.ts b/packages/compiler/src/frontend/lowering/binding-analysis.ts index 83b9aa6f9..1987c86ba 100644 --- a/packages/compiler/src/frontend/lowering/binding-analysis.ts +++ b/packages/compiler/src/frontend/lowering/binding-analysis.ts @@ -10,7 +10,13 @@ import { isGenericCallableMemberType } from "../type-mapper.js"; * Sound file-locally for module-scope bindings because ESM import * bindings are read-only — no other file can write one. Cached per * symbol (the scan walks the whole file once). */ -export function bindingNeverReassigned(lowerer: Lowerer, sym: ts.Symbol, decl: ts.Node): boolean { +export function bindingNeverReassigned( + lowerer: Lowerer, + sym: ts.Symbol, + decl: ts.Node, + resolve: (ident: ts.Identifier) => ts.Symbol | null | undefined = (ident) => + lowerer.resolveValueSymbol(ident), +): boolean { const cached = lowerer.neverReassignedCache.get(sym); if (cached !== undefined) return cached; let written = false; @@ -18,7 +24,7 @@ export function bindingNeverReassigned(lowerer: Lowerer, sym: ts.Symbol, decl: t // identifiers pay a symbol resolution. const symText = ts.symbolName(sym); const namesSym = (e: ts.Node): boolean => - ts.isIdentifier(e) && e.text === symText && lowerer.resolveValueSymbol(e) === sym; + ts.isIdentifier(e) && e.text === symText && resolve(e) === sym; const scanTarget = (t: ts.Expression): void => { let e: ts.Expression = t; while (ts.isParenthesizedExpression(e)) e = e.expression; diff --git a/packages/compiler/src/frontend/lowering/builtins/crypto.ts b/packages/compiler/src/frontend/lowering/builtins/crypto.ts index f4270dbe8..3dd3a4278 100644 --- a/packages/compiler/src/frontend/lowering/builtins/crypto.ts +++ b/packages/compiler/src/frontend/lowering/builtins/crypto.ts @@ -907,7 +907,8 @@ export function v8CalleeText(node: ts.Expression): string { /** Calls on native Cipheriv/Decipheriv handles dispatch through the * handle's methods with checked-dynamic arguments. The stream.Transform - * members the classes inherit are refused. */ + * members the classes inherit lower over the cipher's Transform + * (lowerHashStreamCall, which runs first); other members are refused. */ export function lowerCipherMethodCall( lowerer: Lowerer, call: ts.CallExpression, @@ -920,7 +921,7 @@ export function lowerCipherMethodCall( lowerer.noLowering( `Cipheriv/Decipheriv.${name}`, call, - "update(), final(), setAAD(), setAutoPadding(), getAuthTag(), and setAuthTag() are supported; cipher streams are not", + "update(), final(), setAAD(), setAutoPadding(), getAuthTag(), setAuthTag(), and the stream members are supported", ); } if (call.arguments.nodes.some(ts.isSpreadElement)) diff --git a/packages/compiler/src/frontend/lowering/builtins/hash-stream.ts b/packages/compiler/src/frontend/lowering/builtins/hash-stream.ts index 382d15eaf..f2f3625dd 100644 --- a/packages/compiler/src/frontend/lowering/builtins/hash-stream.ts +++ b/packages/compiler/src/frontend/lowering/builtins/hash-stream.ts @@ -1,17 +1,27 @@ import * as ts from "../../ts7/adapter.js"; import { type Lowerer } from "../lowerer.js"; import { locOf } from "../../program.js"; +import type { SrcLoc } from "../../../ir/ir.js"; import { varRef } from "../../../ir/build.js"; -import { type IrExpr, type IrType } from "../../../ir/ir.js"; +import { type IrExpr, type IrStmt, type IrType } from "../../../ir/ir.js"; +import { withStreamReceiver } from "../dyn-streams.js"; import { EMITTER_API_MEMBERS } from "../lower-classes.js"; import { lowerEmitterMethodCall } from "../lower-event-emitter.js"; -import { STREAM_API_MEMBERS, STREAM_OPERATORS, lowerStreamMethodCall } from "../lower-stream.js"; +import { + STREAM_API_MEMBERS, + STREAM_OPERATORS, + lowerStreamMethodCall, + lowerStreamProperty, +} from "../lower-stream.js"; +import { isCipherType } from "./crypto.js"; -/* node:crypto's Hash and Hmac are Transforms: a value written, read, - * listened to, or piped as a stream reads and writes through its own - * native Transform (scr_stream.c), created on first use. update() and - * digest() keep the value's static lowering; the stream members lower - * through the stream and emitter spokes over that Transform. */ +/* node:crypto's Hash, Hmac, Cipheriv and Decipheriv are Transforms: a + * value written, read, listened to, or piped as a stream reads and writes + * through its own native Transform, which scr_stream.c creates on first + * use. update(), digest() and final() keep the value's own lowering; the + * stream members lower through the stream and emitter spokes over that + * Transform. A Hash or Hmac is known by its IR type; a cipher is a + * checked-dynamic handle and is known by its checker type. */ const TRANSFORM_CLASS = "%Transform"; const TRANSFORM: IrType = { kind: "object", className: TRANSFORM_CLASS }; @@ -20,6 +30,12 @@ export function isHashValueType(type: IrType | null | undefined): boolean { return type?.kind === "cryptoHash" || type?.kind === "cryptoHmac"; } +/** Whether `node` evaluates to a Hash, Hmac, Cipheriv or Decipheriv. */ +export function isValueStream(lowerer: Lowerer, node: ts.Expression): boolean { + const type = lowerer.typeOf(node); + return isHashValueType(lowerer.mapTypeOf(type)) || isCipherType(lowerer, type); +} + /** The Transform of a lowered Hash or Hmac value (+1). A pipe * destination or pipeline stage (`piped`) also keeps the value alive * until its stream flushes or is destroyed. */ @@ -33,9 +49,27 @@ export function hashStreamOf(value: IrExpr, piped = false): IrExpr { }; } -/** A stream member called on a Hash or Hmac (`hash.write(chunk)`, - * `hash.on("data", cb)`, `hash.setEncoding("hex")`). Members that answer - * the stream (`this`) answer the Hash itself. */ +/** The Transform of a lowered Hash, Hmac, Cipheriv or Decipheriv value + * (+1), with hashStreamOf's `piped` (a cipher's stream always holds its + * cipher). */ +export function valueStreamOf(value: IrExpr, piped = false): IrExpr { + if (isHashValueType(value.type)) return hashStreamOf(value, piped); + // A value already lowered as its stream (addAbortSignal answers the + // Transform it was given, which its generic signature types as the + // cipher). + if (value.type.kind === "object") return value; + return { + kind: "libCall", + fn: "stream.handleStream", + args: [value], + type: TRANSFORM, + loc: value.loc, + }; +} + +/** A stream member called on a Hash, Hmac, Cipheriv or Decipheriv + * (`hash.write(chunk)`, `cipher.on("data", cb)`, `hash.setEncoding("hex")`). + * Members that answer the stream (`this`) answer the value itself. */ export function lowerHashStreamCall( lowerer: Lowerer, call: ts.CallExpression, @@ -45,21 +79,48 @@ export function lowerHashStreamCall( const name = access.name.text; const streamMember = STREAM_API_MEMBERS.has(name) || STREAM_OPERATORS.has(name); if (!streamMember && !EMITTER_API_MEMBERS.has(name)) return null; - if (!isHashValueType(lowerer.mapTypeOf(lowerer.typeOf(access.expression)))) return null; + if (!isValueStream(lowerer, access.expression)) return null; const info = lowerer.classes.get(TRANSFORM_CLASS); if (!info) return null; - const loc = locOf(call); - const hash = lowerer.lowerExpr(access.expression); - const local = lowerer.declareHiddenLocal("%hash", hash.type); - // The spokes lower the receiver to the value's Transform. - lowerer.hashStreamReceivers.set(access.expression, { localId: local.id, type: hash.type }); + return withValueStreamReceiver(lowerer, access.expression, locOf(call), () => + streamMember + ? lowerStreamMethodCall(lowerer, call, access, info) + : lowerEmitterMethodCall(lowerer, call, access, info), + ); +} + +/** A stream property read on a Hash, Hmac, Cipheriv or Decipheriv + * (`cipher.destroyed`, `hash.readableEnded`). */ +export function lowerValueStreamProperty( + lowerer: Lowerer, + expr: ts.PropertyAccessExpression, +): IrExpr | null { + if (expr.questionDotToken || !isValueStream(lowerer, expr.expression)) return null; + const info = lowerer.classes.get(TRANSFORM_CLASS); + if (!info) return null; + return withValueStreamReceiver(lowerer, expr.expression, locOf(expr), () => + lowerStreamProperty(lowerer, expr, info), + ); +} + +/** Evaluates the value `receiver` names once into a hidden local and runs + * `lower` with the receiver lowering to that value's Transform. A result + * typed as the Transform (a member that answers `this`) answers the value + * itself. */ +function withValueStreamReceiver( + lowerer: Lowerer, + receiver: ts.Expression, + loc: SrcLoc, + lower: () => IrExpr | null, +): IrExpr | null { + const value = lowerer.lowerExpr(receiver); + const local = lowerer.declareHiddenLocal("%streamValue", value.type); + lowerer.valueStreamReceivers.set(receiver, { localId: local.id, type: value.type }); let lowered: IrExpr | null; try { - lowered = streamMember - ? lowerStreamMethodCall(lowerer, call, access, info) - : lowerEmitterMethodCall(lowerer, call, access, info); + lowered = lower(); } finally { - lowerer.hashStreamReceivers.delete(access.expression); + lowerer.valueStreamReceivers.delete(receiver); } if (!lowered) return null; const answersStream = @@ -68,16 +129,42 @@ export function lowerHashStreamCall( ? { kind: "seqExpr", stmts: [{ kind: "exprStmt", expr: lowered, loc }], - result: varRef(local.id, hash.type, loc), - type: hash.type, + result: varRef(local.id, value.type, loc), + type: value.type, loc, } : lowered; return { kind: "seqExpr", - stmts: [{ kind: "varDecl", localId: local.id, init: hash, loc }], + stmts: [{ kind: "varDecl", localId: local.id, init: value, loc }], result, type: result.type, loc, }; } + +/** `for await (const chunk of value)` over a Hash, Hmac, Cipheriv or + * Decipheriv: the loop iterates the value's Transform. */ +export function lowerForAwaitValueStream( + lowerer: Lowerer, + stmt: ts.ForOfStatement, + lowerLoop: () => IrStmt | null, +): IrStmt | null { + if (!isValueStream(lowerer, stmt.expression)) return null; + const loc = locOf(stmt.expression); + const local = lowerer.declareHiddenLocal("%streamIterated", TRANSFORM); + const init = valueStreamOf(lowerer.lowerExpr(stmt.expression)); + const loop = withStreamReceiver( + lowerer, + stmt.expression, + varRef(local.id, TRANSFORM, loc), + TRANSFORM, + lowerLoop, + ); + if (loop === null) return null; + return { + kind: "block", + body: [{ kind: "varDecl", localId: local.id, init, loc }, loop], + loc: locOf(stmt), + }; +} diff --git a/packages/compiler/src/frontend/lowering/containers/bytes.ts b/packages/compiler/src/frontend/lowering/containers/bytes.ts index 4b20d7bf5..32fb03644 100644 --- a/packages/compiler/src/frontend/lowering/containers/bytes.ts +++ b/packages/compiler/src/frontend/lowering/containers/bytes.ts @@ -1078,11 +1078,20 @@ export function lowerBytesMethodCall( // Array-toString (comma join), a different operation: fenced. const declaredOnBuffer = (() => { const nameSym = lowerer.checker.getSymbolAtLocation(access.name); - return ( + if ( nameSym !== undefined && nameSym.declarations.some( (d) => ts.isInterfaceDeclaration(d.parent) && d.parent.name.text === "Buffer", ) + ) + return true; + // A receiver the lowering types as a Buffer where the checker says + // `any` (a JavaScript pipeline stage's chunk). + const receiverSym = lowerer.typeOf(access.expression).getSymbol(); + return ( + receiverSym !== undefined && + ts.symbolName(receiverSym) === "Buffer" && + receiverSym.declarations.some((d) => lowerer.isStdlibFile(ts.getSourceFileOfNode(d))) ); })(); if (receiverIr.elem !== "u8" || !declaredOnBuffer) { diff --git a/packages/compiler/src/frontend/lowering/containers/iterator-helpers.ts b/packages/compiler/src/frontend/lowering/containers/iterator-helpers.ts index 48b4caabd..ba5d0684d 100644 --- a/packages/compiler/src/frontend/lowering/containers/iterator-helpers.ts +++ b/packages/compiler/src/frontend/lowering/containers/iterator-helpers.ts @@ -1223,7 +1223,7 @@ class HostFrame implements ChainFrame { /** The locals and parameters of a lifted helper function under * construction. */ -class LiftedFrame implements ChainFrame { +export class LiftedFrame implements ChainFrame { readonly locals: IrLocal[] = []; readonly params: IrParam[] = []; private n = 0; @@ -1253,7 +1253,7 @@ class LiftedFrame implements ChainFrame { * step leave it unclosed, as in Node. The generator counts as unfinished * before the first step, so a `pre` that breaks at once (take(0)) closes * a source it never pulled. `genT` must be stepDrivable. */ -function driveGenerator( +export function driveGenerator( lowerer: Lowerer, frame: ChainFrame, genId: string, diff --git a/packages/compiler/src/frontend/lowering/dyn-streams.ts b/packages/compiler/src/frontend/lowering/dyn-streams.ts index 2ac09e8e5..2167eba23 100644 --- a/packages/compiler/src/frontend/lowering/dyn-streams.ts +++ b/packages/compiler/src/frontend/lowering/dyn-streams.ts @@ -76,7 +76,7 @@ function dynNativeStream( /** Runs `lower` with `node` lowering to `value` and its member calls taking * `type` as the receiver representation. */ -function withStreamReceiver( +export function withStreamReceiver( lowerer: Lowerer, node: ts.Expression, value: IrExpr, diff --git a/packages/compiler/src/frontend/lowering/expressions/property-access.ts b/packages/compiler/src/frontend/lowering/expressions/property-access.ts index 6c8dfa3bf..4d8e4d52c 100644 --- a/packages/compiler/src/frontend/lowering/expressions/property-access.ts +++ b/packages/compiler/src/frontend/lowering/expressions/property-access.ts @@ -132,6 +132,7 @@ import { lowerProtocolPropertyRead } from "../class-protocol-dispatch.js"; import { lowerReadlineProperty } from "../builtins/readline.js"; import { lowerEventStaticProperty } from "../builtins/numbers.js"; import { lowerStreamProperty, lowerStreamStateProperty, streamSidesOf } from "../lower-stream.js"; +import { lowerValueStreamProperty } from "../builtins/hash-stream.js"; import { lowerEventObjectProperty } from "../lower-event-target.js"; import { iteratorResultValueRead } from "../lower-generators.js"; import { lowerGlobalValue } from "../lower-global-value.js"; @@ -3995,6 +3996,8 @@ function lowerStreamObjectProperty( expr: ts.PropertyAccessExpression, ): IrExpr | null { if (expr.questionDotToken && !lowerer.chainHandled.has(expr)) return null; + const valueStream = lowerValueStreamProperty(lowerer, expr); + if (valueStream) return valueStream; const recvT = lowerer.mapTypeOf(lowerer.typeOf(expr.expression)); if (recvT?.kind !== "object") return null; const info = lowerer.classes.get(recvT.className); diff --git a/packages/compiler/src/frontend/lowering/lower-stmts.ts b/packages/compiler/src/frontend/lowering/lower-stmts.ts index e3480cf9c..bc3d2e869 100644 --- a/packages/compiler/src/frontend/lowering/lower-stmts.ts +++ b/packages/compiler/src/frontend/lowering/lower-stmts.ts @@ -3,6 +3,7 @@ import { declaredMemberOf, isPromisePrototypeRead } from "./expressions/property import { hostsAll } from "./expressions/hosted-narrowing.js"; import { requestGeneralStreamSurface } from "./general-streams.js"; import { lowerForAwaitDynStream } from "./dyn-streams.js"; +import { lowerForAwaitValueStream } from "./builtins/hash-stream.js"; import { lowerChildPropertyWrite } from "./handle-properties.js"; import { matchAllRowsReadByElements, @@ -9390,10 +9391,11 @@ export function lowerForOf(lowerer: Lowerer, stmt: ts.ForOfStatement): IrStmt { ) { return lowerForAwaitStdin(lowerer, stmt); } - // A checked-dynamic NodeJS.ReadableStream iterates its native stream. - const dynStream = lowerForAwaitDynStream(lowerer, stmt, () => - lowerForAwaitBuiltin(lowerer, stmt), - ); + // A checked-dynamic NodeJS.ReadableStream iterates its native stream, + // and a Hash, Hmac, Cipheriv or Decipheriv iterates its Transform. + const dynStream = + lowerForAwaitDynStream(lowerer, stmt, () => lowerForAwaitBuiltin(lowerer, stmt)) ?? + lowerForAwaitValueStream(lowerer, stmt, () => lowerForAwaitBuiltin(lowerer, stmt)); if (dynStream) return dynStream; const builtin = lowerForAwaitBuiltin(lowerer, stmt); if (builtin) return builtin; diff --git a/packages/compiler/src/frontend/lowering/lower-stream.ts b/packages/compiler/src/frontend/lowering/lower-stream.ts index a75ec43b7..ab21b8a7e 100644 --- a/packages/compiler/src/frontend/lowering/lower-stream.ts +++ b/packages/compiler/src/frontend/lowering/lower-stream.ts @@ -1,6 +1,6 @@ import { keyedHelperName } from "./stable-names.js"; import { nodeThrowExpr, boolLit, numLit, strLit } from "../../ir/build.js"; -import { hashStreamOf, isHashValueType } from "./builtins/hash-stream.js"; +import { isValueStream, valueStreamOf } from "./builtins/hash-stream.js"; import { InternalCompilerError } from "../../errors.js"; /* The node:stream lowering (the spoke-module pattern, like * lower-event-emitter.ts): construction of the five runtime stream classes @@ -52,7 +52,10 @@ import { dynUndefinedExpr, varRef } from "../../ir/build.js"; import { genResultRecord } from "../type-mapper.js"; import { noteUntypedStreamFlow, requestGeneralStreamSurface } from "./general-streams.js"; import { dynNativeBranch } from "./dyn-streams.js"; +import { bindingNeverReassigned } from "./binding-analysis.js"; import { extractIteratorValue, genStep, genValue, stepDrivable } from "./lower-generators.js"; +import { LiftedFrame, driveGenerator } from "./containers/iterator-helpers.js"; +import { lowerArrayIteratorValue, lowerStringIteratorValue } from "./iterator-adapters.js"; import { BOOL, DYN, @@ -74,6 +77,7 @@ import { funcOf, typeEquals, typeKey, + UNDEFINED_T, } from "../../ir/ir.js"; const BYTES = BYTES_U8; @@ -2084,10 +2088,10 @@ function lowerStreamArg( what: "finished" | "pipeline" | "addAbortSignal" | "compose", ): IrExpr { const lowered = lowerer.lowerExpr(node); - // A Hash or Hmac stage is its Transform; pipeline and compose stages - // keep the value alive until the stream settles. - const v = isHashValueType(lowered.type) - ? hashStreamOf(lowered, what === "pipeline" || what === "compose") + // A Hash, Hmac, Cipheriv or Decipheriv stage is its Transform; pipeline + // and compose stages keep the value alive until the stream settles. + const v = isValueStream(lowerer, node) + ? valueStreamOf(lowered, what === "pipeline" || what === "compose") : lowered; // A checked-dynamic pipeline stage (an untyped source or destination): // a boxed native stream unwraps at run time; other values (iterables, @@ -2429,10 +2433,10 @@ export function lowerStreamModuleCall( ? pipelineOptions(lowerer, lastArg) : null; const stageArgs = opts ? args.slice(0, -1) : args; - if (stageArgs.some(isPipelineFunctionStage)) { + if (hasPipelineFunctionStage(lowerer, stageArgs)) { if (opts) { lowerer.noLowering( - "pipeline with function stages and options", + "pipeline with function stages or iterable sources and options", lastArg, "pipeline(source, async function* (source) { ... }, destination) without options is supported", ); @@ -2503,7 +2507,10 @@ export function lowerStreamModuleCall( // (finished/pipeline throw synchronously; consumers do not). A // sync throw here would be observably wrong timing, so the // provable case takes the named fence instead. - const argV = lowerer.lowerExpr(args[0]!); + // A Hash, Hmac, Cipheriv or Decipheriv is read through its Transform. + const argV = isValueStream(lowerer, args[0]!) + ? valueStreamOf(lowerer.lowerExpr(args[0]!)) + : lowerer.lowerExpr(args[0]!); const argInfo = argV.type.kind === "object" ? lowerer.classes.get(argV.type.className) : undefined; if (argV.type.kind !== "dyn" && !streamSidesOf(lowerer, argInfo)) { @@ -2621,7 +2628,7 @@ export function lowerStreamModuleCall( "the supported form is pipeline(source, ...streams, callback)", ); } - if (args.slice(0, -1).some(isPipelineFunctionStage)) { + if (hasPipelineFunctionStage(lowerer, args.slice(0, -1))) { const staged = lowerPipelineStages(lowerer, args.slice(0, -1), loc); if (staged.dest !== 0) { // callback(err, value): the destination's value reaches the @@ -2680,19 +2687,215 @@ export function lowerStreamModuleCall( return null; } -/** A pipeline() argument written as a function: an async generator - * function stage. */ -function isPipelineFunctionStage(node: ts.Expression): boolean { - return ts.isFunctionExpression(node) || ts.isArrowFunction(node); +/** A pipeline() function stage: an inline function or arrow function, or + * an identifier bound to a function declaration or to a `const` + * initialized with one, which nothing reassigns. */ +type StageFunction = ts.FunctionExpression | ts.ArrowFunction | ts.FunctionDeclaration; + +/** The function a pipeline() argument writes or names, or null when the + * argument is not a function stage. */ +function pipelineStageFunction(lowerer: Lowerer, node: ts.Expression): StageFunction | null { + if (ts.isFunctionExpression(node) || ts.isArrowFunction(node)) return node; + if (!ts.isIdentifier(node)) return null; + // The stage lowers the declaration's body again with its source typed + // by the pipeline, so resolving it here must not report what collecting + // the declaration's own signature deferred (resolveValueSymbol would). + const resolve = (ident: ts.Identifier): ts.Symbol | undefined => { + const found = lowerer.checker.getSymbolAtLocation(ident); + return found && found.flags & ts.SymbolFlags.Alias + ? lowerer.checker.getAliasedSymbol(found) + : found; + }; + const symbol = resolve(node); + const declarations = symbol?.declarations ?? []; + const bodies = declarations.filter((d) => !ts.isFunctionDeclaration(d) || d.body !== undefined); + if (!symbol || bodies.length !== 1) return null; + const decl = bodies[0]!; + if (lowerer.isStdlibFile(ts.getSourceFileOfNode(decl))) return null; + if (ts.isFunctionDeclaration(decl)) { + return bindingNeverReassigned(lowerer, symbol, decl, resolve) ? decl : null; + } + if ( + ts.isVariableDeclaration(decl) && + decl.initializer && + ts.isVariableDeclarationList(decl.parent) && + (decl.parent.flags & ts.NodeFlags.Const) !== 0 + ) { + let init: ts.Expression = decl.initializer; + while (ts.isParenthesizedExpression(init)) init = init.expression; + if (ts.isFunctionExpression(init) || ts.isArrowFunction(init)) return init; + } + return null; } -/** The lowered function stages of a pipeline: the libcall head [n, at, k, - * flags, mask, streams..., stages..., (next, read, return)?, box?, +function hasPipelineFunctionStage(lowerer: Lowerer, args: readonly ts.Expression[]): boolean { + return ( + args.some((a) => pipelineStageFunction(lowerer, a) !== null) || + (args.length > 0 && iterableSourceType(lowerer, args[0]!) !== null) + ); +} + +type SyncGenType = IrType & { kind: "generator" }; + +/** The element type of a pipeline source that is an iterable value (an + * array, a string or a synchronous generator object), or null. */ +function iterableSourceType(lowerer: Lowerer, node: ts.Expression): IrType | null { + if (pipelineStageFunction(lowerer, node) !== null) return null; + let t: IrType | null; + try { + t = lowerer.mapTypeOf(lowerer.typeOf(node)); + } catch { + return null; + } + if (t?.kind === "array") return t.elem; + if (t?.kind === "string") return STRING; + if (t?.kind === "generator" && t.async !== true && stepDrivable(t)) return t.yieldT; + return null; +} + +/** A pipeline source that is an iterable value or a synchronous generator + * function without parameters, as an async generator closure that steps + * the value's iterator (or the generator the function returns) and yields + * each element. Node's pipeline iterates such a source with `for await`, + * through an async-from-sync iterator, and an async generator that + * yields the same elements resumes its consumer on the same ticks. Null + * when the source has no such lowering. */ +function lowerIterableSource( + lowerer: Lowerer, + node: ts.Expression, + fn: StageFunction | null, + loc: SrcLoc, +): { closure: IrExpr; out: SyncGenType } | null { + let value: IrExpr; + if (fn !== null) { + if (fn.parameters.nodes.length > 0) { + lowerer.noLowering( + "a synchronous generator function pipeline source with parameters", + fn.parameters.nodes[0]!, + "declare the source without parameters, or make it an async generator function", + ); + } + value = lowerer.lowerLambda(fn); + if (value.type.kind !== "func" || value.type.ret.kind !== "generator") return null; + } else { + value = lowerer.lowerExpr(node); + } + const srcT = value.type; + const elemT: IrType | null = + srcT.kind === "func" && srcT.ret.kind === "generator" + ? srcT.ret.yieldT + : srcT.kind === "array" + ? srcT.elem + : srcT.kind === "string" + ? STRING + : srcT.kind === "generator" && srcT.async !== true + ? srcT.yieldT + : null; + if (elemT === null) return null; + const out: SyncGenType = { + kind: "generator", + async: true, + yieldT: elemT, + retT: VOID, + nextT: UNDEFINED_T, + }; + const resultType = genResultRecord(elemT, VOID, lowerer.shapes, lowerer.unions); + if (!resultType) return null; + const key = `pipelineSource:${typeKey(srcT)}`; + let maker = lowerer.valueHelpers.get(key); + if (!maker) { + maker = `%stream.pipelineSource.${lowerer.valueHelpers.size}`; + const genName = `${maker}.gen`; + // The generator: g = the source's iterator; for (v of g) yield v. + const frame = new LiftedFrame(loc); + const srcId = frame.local("src", srcT, false); + const src = varRef(srcId, srcT, loc); + const syncT: SyncGenType = { + kind: "generator", + yieldT: elemT, + retT: VOID, + nextT: UNDEFINED_T, + }; + let iterator: IrExpr | null; + if (srcT.kind === "func") { + iterator = { kind: "callValue", callee: src, args: [], type: srcT.ret, loc }; + } else if (srcT.kind === "array") { + iterator = lowerArrayIteratorValue(lowerer, src, "values", syncT, loc); + } else if (srcT.kind === "string") { + iterator = lowerStringIteratorValue(lowerer, src, syncT, loc); + } else { + iterator = src; + } + if (iterator === null || iterator.type.kind !== "generator" || !stepDrivable(iterator.type)) + return null; + const iterT = iterator.type; + const gId = frame.local("g", iterT, false); + const body: IrStmt[] = [ + { kind: "varDecl", localId: gId, init: iterator, loc }, + ...driveGenerator( + lowerer, + frame, + gId, + iterT, + elemT, + [], + (v) => [{ kind: "exprStmt", expr: { kind: "yieldExpr", value: v, type: VOID, loc }, loc }], + loc, + ), + { kind: "return", value: null, loc }, + ]; + const srcLocal = frame.locals.find((l) => l.id === srcId)!; + srcLocal.boxed = true; + lowerer.valueHelpers.set(key, maker); + lowerer.liftedFns.push({ + name: genName, + params: [], + returnType: VOID, + async: true, + captures: [{ localId: srcId, name: "src", type: srcT }], + locals: frame.locals, + body, + generator: { yieldT: elemT, nextT: UNDEFINED_T, resultType }, + loc, + }); + // The maker: (src) => the generator function closed over src. + const makerSrc: IrLocal = { id: "src.0", name: "src", type: srcT, mutable: false, boxed: true }; + lowerer.liftedFns.push({ + name: maker, + params: [{ localId: makerSrc.id, name: makerSrc.name, type: srcT }], + returnType: funcOf([], out), + locals: [makerSrc], + body: [ + { + kind: "return", + value: { + kind: "closure", + fnName: genName, + captures: [makerSrc.id], + type: funcOf([], out), + loc, + }, + loc, + }, + ], + loc, + }); + } + return { + closure: { kind: "call", callee: maker, args: [value], type: funcOf([], out), loc }, + out, + }; +} + +/** The lowered function stages of a pipeline: the libcall head [n, k, + * positions, flags, mask, streams..., stages..., helpers..., box?, * (signal, abort)?], the destination's type (the last stream, or the * PassThrough a function destination answers), the destination kind * (SCR_PIPE_DEST_*), a promise destination's promise type, and a wrapper * that binds the pipeline's AbortController around the call when a - * stage takes `{ signal }`. */ + * stage takes `{ signal }`. positions has bit j set when the pipeline's + * j-th stage is a function; helpers holds (next, read, return) for each + * generator pumped into a stream. */ interface PipelineStagesLowering { args: IrExpr[]; last: IrType; @@ -2701,56 +2904,89 @@ interface PipelineStagesLowering { wrap: (call: IrExpr) => IrExpr; } -/** pipeline(...) with one run of function stages (pipeline.js). The run - * may start the pipeline (a source function, called with `{ signal }`) - * and may end it (a destination function: an async generator pumped into - * a PassThrough, or an async function whose promise's value reaches the - * callback or the promise). Each stage's `source` parameter is typed as - * what the pipeline passes it: makeAsyncIterable answers the previous - * stream itself (a Readable is async iterable) or the previous stage's +/** pipeline(...) with function stages (pipeline.js). A function may + * start the pipeline (a source function, called with `{ signal }`), may + * end it (an async function whose promise's value reaches the callback or + * the promise), and runs of async generator functions may sit between + * streams; the last generator of a run that a stream follows is pumped + * into that stream. Each stage's `source` parameter is typed as what the + * pipeline passes it: makeAsyncIterable answers the previous stream + * itself (a Readable is async iterable) or the previous stage's * generator. */ function lowerPipelineStages( lowerer: Lowerer, stageArgs: readonly ts.Expression[], loc: SrcLoc, ): PipelineStagesLowering { - const first = stageArgs.findIndex(isPipelineFunctionStage); - let end = first; - while (end < stageArgs.length && isPipelineFunctionStage(stageArgs[end]!)) end++; - if (stageArgs.slice(end).some(isPipelineFunctionStage)) { + if (stageArgs.length > 52) { lowerer.noLowering( - "pipeline with function stages separated by streams", - stageArgs[end]!, - "one run of function stages is supported", + "pipeline with function stages and more than 52 stages", + stageArgs[52]!, + "split the pipeline", ); } - const source = first === 0; - const destFn = end === stageArgs.length; - const before: IrExpr[] = []; - const after: IrExpr[] = []; - try { - for (const a of stageArgs.slice(0, first)) before.push(lowerStreamArg(lowerer, a, "pipeline")); - } catch (e) { - if (e instanceof StreamArgTypeThrow) - lowerer.noLowering("pipeline over this value", stageArgs[0]!); - throw e; - } - let srcT: IrType | null = source ? null : before[before.length - 1]!.type; + const streams: IrExpr[] = []; const stages: IrExpr[] = []; + const helperArgs: IrExpr[] = []; + let positions = 0; let mask = 0; let dest: 0 | 2 = 0; let destPromise: (IrType & { kind: "promise" }) | null = null; - for (let i = first; i < end; i++) { - const fn = stageArgs[i] as ts.FunctionExpression | ts.ArrowFunction; - const isLast = destFn && i === end - 1; + // What the next function stage receives: null before the first stage, + // else the previous stream's type or the previous stage's generator. + let srcT: IrType | null = null; + for (let j = 0; j < stageArgs.length; j++) { + const arg = stageArgs[j]!; + const fn = pipelineStageFunction(lowerer, arg); + const syncSourceFn = + j === 0 && + fn !== null && + !ts.isArrowFunction(fn) && + fn.asteriskToken !== undefined && + !(fn.modifiers?.nodes ?? []).some((m) => m.kind === ts.SyntaxKind.AsyncKeyword); + if (j === 0 && (syncSourceFn || (fn === null && iterableSourceType(lowerer, arg) !== null))) { + const source = lowerIterableSource(lowerer, arg, syncSourceFn ? fn : null, loc); + if (source === null) lowerer.noLowering("pipeline over this source", arg); + positions += 1; + stages.push(source!.closure); + srcT = source!.out; + continue; + } + if (fn === null) { + if (srcT?.kind === "generator") { + // the generator before this stream, pumped into it + if (!fromGenChunk(lowerer, srcT.yieldT)) { + lowerer.noLowering( + `a pipeline stage yielding '${lowerer.fmt(srcT.yieldT)}'`, + stageArgs[j - 1]!, + "stage values written to a stream are strings or Buffers", + ); + } + const helpers = fromGenHelpers(lowerer, srcT, loc); + if (!helpers) + lowerer.noLowering("a pipeline stage with this generator type", stageArgs[j - 1]!); + helperArgs.push(helpers![0]!, helpers![1]!, helpers![2]!); + } + let stream: IrExpr; + try { + stream = lowerStreamArg(lowerer, arg, "pipeline"); + } catch (e) { + if (e instanceof StreamArgTypeThrow) lowerer.noLowering("pipeline over this value", arg); + throw e; + } + streams.push(stream); + srcT = stream.type; + continue; + } + const isLast = j === stageArgs.length - 1; const isAsync = (fn.modifiers?.nodes ?? []).some((m) => m.kind === ts.SyntaxKind.AsyncKeyword); - const isGen = ts.isFunctionExpression(fn) && fn.asteriskToken !== undefined; + const isGen = !ts.isArrowFunction(fn) && fn.asteriskToken !== undefined; if (isLast && isGen) { // pipeline() pumps it into an object-mode PassThrough, whose // buffering decides when the pipeline finishes lowerer.noLowering( "a pipeline destination that is an async generator function", - fn, + arg, "end the pipeline with a stream or an async function (source) { for await (const chunk of source) ... }", ); } @@ -2761,7 +2997,7 @@ function lowerPipelineStages( : srcT === null ? "a pipeline source function that is not an async generator function" : "pipeline stages that are not async generator functions", - fn, + arg, "write the stage as async function* (source) { for await (const chunk of source) ... }", ); } @@ -2783,8 +3019,9 @@ function lowerPipelineStages( "type it as { signal: AbortSignal }", ); } - mask |= 1 << (i - first); + mask |= 1 << stages.length; } + positions += 2 ** j; stages.push(lowered.closure); if (lowered.out.kind === "promise") { dest = 2; @@ -2794,29 +3031,7 @@ function lowerPipelineStages( srcT = lowered.out; } } - try { - for (const a of stageArgs.slice(end)) after.push(lowerStreamArg(lowerer, a, "pipeline")); - } catch (e) { - if (e instanceof StreamArgTypeThrow) - lowerer.noLowering("pipeline over this value", stageArgs[end]!); - throw e; - } - const helperArgs: IrExpr[] = []; - if (dest !== 2) { - // the generator pumped into the next stream or the PassThrough - const lastGen = srcT as IrType & { kind: "generator" }; - if (!fromGenChunk(lowerer, lastGen.yieldT)) { - lowerer.noLowering( - `a pipeline stage yielding '${lowerer.fmt(lastGen.yieldT)}'`, - stageArgs[end - 1]!, - "stage values written to a stream are strings or Buffers", - ); - } - const helpers = fromGenHelpers(lowerer, lastGen, loc); - if (!helpers) - lowerer.noLowering("a pipeline stage with this generator type", stageArgs[end - 1]!); - helperArgs.push(helpers![0]!, helpers![1]!, helpers![2]!); - } else { + if (dest === 2) { const valueT = destPromise!.inner; if ( valueT.kind !== "void" && @@ -2828,28 +3043,25 @@ function lowerPipelineStages( ) { lowerer.noLowering( `a pipeline destination resolving to '${lowerer.fmt(valueT)}'`, - stageArgs[end - 1]!, + stageArgs[stageArgs.length - 1]!, ); } helperArgs.push(pipelineBoxHelper(lowerer, destPromise!, loc)); } - const streams = [...before, ...after]; - // SCR_PIPE_* flags: bit 0 a source function, bits 1-2 the destination, - // bit 3 the helpers, bit 4 the box, bit 5 the signal and abort. - const flags = (source ? 1 : 0) | (dest << 1) | (dest !== 2 ? 8 : 16) | (mask !== 0 ? 32 : 0); + // SCR_PIPE_* flags: bits 1-2 the destination, bit 5 the signal and abort. + const flags = (dest << 1) | (mask !== 0 ? 32 : 0); const head: IrExpr[] = [ numLit(streams.length, loc), - numLit(before.length, loc), numLit(stages.length, loc), + numLit(positions, loc), numLit(flags, loc), numLit(mask, loc), ...streams, ...stages, ...helperArgs, ]; - const last: IrType = destFn - ? { kind: "object", className: "%PassThrough" } - : streams[streams.length - 1]!.type; + const last: IrType = + dest === 2 ? { kind: "object", className: "%PassThrough" } : streams[streams.length - 1]!.type; if (mask === 0) return { args: head, last, dest, destPromise, wrap: (call) => call }; // const ac = new AbortController(): its signal goes to the stages, and // finishImpl's ac.abort() runs through the abort closure. @@ -2980,7 +3192,7 @@ function pipelineAbortHelper(lowerer: Lowerer, ac: IrExpr, loc: SrcLoc): IrExpr * generator, or a promise for a destination async function. */ function lowerStageFunction( lowerer: Lowerer, - fn: ts.FunctionExpression | ts.ArrowFunction, + fn: StageFunction, srcT: IrType | null, ): { closure: IrExpr; out: IrType & ({ kind: "generator" } | { kind: "promise" }) } { const param = srcT === null ? undefined : fn.parameters.nodes[0]; @@ -2995,6 +3207,8 @@ function lowerStageFunction( "name the source parameter", ); } + // The loop variables of the `for await` loops over the source. + const chunkSymbols: ts.Symbol[] = []; if (symbol && fn.body) { const visit = (node: ts.Node): void => { if ( @@ -3017,14 +3231,36 @@ function lowerStageFunction( "iterate the source with for await (const chunk of source)", ); } + const binding = ts.isVariableDeclarationList(parent.initializer) + ? parent.initializer.declarations.nodes[0]?.name + : undefined; + const chunk = + binding && ts.isIdentifier(binding) + ? lowerer.checker.getSymbolAtLocation(binding) + : undefined; + if (chunk) chunkSymbols.push(chunk); } ts.forEachChild(node, visit); }; visit(fn.body); } if (symbol) lowerer.stageSourceTypes.set(symbol, srcT!); + // In JavaScript the checker types the chunks of a source that is the + // previous stage's generator `any`; they read as the strings or Buffers + // the generator yields, so their members take the string and Buffer + // lowerings. + const previousImplicit = lowerer.implicitParamTypes; + const chunkType = + srcT?.kind === "generator" && isJsSourceFile(ts.getSourceFileOfNode(fn)) + ? stageChunkCheckerType(lowerer, srcT.yieldT, fn) + : null; + if (chunkType && chunkSymbols.length > 0) { + const implicit = new Map(previousImplicit ?? []); + for (const chunk of chunkSymbols) implicit.set(chunk, chunkType); + lowerer.implicitParamTypes = implicit; + } try { - const closure = lowerer.lowerExpr(fn); + const closure = lowerer.lowerLambda(fn); const out = closure.type.kind === "func" ? closure.type.ret : null; if (!(out?.kind === "generator" && out.async) && out?.kind !== "promise") { lowerer.noLowering("a pipeline stage that does not return an async generator", fn); @@ -3032,9 +3268,19 @@ function lowerStageFunction( return { closure, out: out as IrType & ({ kind: "generator" } | { kind: "promise" }) }; } finally { if (symbol) lowerer.stageSourceTypes.delete(symbol); + lowerer.implicitParamTypes = previousImplicit; } } +/** The checker type of a stage's string or Buffer chunks, or null for + * other chunk types. */ +function stageChunkCheckerType(lowerer: Lowerer, chunk: IrType, at: ts.Node): ts.Type | null { + if (chunk.kind === "string") return lowerer.checker.getStringType(); + if (chunk.kind !== "bytes") return null; + const buffer = lowerer.checker.resolveName("Buffer", ts.SymbolFlags.Type, at, false); + return buffer ? lowerer.checker.getDeclaredTypeOfSymbol(buffer) : null; +} + /** `stream.compose(...streams)` over stream instances: a Duplex writing * to the first stream and reading from the last (scr_stream_compose). The * classes fix which positions can be readable and writable; a stream in @@ -3806,9 +4052,10 @@ export function lowerStreamMethodCall( "the supported forms are (destination) and (destination, { end })", ); } - // A Hash or Hmac destination pipes into its Transform, and pipe() - // answers the Hash itself (the receiver evaluates first, as in JS). - if (isHashValueType(lowerer.mapTypeOf(lowerer.typeOf(args[0]!)))) { + // A Hash, Hmac, Cipheriv or Decipheriv destination pipes into its + // Transform, and pipe() answers the value itself (the receiver + // evaluates first, as in JS). + if (isValueStream(lowerer, args[0]!)) { const receiverValue = lowerer.lowerExpr(access.expression); const source = lowerer.declareHiddenLocal("%pipeSource", receiverValue.type); const hashValue = lowerer.lowerExpr(args[0]!); @@ -3826,7 +4073,10 @@ export function lowerStreamMethodCall( fn: "readable.pipe", args: [ { kind: "varRef", localId: source.id, type: receiverValue.type, loc }, - hashStreamOf({ kind: "varRef", localId: hash.id, type: hashValue.type, loc }, true), + valueStreamOf( + { kind: "varRef", localId: hash.id, type: hashValue.type, loc }, + true, + ), end, ], type: { kind: "object", className: "%Transform" }, diff --git a/packages/compiler/src/frontend/lowering/lowerer.ts b/packages/compiler/src/frontend/lowering/lowerer.ts index 3f8a0a66e..f55498599 100644 --- a/packages/compiler/src/frontend/lowering/lowerer.ts +++ b/packages/compiler/src/frontend/lowering/lowerer.ts @@ -62,7 +62,7 @@ import { resolveRequireRuntime } from "../runtime-resolve.js"; import { lowerKeyObjectCall } from "./builtins/key-object.js"; import { isEcdhType, lowerEcdhCall } from "./builtins/ecdh.js"; import { fenceWebCryptoMember, lowerWebCryptoCall } from "./builtins/webcrypto.js"; -import { hashStreamOf } from "./builtins/hash-stream.js"; +import { valueStreamOf } from "./builtins/hash-stream.js"; import { lowerNetAddressCall, lowerNetAddressNew } from "./builtins/net-address.js"; import { dynUndefinedExpr, nodeThrowExpr, numLit, varRef } from "../../ir/build.js"; import { iteratorSlotAdapter } from "./iterator-adapters.js"; @@ -3030,10 +3030,10 @@ export class Lowerer { /** Inline Promise.then parameters whose contextual type is a builtin * module namespace anonymous object. The settled native token is the * ABI truth even when the checker type has no stable module symbol. */ - /** Receivers of a stream call on a Hash or Hmac (lowerHashStreamCall): - * for the duration of that call, the receiver lowers to the Transform of - * the value held in the named local. */ - readonly hashStreamReceivers = new Map(); + /** Receivers of a stream call on a Hash, Hmac, Cipheriv or Decipheriv + * (lowerHashStreamCall): for the duration of that call, the receiver + * lowers to the Transform of the value held in the named local. */ + readonly valueStreamReceivers = new Map(); /** fs call arguments evaluated ahead of a URL path's conversion * (builtins/fs-paths.ts): each lowers to the hidden local that holds its * value, which is null until the path's lowering has saved it. */ @@ -12486,8 +12486,9 @@ export class Lowerer { /* ── expressions ──────────────────────────────────────────────────── */ lowerExpr(expr: ts.Expression): IrExpr { - const hash = this.hashStreamReceivers.get(expr); - if (hash) return hashStreamOf(varRef(hash.localId, hash.type, locOf(expr))); + const streamValue = this.valueStreamReceivers.get(expr); + if (streamValue) + return valueStreamOf(varRef(streamValue.localId, streamValue.type, locOf(expr))); return this.stagedArgument(expr) ?? lowerExpr(this, expr); } diff --git a/packages/compiler/src/ir/builtin-effects.ts b/packages/compiler/src/ir/builtin-effects.ts index ca8c52118..89aa5b001 100644 --- a/packages/compiler/src/ir/builtin-effects.ts +++ b/packages/compiler/src/ir/builtin-effects.ts @@ -80,6 +80,7 @@ export const MAY_THROW_LIB_FNS: ReadonlySet = new Set([ "procStream.write", "child.stdioStream", "stream.unboxDyn", + "stream.handleStream", "stream.pipeDyn", "stream.unpipeDyn", "weakMap.new", diff --git a/packages/compiler/src/ir/builtin-signatures.ts b/packages/compiler/src/ir/builtin-signatures.ts index fa7876579..9490ed78c 100644 --- a/packages/compiler/src/ir/builtin-signatures.ts +++ b/packages/compiler/src/ir/builtin-signatures.ts @@ -2712,6 +2712,12 @@ export const LIB_FN_SIGS = defineLibFnSignatures({ argTypes: [DYN, STRING], result: { kind: "object", className: "%Duplex" }, }, + /** The Transform a Cipheriv or Decipheriv reads and writes as a stream + * (+1, created on first use and owned by the cipher). */ + "stream.handleStream": { + argTypes: [DYN], + result: { kind: "object", className: "%Transform" }, + }, /** Checked-dynamic stream slots (dyn-streams.ts): the native stream a * checked-dynamic value boxes (+1) — a runtime stream with the readable * (1) or writable (2) side — or null for any other value. Never throws; @@ -3936,14 +3942,15 @@ export const LIB_FN_SIGS = defineLibFnSignatures({ "stream.finishedOpts": { argTypes: [null, F64, F64, BOOL, null], result: VOID }, "stream.finishedOptsSignal": { argTypes: [null, F64, F64, BOOL, DYN, null], result: VOID }, "sp.pipeline": { argTypes: [F64], result: { kind: "promise", inner: VOID } }, - /** pipeline with a run of function stages: [n, at, k, flags, mask, - * s1..sn, stage1..stagek, (next, read, return)?, box?, (signal, - * abort)?] (+ the callback for the stream.* forms): the user's stage - * functions, the frontend's helpers over the pumped generator or a - * promise destination's box, and the pipeline's AbortSignal with its - * abort closure; flags names which groups are present - * (scr_stream_pipeline_stages). A promise destination's promise type is - * sp.pipelineStages' result. */ + /** pipeline with function stages: [n, k, positions, flags, mask, + * s1..sn, stage1..stagek, helpers..., box?, (signal, abort)?] (+ the + * callback for the stream.* forms): the user's stage functions at the + * positions the bitmask names, the frontend's (next, read, return) + * helpers for each pumped generator (pipelinePumpedRuns) and a promise + * destination's box, and the pipeline's AbortSignal with its abort + * closure; flags bits 1-2 give the destination kind and bit 5 the + * signal group (scr_stream_pipeline_stages). A promise destination's + * promise type is sp.pipelineStages' result. */ "sp.pipelineStages": { argTypes: [F64, F64, F64, F64, F64], result: { kind: "promise", inner: VOID }, @@ -4734,3 +4741,15 @@ export const LIB_FN_SIGS = defineLibFnSignatures({ /** A builtin name exists exactly when its runtime signature exists. */ export type IrLibFn = keyof typeof LIB_FN_SIGS; + +/** The function-stage runs of a pipeline of `total` stages that a stream + * follows (positions bit j: stage j is a function): each pumps its last + * generator into that stream through one (next, read, return) triple. */ +export function pipelinePumpedRuns(total: number, positions: number): number { + let runs = 0; + for (let j = 0; j + 1 < total; j++) { + if (Math.floor(positions / 2 ** j) % 2 === 1 && Math.floor(positions / 2 ** (j + 1)) % 2 === 0) + runs++; + } + return runs; +} diff --git a/packages/compiler/src/ir/runtime-features.ts b/packages/compiler/src/ir/runtime-features.ts index 4142b6efe..7d8626e4b 100644 --- a/packages/compiler/src/ir/runtime-features.ts +++ b/packages/compiler/src/ir/runtime-features.ts @@ -237,6 +237,7 @@ const STREAM_LIB_FNS: ReadonlySet = new Set([ "stream.prop", "stream.stdioWritable", "stream.unboxDyn", + "stream.handleStream", "stream.unpipeDyn", "stream.workerStdio", ]); diff --git a/packages/compiler/src/ir/validate.ts b/packages/compiler/src/ir/validate.ts index 88ccbbc33..9de1b5b62 100644 --- a/packages/compiler/src/ir/validate.ts +++ b/packages/compiler/src/ir/validate.ts @@ -60,7 +60,7 @@ import { DYN_CLASS_PROPERTIES, callSiteReturnType, classValueConstructor } from import { unionWideningTags } from "./analysis.js"; import { alwaysReturns } from "./control-flow.js"; import { dynamicOperationOf, functionTier } from "./tiers.js"; -import { LIB_FN_SIGS } from "./builtin-signatures.js"; +import { LIB_FN_SIGS, pipelinePumpedRuns } from "./builtin-signatures.js"; /** The promisified crypto calls' value kinds besides checked-dynamic. */ const CRYPTO_PROMISE_VALUES: Readonly> = { @@ -5614,23 +5614,34 @@ function validateFunction( e.fn === "stream.pipelineStagesDyn" || e.fn === "sp.pipelineStages" ) { - // (n, at, k, flags, mask, s1..sn, stage1..stagek, (next, read, - // return)?, box?, (signal, abort)?[, cb]): flags bit 0 a source - // function, bits 1-2 the destination, bits 3/4/5 the groups. - const [n, at, k, flags] = e.args + // (n, k, positions, flags, mask, s1..sn, stage1..stagek, + // helpers..., box?, (signal, abort)?[, cb]): positions bit j marks + // stage j a function, flags bits 1-2 the destination, bit 5 the + // signal group; helpers are three per function run a stream + // follows. + const [n, k, positions, flags] = e.args .slice(0, 4) .map((a) => (a?.kind === "numLit" ? a.value : -1)); - const source = (flags! & 1) !== 0; + const total = n! + k!; const destKind = (flags! >> 1) & 3; - const groups = (flags! & 8 ? 3 : 0) + (flags! & 16 ? 1 : 0) + (flags! & 32 ? 2 : 0); + let functions = 0; + for (let j = 0; j < total && j < 53; j++) { + if (Math.floor(positions! / 2 ** j) % 2 === 1) functions++; + } + const lastIsFunction = Math.floor(positions! / 2 ** (total - 1)) % 2 === 1; + const helperCount = 3 * pipelinePumpedRuns(total, positions!); + const groups = helperCount + (destKind === 2 ? 1 : 0) + (flags! & 32 ? 2 : 0); const cbArgs = e.fn === "sp.pipelineStages" ? 0 : 1; if ( flags! < 0 || n! < 0 || k! < 1 || - (source ? at !== 0 : at! < 1) || - (destKind === 0 ? at! + 0 >= n! : at! !== n!) || - n! + k! < 2 || + positions! < 1 || + total > 52 || + positions! >= 2 ** total || + functions !== k || + (destKind === 2) !== lastIsFunction || + total < 2 || e.args.length !== 5 + n! + k! + groups + cbArgs ) { err(`libCall ${e.fn} count/arity mismatch`, e.loc); @@ -5647,7 +5658,7 @@ function validateFunction( err(`libCall ${e.fn} stage function ${i} must be a func`, e.loc); } at2 += k!; - const helpers = (flags! & 8 ? 3 : 0) + (flags! & 16 ? 1 : 0); + const helpers = helperCount + (destKind === 2 ? 1 : 0); for (let i = at2; i < at2 + helpers; i++) { if (e.args[i]?.type.kind !== "func") err(`libCall ${e.fn} helper ${i} must be a func`, e.loc); diff --git a/packages/compiler/test/native-codegen-integration.test.ts b/packages/compiler/test/native-codegen-integration.test.ts index d8f158ebc..c05e7f994 100644 --- a/packages/compiler/test/native-codegen-integration.test.ts +++ b/packages/compiler/test/native-codegen-integration.test.ts @@ -378,7 +378,7 @@ done: expect(await symbols(helperObject, ["-u"])).toEqual(await symbols(clangObject, ["-u"])); expect(await symbols(helperObject, ["-gU"])).toEqual(await symbols(clangObject, ["-gU"])); expect(await symbols(helperObject, ["-gU"])).toEqual(["0000000000000000 T _main"]); - expect(await symbols(helperObject, ["-u"])).toContain("_scr_runtime_abi_v24"); + expect(await symbols(helperObject, ["-u"])).toContain("_scr_runtime_abi_v25"); const clangExe = join(dir, "clang-program"); const linkDriver = join(dir, "clang-driver.c"); @@ -712,7 +712,7 @@ done: .split("\n"); expect(undefinedSymbols).toContain("_sf_scale"); expect(undefinedSymbols).toContain("_sf_callback_mix"); - expect(undefinedSymbols).toContain("_scr_runtime_abi_v24"); + expect(undefinedSymbols).toContain("_scr_runtime_abi_v25"); }); test("helper and clang object paths retain the same outbound FFI C ABI", async () => { diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7055-cipher-streams.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7055-cipher-streams.ts.json new file mode 100644 index 000000000..989c7f452 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7055-cipher-streams.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7055-cipher-streams.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7056-cipher-stream-errors.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7056-cipher-stream-errors.ts.json new file mode 100644 index 000000000..16bd2586c --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7056-cipher-stream-errors.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7056-cipher-stream-errors.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7057-cipher-streams.mjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7057-cipher-streams.mjs.json new file mode 100644 index 000000000..1304933ae --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7057-cipher-streams.mjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7057-cipher-streams.mjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7058-pipeline-function-stages.mjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7058-pipeline-function-stages.mjs.json new file mode 100644 index 000000000..3b9fd5a0f --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7058-pipeline-function-stages.mjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7058-pipeline-function-stages.mjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7059-transform-backpressure.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7059-transform-backpressure.ts.json new file mode 100644 index 000000000..311882a39 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7059-transform-backpressure.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7059-transform-backpressure.ts" + ], + "diags": [] +} diff --git a/packages/runtime-darwin-arm64/scripts/build.mjs b/packages/runtime-darwin-arm64/scripts/build.mjs index 85c5ffb00..c18a1d080 100644 --- a/packages/runtime-darwin-arm64/scripts/build.mjs +++ b/packages/runtime-darwin-arm64/scripts/build.mjs @@ -337,7 +337,7 @@ async function build() { package: packageManifest.name, version: packageManifest.version, target: RUNTIME_PACK_MATRIX.target, - runtime_abi: { version: 24, marker: "scr_runtime_abi_v24" }, + runtime_abi: { version: 25, marker: "scr_runtime_abi_v25" }, compiler: { command: compiler, identity: compilerVersion, diff --git a/packages/runtime-pack-common/scripts/build.mjs b/packages/runtime-pack-common/scripts/build.mjs index b242b78aa..f0eab2cb5 100644 --- a/packages/runtime-pack-common/scripts/build.mjs +++ b/packages/runtime-pack-common/scripts/build.mjs @@ -434,7 +434,7 @@ async function build() { package: packageManifest.name, version: packageManifest.version, target: matrix.target, - runtime_abi: { version: 24, marker: "scr_runtime_abi_v24" }, + runtime_abi: { version: 25, marker: "scr_runtime_abi_v25" }, compiler: { command: compiler, identity: compilerVersion, diff --git a/packages/runtime/src/scr_cipher.c b/packages/runtime/src/scr_cipher.c index df62a79a3..c8a10a731 100644 --- a/packages/runtime/src/scr_cipher.c +++ b/packages/runtime/src/scr_cipher.c @@ -9,7 +9,9 @@ * the runtime StringDecoder, which carries partial characters and base64 * groups between calls as Node's does. chacha20-poly1305 follows OpenSSL's * construction, which also accepts AAD after data. Authentication tags are - * compared in constant time. All arguments are borrowed. */ + * compared in constant time. A cipher used as a stream reads and writes + * through its own native Transform, which the stream unit builds on first + * use, as Node's LazyTransform is. All arguments are borrowed. */ #include "scr_runtime.h" #include @@ -103,6 +105,12 @@ typedef struct ScCipher { bool aad_open; /* AAD was added since the last data or padding */ ScrStr *decoder; /* the StringDecoder encoding once string output starts */ double pending; /* the decoder's carried bytes */ + /* The Transform of a cipher used as a stream (+1) and its release + * (scr_stream.c). Its steps hold the cipher, so the cipher is a + * collector-headered object behind a traced handle, and the pair is + * reclaimed as a cycle. */ + void *stream; + void (*stream_release)(void *); } ScCipher; static void *sc_retain(void *h) { @@ -110,16 +118,38 @@ static void *sc_retain(void *h) { return h; } -static void sc_release(void *h) { +static void sc_trace(void *h, ScrTraceVisit visit, void *ctx) { + ScCipher *c = h; + if (c->stream) visit(c->stream, ctx); +} + +/* The collector frees the stream itself; the rest is released here. */ +static void sc_gcfree(void *h) { ScCipher *c = h; - if (--c->rc) return; mbedtls_aes_free(&c->aes); mbedtls_gcm_free(&c->gcm); mbedtls_chacha20_free(&c->chacha); mbedtls_poly1305_free(&c->poly); scr_str_release(c->decoder); mbedtls_platform_zeroize(c, sizeof *c); - free(c); + scr_obj_free_note(); + scr_cyc_free(c); +} + +static void sc_destroy(void *h) { + ScCipher *c = h; + void *stream = c->stream; + c->stream = NULL; + if (stream) c->stream_release(stream); + sc_gcfree(c); +} + +static void sc_release(void *h) { + ScCipher *c = h; + if (--c->rc == 0) { + scr_cyc_dead(c); + scr_rc_destroy(c, sc_destroy); + } else scr_cyc_released(c); } /* ── errors ───────────────────────────────────────────────────────────── */ @@ -532,9 +562,10 @@ static ScrDyn *sc_cipher_new(bool decrypt, ScrDyn *cipher, ScrDyn *key, ScrDyn * sc_throw(SCR_ERR_RANGE, "ERR_CRYPTO_INVALID_KEYLEN", "Invalid key length"); goto done; } - ScCipher *c = calloc(1, sizeof *c); - if (!c) scr_trap("scriptc: out of memory\n"); + static ScrCycType cyc_type = {&sc_trace, &sc_gcfree, SCR_CYC_KIND_NONE, 0}; + ScCipher *c = scr_cyc_alloc(sizeof *c, &cyc_type); c->rc = 1; + scr_obj_alloc_note(); c->mode = spec->mode; c->decrypt = decrypt; c->active = true; @@ -883,7 +914,8 @@ static bool sc_aead(const ScCipher *c) { return c->mode == SC_GCM || c->mode == SC_CHACHA; } -static ScrDyn *sc_final(ScCipher *c, ScrDyn *output_encoding) { +/* final()'s output bytes (+1 Buffer), or NULL with the error thrown. */ +static ScrBytes *sc_final_bytes(ScCipher *c) { if (!c->active) { sc_throw(SCR_ERR_ERROR, "ERR_CRYPTO_INVALID_STATE", "Invalid state"); return NULL; @@ -953,7 +985,12 @@ static ScrDyn *sc_final(ScCipher *c, ScrDyn *output_encoding) { sc_throw(SCR_ERR_ERROR, NULL, "Unsupported state or unable to authenticate data"); return NULL; } - return sc_output(c, out, output_encoding, true); + return out; +} + +static ScrDyn *sc_final(ScCipher *c, ScrDyn *output_encoding) { + ScrBytes *out = sc_final_bytes(c); + return out ? sc_output(c, out, output_encoding, true) : NULL; } static ScrDyn *sc_set_aad(ScCipher *c, ScrDyn *self, ScrDyn *buffer, ScrDyn *options) { @@ -1055,6 +1092,49 @@ static ScrDyn *sc_set_auth_tag(ScCipher *c, ScrDyn *self, ScrDyn *buffer, ScrDyn return scr_dyn_retain(self); } +/* ── Cipheriv and Decipheriv as streams ─────────────────────────────── + * The stream unit builds a cipher's Transform from these operations + * (ScrDynHandleTransform): each chunk goes through the handle's update(), + * and the end pushes final() or reports its error, as Node's cipher.js + * _transform and _flush do. */ + +static void *sc_stream_get(void *h) { + return ((ScCipher *)h)->stream; +} + +static void sc_stream_set(void *h, void *stream, void (*release)(void *)) { + ScCipher *c = h; + c->stream = stream; + c->stream_release = release; +} + +/* The handle's update(): it throws out of the write, as in Node, once + * the cipher has finished. */ +static ScrBytes *sc_stream_update(void *h, ScrBytes *chunk) { + ScCipher *c = h; + if (!c->active) { + sc_throw(SCR_ERR_ERROR, NULL, "Trying to add data in unsupported state"); + return NULL; + } + SCR_BYTES_SNAPSHOT(chunk); + return sc_transform(c, chunk->data, chunk->len * scr_bytes_elem_size(chunk->elem)); +} + +static ScrBytes *sc_stream_final(void *h) { + return sc_final_bytes(h); +} + +static const ScrDynHandleTransform sc_transform_ops = { + sc_stream_get, sc_stream_set, sc_stream_update, sc_stream_final, sc_trace, +}; + +static ScrDynHandleOps sc_ops_cipher; +static ScrDynHandleOps sc_ops_decipher; + +static const ScrDynHandleOps *sc_ops_of(const ScCipher *c) { + return c->decrypt ? &sc_ops_decipher : &sc_ops_cipher; +} + static bool sc_name_is(const char *method, const char *name) { return strcmp(method, name) == 0; } @@ -1076,8 +1156,12 @@ static bool sc_stream_member(const char *name) { static void sc_stream_fence(void) { static const char message[] = - "Cipheriv and Decipheriv stream members have no lowering; call update() and final()"; - scr_throw_error_msg_code(SCR_ERR_ERROR, message, sizeof message - 1, "SC2020"); + "This Cipheriv and Decipheriv stream member has no lowering when called through an untyped value"; + static const char streamless[] = + "Cipheriv and Decipheriv stream members called through an untyped value have no lowering in a " + "program that uses no streams"; + if (scr_handle_stream_hooks) scr_throw_error_msg_code(SCR_ERR_ERROR, message, sizeof message - 1, "SC2020"); + else scr_throw_error_msg_code(SCR_ERR_ERROR, streamless, sizeof streamless - 1, "SC2020"); } static ScrDyn *sc_dispatch(ScCipher *c, ScrDyn *self, const char *method, ScrDyn *const *args, size_t argc, @@ -1091,6 +1175,12 @@ static ScrDyn *sc_dispatch(ScCipher *c, ScrDyn *self, const char *method, ScrDyn if (c->decrypt && sc_name_is(method, "setAuthTag")) return sc_set_auth_tag(c, self, sc_arg(args, argc, 0), sc_arg(args, argc, 1)); if (sc_stream_member(method)) { + /* The members a native stream answers to checked-dynamic calls run + * on the cipher's Transform; the others have no lowering. */ + ScrDyn *result = NULL; + if (scr_handle_stream_hooks && + scr_handle_stream_hooks->call(sc_ops_of(c), c, self, method, args, argc, &result)) + return result; sc_stream_fence(); return NULL; } @@ -1117,11 +1207,19 @@ done: } static ScrDyn *sc_get(void *h, const char *key, size_t key_len) { - (void)h; (void)key; (void)key_len; + /* The stream properties read the cipher's Transform. */ + if (scr_handle_stream_hooks) { + ScrDyn *value = scr_handle_stream_hooks->get(sc_ops_of(h), h, key, key_len); + if (value || scr_exc_pending()) return value; + } static const char message[] = - "Cipheriv and Decipheriv property reads have no lowering; call update(), final(), setAAD(), " - "setAutoPadding(), getAuthTag() or setAuthTag()"; - scr_throw_error_msg_code(SCR_ERR_ERROR, message, sizeof message - 1, "SC2020"); + "Cipheriv and Decipheriv property reads have no lowering other than the stream properties; call " + "update(), final(), setAAD(), setAutoPadding(), getAuthTag() or setAuthTag()"; + static const char streamless[] = + "Cipheriv and Decipheriv property reads through an untyped value have no lowering in a program " + "that uses no streams"; + if (scr_handle_stream_hooks) scr_throw_error_msg_code(SCR_ERR_ERROR, message, sizeof message - 1, "SC2020"); + else scr_throw_error_msg_code(SCR_ERR_ERROR, streamless, sizeof streamless - 1, "SC2020"); return NULL; } @@ -1132,5 +1230,7 @@ static bool sc_set(void *h, const char *key, size_t key_len, const ScrDyn *value return true; } -static ScrDynHandleOps sc_ops_cipher = {"Cipheriv", sc_retain, sc_release, sc_invoke, sc_get, sc_set, NULL, NULL, NULL}; -static ScrDynHandleOps sc_ops_decipher = {"Decipheriv", sc_retain, sc_release, sc_invoke, sc_get, sc_set, NULL, NULL, NULL}; +static ScrDynHandleOps sc_ops_cipher = {"Cipheriv", sc_retain, sc_release, sc_invoke, sc_get, sc_set, NULL, NULL, NULL, + .transform = &sc_transform_ops}; +static ScrDynHandleOps sc_ops_decipher = {"Decipheriv", sc_retain, sc_release, sc_invoke, sc_get, sc_set, NULL, NULL, NULL, + .transform = &sc_transform_ops}; diff --git a/packages/runtime/src/scr_console.c b/packages/runtime/src/scr_console.c index c29126a4a..c8fa40b38 100644 --- a/packages/runtime/src/scr_console.c +++ b/packages/runtime/src/scr_console.c @@ -148,7 +148,7 @@ static void scr_enable_utf8_console_output(void) { } #endif -void scr_runtime_abi_v24(void) {} +void scr_runtime_abi_v25(void) {} void scr_init(void) { #ifdef SCR_WORKERS diff --git a/packages/runtime/src/scr_core.h b/packages/runtime/src/scr_core.h index 94936a8aa..5e5278588 100644 --- a/packages/runtime/src/scr_core.h +++ b/packages/runtime/src/scr_core.h @@ -81,7 +81,7 @@ void scr_init(void); /* Program objects emitted by the bundled LLVM helper reference this symbol. * Its versioned spelling makes a mismatched manual runtime link fail before * the program can start. */ -void scr_runtime_abi_v24(void); +void scr_runtime_abi_v25(void); void scr_runtime_workers_v8(void); /* ── the trap funnel (scr_console.c; scr_library.c under -DSCR_LIB) ────── diff --git a/packages/runtime/src/scr_dyn.h b/packages/runtime/src/scr_dyn.h index 02c4d828f..cafc8d07e 100644 --- a/packages/runtime/src/scr_dyn.h +++ b/packages/runtime/src/scr_dyn.h @@ -1388,8 +1388,37 @@ typedef struct ScrDynHandleOps { /* Cross-thread structured clone (postMessage, workerData); NULL = * DataCloneError. See ScrDynHandleTransfer. */ const struct ScrDynHandleTransfer *transfer; + /* A handle that is also a Node Transform (Cipheriv, Decipheriv): the + * stream unit builds the handle's Transform from these operations on its + * first stream use. NULL = the class is not a stream. */ + const struct ScrDynHandleTransform *transform; } ScrDynHandleOps; +/* The operations a Transform handle gives the stream unit (scr_stream.c): + * the slot where the handle keeps its stream (+1) with the stream's + * release, update() and final() over bytes (+1 Buffer, or NULL with the + * exception pending), and the handle's collector trace, since the + * stream's steps hold the handle. */ +typedef struct ScrDynHandleTransform { + void *(*stream)(void *h); + void (*set_stream)(void *h, void *stream, void (*release)(void *)); + struct ScrBytes *(*update)(void *h, struct ScrBytes *chunk); + struct ScrBytes *(*final)(void *h); + ScrTraceFn trace; +} ScrDynHandleTransform; + +/* Checked-dynamic member calls and property reads on a Transform handle, + * installed by the stream unit (scr_stream_install); NULL in a program + * without it. call answers true with *result set (+1, or NULL with the + * exception pending) for the members a native stream answers, and get + * answers a stream property (+1), or NULL for other keys. */ +typedef struct ScrHandleStreamHooks { + bool (*call)(const ScrDynHandleOps *ops, void *h, ScrDyn *self, const char *method, + ScrDyn *const *args, size_t argc, ScrDyn **result); + ScrDyn *(*get)(const ScrDynHandleOps *ops, void *h, const char *key, size_t key_len); +} ScrHandleStreamHooks; +extern SCR_TL const ScrHandleStreamHooks *scr_handle_stream_hooks; + /* How a handle crosses threads, supplied by the handle's owning unit so the * message encoder (scr_message.c) needs no reference to gated units and no * per-tag registry. share runs on the sending thread: it answers a payload diff --git a/packages/runtime/src/scr_json.c b/packages/runtime/src/scr_json.c index b8f16630b..194ade887 100644 --- a/packages/runtime/src/scr_json.c +++ b/packages/runtime/src/scr_json.c @@ -37,6 +37,8 @@ static SCR_TL ScrDyn *scr_builtin_object_prototype; +SCR_TL const ScrHandleStreamHooks *scr_handle_stream_hooks; + /* Constructor capsules expose identity and name/length, but do not model * per-class static property tables yet. Never report a fabricated empty view. */ static bool scr_dyn_class_reflection_fence(const ScrDyn *value) { @@ -4309,6 +4311,7 @@ ScrDyn *scr_dyn_new_handle(void *h, ScrDynHandleTag tag) { d->v.handle.tag = tag; d->v.handle.traced = tag == SCR_DYNH_WEAK_MAP || tag == SCR_DYNH_WEAK_SET || tag == SCR_DYNH_WORKER || tag == SCR_DYNH_MESSAGE_PORT || tag == SCR_DYNH_DISPOSABLE_STACK || + tag == SCR_DYNH_CIPHER || tag == SCR_DYNH_DECIPHER || ((tag == SCR_DYNH_SET || tag == SCR_DYNH_MAP) && ((ScrMap *)h)->traced != 0); return d; diff --git a/packages/runtime/src/scr_stream.c b/packages/runtime/src/scr_stream.c index f7330b615..271937f2e 100644 --- a/packages/runtime/src/scr_stream.c +++ b/packages/runtime/src/scr_stream.c @@ -175,6 +175,11 @@ struct ScrStreamState { ScrStreamPlainInv flush_inv; bool flush_called; bool flush_pending; /* _flush called, its callback not yet run */ + /* Transform._write's backpressure: the readable length when the + * transform started, and whether its completed write waits for the + * readable side to read (Node's kCallback). */ + size_t xf_length; + bool xf_held; /* A _final that completes at once (zlib's ZlibBase): 'prefinish' starts * the flush and 'finish' does not wait for it. */ bool flush_on_prefinish; @@ -412,6 +417,7 @@ typedef enum { SCR_ST_ERROR_BUFFER, /* errorBuffer: buffered write and end() callbacks */ SCR_ST_CONSTRUCT, /* constructNT: run the construct hook */ SCR_ST_CONSTRUCTED, /* onConstruct: resume reads/writes/destroy */ + SCR_ST_XF_DONE, /* a transform's write callback after push(null) */ } ScrStreamTickOp; typedef struct ScrStreamTick { @@ -688,8 +694,13 @@ static void scr_stream_call_read(ScrStream *s) { } if (st->r.read_cb == NULL) { /* transform reads are push-driven; the attempt still clears the - * initial sync flag (Node's Transform._read ran) */ + * initial sync flag (Node's Transform._read ran) and runs a held + * write callback */ st->r.in_read_sync = false; + if (st->xf_held) { + st->xf_held = false; + scr_stream_write_done(s, NULL); + } return; } st->r.reading = true; @@ -1706,17 +1717,20 @@ static void scr_stream_wq_push(ScrStreamState *st, ScrBytes *chunk /*moves*/, Sc st->w.n++; } +static void scr_stream_transform_callback(ScrStream *s); + static void scr_stream_do_write(ScrStream *s, ScrBytes *chunk /*moves*/, ScrClosure *cb /*moves*/) { ScrStreamState *st = s->st; st->w.writing = true; st->w.wsync = true; st->w.inflight_len = chunk->len; st->w.inflight_cb = cb; + if (st->is_transform || st->passthrough) st->xf_length = st->r.length; if (st->passthrough && st->transform_cb == NULL) { /* identity: push through, complete synchronously */ scr_stream_add_chunk(s, chunk, false); scr_bytes_release(chunk); - if (!scr_exc_pending()) scr_stream_write_done(s, NULL); + if (!scr_exc_pending()) scr_stream_transform_callback(s); st->w.wsync = false; return; } @@ -3294,13 +3308,13 @@ ScrDyn *scr_stream_from_done(void) { } /* ── pipeline function stages (pipeline.js: makeAsyncIterable, pumpToNode) ── - * A run of async generator functions between two node streams. Each - * stage is called while the pipeline is set up, in pipeline order: - * makeAsyncIterable hands the first one the previous stream itself (a - * Readable is async iterable) and each later one the previous stage's - * generator. The last generator is pumped into the stream after the run - * with pumpToNode. The stages and helpers use the closure ABI (the - * closure first; the callee owns its parameters). */ + * Runs of async generator functions between node streams. Each stage is + * called while the pipeline is set up, in pipeline order: + * makeAsyncIterable hands a stage that follows a stream the stream itself + * (a Readable is async iterable) and any other stage the previous stage's + * generator. The last generator of a run that a stream follows is pumped + * into that stream with pumpToNode. The stages and helpers use the closure + * ABI (the closure first; the callee owns its parameters). */ typedef ScrPromise *(*ScrPipeNextFn)(ScrClosure *, ScrGen *); typedef ScrDyn *(*ScrPipeReadFn)(ScrClosure *, void *); @@ -3309,20 +3323,25 @@ typedef ScrDyn *(*ScrPipeBoxFn)(ScrClosure *, ScrPromise *); enum { SCR_PIPE_DEST_STREAM, SCR_PIPE_DEST_PROMISE = 2 }; typedef struct { - size_t at; /* streams[0..at-1] come before the stages */ size_t count; /* the function stages */ + uint64_t positions; /* bit j: the pipeline's j-th stage is a function */ /* stage(source[, { signal }]) answers an async generator, or a promise * for a promise destination; the source is absent for a source - * function, else the previous stream or async generator */ + * function (position 0), else the previous stream or async generator */ ScrClosure **stages; - bool source; /* stages[0] is the source (at == 0) */ int dest; /* the last stage: SCR_PIPE_DEST_* */ unsigned opts_mask; /* bit i: stages[i] takes the { signal } argument */ ScrDyn *signal; /* the pipeline's AbortSignal when opts_mask != 0 */ - ScrClosure *next, *read, *ret; /* the pumped generator's helpers, or NULL */ + /* (next, read, return) for each generator pumped into a stream, in + * pipeline order, or NULL */ + ScrClosure **helpers; ScrClosure *box; /* SCR_PIPE_DEST_PROMISE: boxes its promise as a dyn promise */ } ScrPipeStages; +static bool scr_pipe_is_stage(const ScrPipeStages *fs, size_t j) { + return fs && j < 64 && ((fs->positions >> j) & 1u); +} + /* stages[i](source[, { signal }]): the callee owns its parameters. */ static void *scr_pipe_call_stage(const ScrPipeStages *fs, size_t i, void *src /*moves*/) { ScrClosure *st = fs->stages[i]; @@ -3331,7 +3350,7 @@ static void *scr_pipe_call_stage(const ScrPipeStages *fs, size_t i, void *src /* opts = scr_dyn_new_obj(); scr_dyn_obj_set(opts, "signal", 6, scr_dyn_retain(fs->signal)); } - if (i == 0 && fs->source) { + if (i == 0 && (fs->positions & 1u)) { return opts ? ((void *(*)(ScrClosure *, ScrDyn *))st->fn)(st, opts) : ((void *(*)(ScrClosure *))st->fn)(st); } @@ -3580,17 +3599,17 @@ static void scr_pump_after_wait(ScrPump *pm, int stage, bool rejected) { } /* pumpToNode(gen, w, finish, { end }): starts at once, as the async - * function does. The helpers are borrowed. */ -static void scr_pump_start(ScrStreamPipeCtx *ctx, ScrGen *gen /*moves*/, const ScrPipeStages *fs, + * function does. The helpers (next, read, return) are borrowed. */ +static void scr_pump_start(ScrStreamPipeCtx *ctx, ScrGen *gen /*moves*/, ScrClosure *const *helpers, ScrStream *w, bool end) { ScrPump *pm = calloc(1, sizeof *pm); if (!pm) scr_stream_oom(); pm->rc = 1; pm->ctx = scr_stream_pipectx_retain_v(ctx); pm->gen = gen; - pm->next = scr_closure_retain(fs->next); - pm->read = scr_closure_retain(fs->read); - pm->ret = scr_closure_retain(fs->ret); + pm->next = scr_closure_retain(helpers[0]); + pm->read = scr_closure_retain(helpers[1]); + pm->ret = scr_closure_retain(helpers[2]); pm->w = scr_stream_retain(w); pm->end = end; pm->drain_l = scr_pump_closure((void *)&scr_pump_drain_l, pm); @@ -3721,27 +3740,27 @@ static ScrStream *scr_stream_pipeline_ex(size_t n, ScrStream **streams, ScrClosu ctx->dispose = add(signal, listener); scr_dyn_release(listener); } - /* pipelineImpl's loop over the stages: the n streams, with fs->count - * function stages before streams[fs->at] */ + /* pipelineImpl's loop over the stages: the n streams and fs->count + * function stages, at the positions fs->positions names */ size_t stages = n + (fs ? fs->count : 0); - size_t fn_first = fs ? fs->at : stages, fn_end = fs ? fs->at + fs->count : stages; ScrStream *ret = NULL; ScrGen *ret_gen = NULL; /* +1: ret when the previous stage was a function */ size_t i = 0; /* the next stream */ + size_t k = 0; /* the next function stage */ + size_t h = 0; /* the next pumped generator's helpers */ for (size_t j = 0; j < stages; j++) { bool reading = j + 1 < stages; bool end = reading || end_opt; bool last = j + 1 == stages; - if (j >= fn_first && j < fn_end) { + if (scr_pipe_is_stage(fs, j)) { /* ret = stage({ signal }) for the source, else * stage(makeAsyncIterable(ret), { signal }) */ - size_t k = j - fn_first; bool dest = last && fs->dest != SCR_PIPE_DEST_STREAM; - void *src = k == 0 && fs->source ? NULL - : ret_gen ? (void *)ret_gen - : (void *)scr_stream_retain(ret); + void *src = j == 0 ? NULL + : ret_gen ? (void *)ret_gen + : (void *)scr_stream_retain(ret); ret_gen = NULL; - void *out = scr_pipe_call_stage(fs, k, src); + void *out = scr_pipe_call_stage(fs, k++, src); if (scr_exc_pending()) { if (out && dest) scr_promise_release(out); else if (out) scr_gen_release(out); @@ -3776,7 +3795,7 @@ static ScrStream *scr_stream_pipeline_ex(size_t n, ScrStream **streams, ScrClosu } ScrStream *s = streams[i]; bool writing = j > 0; - if (reading && !(j + 1 >= fn_first && j + 1 < fn_end) && + if (reading && !scr_pipe_is_stage(fs, j + 1) && (streams[i + 1]->st->close_emitted || streams[i + 1]->st->destroyed)) { scr_pipe_throw("ERR_STREAM_UNABLE_TO_PIPE", "Cannot pipe to a closed or destroyed stream", SCR_ERR_ERROR); @@ -3803,7 +3822,7 @@ static ScrStream *scr_stream_pipeline_ex(size_t n, ScrStream **streams, ScrClosu ctx->finish_count++; ScrGen *g = ret_gen; ret_gen = NULL; - scr_pump_start(ctx, g, fs, s, end); + scr_pump_start(ctx, g, fs->helpers + 3 * h++, s, end); } else if (j > 0) { if (!ret->st->has_r) { static const char msg[] = @@ -3841,12 +3860,12 @@ static ScrStream *scr_stream_pipeline_ex(size_t n, ScrStream **streams, ScrClosu return scr_stream_retain(ret); } -static ScrPipeStages scr_pipe_stages_of(double at, double count, ScrClosure **stages, double flags, - double mask, ScrClosure *next, ScrClosure *read, - ScrClosure *ret, ScrClosure *box, ScrDyn *signal) { +static ScrPipeStages scr_pipe_stages_of(double count, ScrClosure **stages, double positions, + double flags, double mask, ScrClosure **helpers, + ScrClosure *box, ScrDyn *signal) { int bits = (int)flags; - ScrPipeStages fs = {(size_t)at, (size_t)count, stages, (bits & 1) != 0, (bits >> 1) & 3, - (unsigned)mask, signal, next, read, ret, box}; + ScrPipeStages fs = {(size_t)count, (uint64_t)positions, stages, (bits >> 1) & 3, (unsigned)mask, + signal, helpers, box}; return fs; } @@ -3864,25 +3883,25 @@ static void scr_pipe_dyn_value_inv(ScrClosure *cb, ScrError *err, ScrDyn *value, if (r != NULL) scr_dyn_release(r); } -/* pipeline(...) with a run of function stages (see ScrPipeStages): flags - * bit 0 marks a source function and bits 1-2 the destination kind; mask - * names the stages taking { signal }. The helpers, box, signal and abort - * are borrowed (each NULL when unused); cb moves. */ -ScrStream *scr_stream_pipeline_stages(double n, ScrStream **streams, double at, double count, - ScrClosure **stages, double flags, double mask, - ScrClosure *next, ScrClosure *read, ScrClosure *ret, - ScrClosure *box, ScrDyn *signal, ScrClosure *abort, - ScrClosure *cb, ScrStreamErrInv inv) { - ScrPipeStages fs = scr_pipe_stages_of(at, count, stages, flags, mask, next, read, ret, box, signal); +/* pipeline(...) with function stages (see ScrPipeStages): positions + * marks the function stages among the n + count stages, flags bits 1-2 + * give the destination kind, and mask names the stages taking + * { signal }. The helpers, box, signal and abort are borrowed (each NULL + * when unused); cb moves. */ +ScrStream *scr_stream_pipeline_stages(double n, ScrStream **streams, double count, + ScrClosure **stages, double positions, double flags, + double mask, ScrClosure **helpers, ScrClosure *box, + ScrDyn *signal, ScrClosure *abort, ScrClosure *cb, + ScrStreamErrInv inv) { + ScrPipeStages fs = scr_pipe_stages_of(count, stages, positions, flags, mask, helpers, box, signal); return scr_stream_pipeline_ex((size_t)n, streams, cb, inv, true, NULL, NULL, &fs, abort, NULL); } -ScrStream *scr_stream_pipeline_stages_dyn(double n, ScrStream **streams, double at, double count, - ScrClosure **stages, double flags, double mask, - ScrClosure *next, ScrClosure *read, ScrClosure *ret, - ScrClosure *box, ScrDyn *signal, ScrClosure *abort, - ScrDyn *cb) { - ScrPipeStages fs = scr_pipe_stages_of(at, count, stages, flags, mask, next, read, ret, box, signal); +ScrStream *scr_stream_pipeline_stages_dyn(double n, ScrStream **streams, double count, + ScrClosure **stages, double positions, double flags, + double mask, ScrClosure **helpers, ScrClosure *box, + ScrDyn *signal, ScrClosure *abort, ScrDyn *cb) { + ScrPipeStages fs = scr_pipe_stages_of(count, stages, positions, flags, mask, helpers, box, signal); ScrClosure *c = scr_stream_dynopt_clo(cb, (void *)&scr_stream_finished_dyn_inv); /* a function destination's value reaches the callback */ ScrPipeValInv vi = fs.dest != SCR_PIPE_DEST_STREAM ? &scr_pipe_dyn_value_inv : NULL; @@ -4263,12 +4282,12 @@ static void scr_sp_value_inv(ScrClosure *cb, ScrError *err, ScrDyn *value, ScrPr scr_promise_release(p); } -ScrPromise *scr_sp_pipeline_stages(double n, ScrStream **streams, double at, double count, - ScrClosure **stages, double flags, double mask, - ScrClosure *next, ScrClosure *read, ScrClosure *ret, - ScrClosure *box, ScrDyn *signal, ScrClosure *abort) { +ScrPromise *scr_sp_pipeline_stages(double n, ScrStream **streams, double count, + ScrClosure **stages, double positions, double flags, + double mask, ScrClosure **helpers, ScrClosure *box, + ScrDyn *signal, ScrClosure *abort) { ScrPromise *p = scr_promise_new(); - ScrPipeStages fs = scr_pipe_stages_of(at, count, stages, flags, mask, next, read, ret, box, signal); + ScrPipeStages fs = scr_pipe_stages_of(count, stages, positions, flags, mask, helpers, box, signal); ScrPipeValInv vi = fs.dest != SCR_PIPE_DEST_STREAM ? &scr_sp_value_inv : NULL; ScrStream *dst = scr_stream_pipeline_ex((size_t)n, streams, scr_sp_watcher(p), &scr_sp_settle_inv, true, NULL, NULL, &fs, abort, vi); @@ -5800,6 +5819,19 @@ void scr_stream_destroy_done(ScrStream *s, ScrError *err /*moves*/) { } } +/* Transform._write's callback after the transform pushed its output: + * on a tick once the readable side has ended, at once when the writable + * side has ended, the transform pushed nothing, or the readable buffer is + * below its high-water mark, and otherwise when the readable side next + * reads (Transform._read). */ +static void scr_stream_transform_callback(ScrStream *s) { + ScrStreamState *st = s->st; + if (st->r.ended) scr_st_tick(s, SCR_ST_XF_DONE, NULL, NULL); + else if (st->w.ending || st->xf_length == st->r.length || st->r.length < st->r.hwm) + scr_stream_write_done(s, NULL); + else st->xf_held = true; +} + void scr_stream_transform_done(ScrStream *s, ScrError *err /*moves*/, ScrBytes *data /*moves*/, ScrStr *data_str /*moves*/) { if (!err) { @@ -5820,7 +5852,8 @@ void scr_stream_transform_done(ScrStream *s, ScrError *err /*moves*/, ScrBytes * if (err) scr_error_release(err); return; } - scr_stream_write_done(s, err); + if (err) scr_stream_write_done(s, err); + else scr_stream_transform_callback(s); } void scr_stream_flush_done(ScrStream *s, ScrError *err /*moves*/, ScrBytes *data /*moves*/, @@ -6171,6 +6204,9 @@ static void scr_stream_run_tick(ScrStreamTick *t) { } break; } + case SCR_ST_XF_DONE: + if (st->w.writing) scr_stream_write_done(s, NULL); + break; case SCR_ST_AFTER_WRITE: { st->w.after_scheduled = false; ScrClosure *cb = t->cb; @@ -6369,8 +6405,11 @@ static void scr_stream_dispatch(void) { } } +static const ScrHandleStreamHooks scr_handle_stream_hooks_impl; + void scr_stream_install(void) { scr_emitter_on_hook = &scr_stream_on_listener; + scr_handle_stream_hooks = &scr_handle_stream_hooks_impl; scr_loop_set_stream(&scr_stream_ticks_pending, &scr_stream_dispatch); } @@ -6409,23 +6448,138 @@ void scr_stream_worker_stdio(void) { #endif } -static ScrStream *scr_stream_view_receiver(const ScrDyn *value) { - if (value->kind == SCR_DYN_TYPED_REF && value->v.typed_ref.type_key_len > 7 && - !memcmp(value->v.typed_ref.type_key, "object:", 7)) { - ScrStream *stream = scr_dyn_typed_ref_unbox(value); - const ScrVt *vt = scr_stream_vt_of(stream); - if (vt && ((vt->pre >= scr_readable_vt.pre && vt->pre <= scr_readable_vt.post) || - (vt->pre >= scr_writable_vt.pre && vt->pre <= scr_writable_vt.post))) return stream; - value->v.typed_ref.release(stream); +/* ── handles that are Transforms (Cipheriv, Decipheriv) ────────────────── + * A Transform handle (ScrDynHandleOps.transform) used as a stream gets one + * native Transform, built here on first use and kept in the handle's + * slot. Its steps share the handle (held, and traced: the handle and its + * stream form a cycle the collector reclaims) and the handle's static + * ops: each chunk goes through the handle's update(), and the flush pushes + * final() or reports its error. */ + +static void *scr_handle_ops_retain(void *p) { return p; } +static void scr_handle_ops_release(void *p) { (void)p; } + +static const ScrDynHandleOps *scr_handle_step_handle_ops(ScrClosure *cb) { + return scr_box_get_ref(cb->caps[1]); +} + +/* The step's handle, borrowed: scr_box_get_ref answers +1, and the box + * keeps the handle alive. */ +static void *scr_handle_step_handle(ScrClosure *cb) { + void *h = scr_box_get_ref(cb->caps[0]); + scr_handle_step_handle_ops(cb)->release(h); + return h; +} + +static const ScrDynHandleTransform *scr_handle_step_ops(ScrClosure *cb) { + return scr_handle_step_handle_ops(cb)->transform; +} + +/* The pending exception as a stream error (+1). */ +static ScrError *scr_handle_step_caught(void) { + ScrCaught *caught = scr_exc_take(); + ScrDyn *value = scr_caught_to_dyn(caught); + scr_caught_release(caught); + ScrError *err = value ? scr_errdyn_err_of(value) : NULL; + if (value) scr_dyn_release(value); + return err ? err : scr_stream_mkerr("ERR_INTERNAL_ASSERTION", "a Transform handle threw a value that is not an Error"); +} + +static void scr_handle_step_transform(ScrClosure *cb, ScrStream *s, ScrBytes *chunk) { + /* A throwing update() leaves the exception pending: it throws out of + * the write, as Node's _transform does. */ + ScrBytes *out = scr_handle_step_ops(cb)->update(scr_handle_step_handle(cb), chunk); + if (out) scr_stream_transform_done(s, NULL, out, NULL); +} + +static void scr_handle_step_flush(ScrClosure *cb, ScrStream *s) { + ScrBytes *out = scr_handle_step_ops(cb)->final(scr_handle_step_handle(cb)); + scr_stream_flush_done(s, out ? NULL : scr_handle_step_caught(), out, NULL); +} + +static void scr_handle_step_destroy(ScrClosure *cb, ScrStream *s, ScrError *err) { + (void)cb; + scr_stream_destroy_done(s, err ? scr_error_retain(err) : NULL); +} + +static ScrClosure *scr_handle_step(const ScrDynHandleOps *ops, void *h) { + ScrClosure *step = scr_closure_new(NULL, 2); + step->caps[0] = scr_box_new_obj(ops->retain, ops->release, ops->transform->trace); + scr_box_set_ref(step->caps[0], ops->retain(h)); + step->caps[1] = scr_box_new_obj(scr_handle_ops_retain, scr_handle_ops_release, NULL); + scr_box_set_ref(step->caps[1], (void *)ops); + return step; +} + +/* The Transform of handle h (+1), built on first use. */ +static ScrStream *scr_handle_transform(const ScrDynHandleOps *ops, void *h) { + const ScrDynHandleTransform *t = ops->transform; + ScrStream *s = t->stream(h); + if (!s) { + s = scr_stream_new_transform(-1, -1, true, true, true, true, true, + scr_handle_step(ops, h), scr_handle_step_transform, + scr_handle_step(ops, h), scr_handle_step_flush, + scr_handle_step(ops, h), scr_handle_step_destroy); + s->cls = ops->cls; + t->set_stream(h, s, scr_stream_release_v); } + return scr_stream_retain(s); +} + +/* The native stream of a checked-dynamic Transform handle (+1), or NULL + * when the value is not one. */ +ScrStream *scr_stream_of_handle(const ScrDyn *value) { + if (value->kind != SCR_DYN_HANDLE) return NULL; + const ScrDynHandleOps *ops = scr_dyn_handle_ops_of(value); + return ops->transform ? scr_handle_transform(ops, value->v.handle.ptr) : NULL; +} + +/* A boxed native stream (+1), or NULL for any other value. */ +static ScrStream *scr_stream_boxed(const ScrDyn *value) { + if (value->kind != SCR_DYN_TYPED_REF || value->v.typed_ref.type_key_len <= 7 || + memcmp(value->v.typed_ref.type_key, "object:", 7) != 0) + return NULL; + ScrStream *stream = scr_dyn_typed_ref_unbox(value); + const ScrVt *vt = scr_stream_vt_of(stream); + if (vt && ((vt->pre >= scr_readable_vt.pre && vt->pre <= scr_readable_vt.post) || + (vt->pre >= scr_writable_vt.pre && vt->pre <= scr_writable_vt.post))) + return stream; + value->v.typed_ref.release(stream); + return NULL; +} + +/* stream.handleStream: the stream of a value typed as a stream handle + * class (a Cipheriv or Decipheriv), +1. A value of that type can also be + * the stream itself, boxed (the stream addAbortSignal answers). */ +ScrStream *scr_stream_handle(const ScrDyn *value) { + ScrStream *stream = scr_stream_of_handle(value); + if (!stream) stream = scr_stream_boxed(value); + if (stream) return stream; + static const char message[] = "Stream use of a value that is not a native stream has no lowering"; + scr_throw_error_msg_code(SCR_ERR_ERROR, message, sizeof message - 1, "SC2020"); + return NULL; +} + +static ScrStream *scr_stream_view_receiver(const ScrDyn *value) { + ScrStream *stream = scr_stream_of_handle(value); + if (!stream) stream = scr_stream_boxed(value); + if (stream) return stream; scr_dyn_arg_type_fail("destination", "an instance of Stream", value); return NULL; } -static ScrDyn *scr_stream_view_call(ScrClosure *cb, ScrDyn *const *args, size_t argc) { - ScrStream *stream = scr_box_get_ref(cb->caps[0]); - ScrStr *name = scr_box_get_ref(cb->caps[1]); - const char *method = name->data; +/* The members a native stream answers to checked-dynamic calls. */ +static const char *const scr_stream_dyn_methods[] = { + "on", "once", "addListener", "prependListener", "prependOnceListener", + "off", "removeListener", "pipe", "unpipe", "read", "pause", "resume", "isPaused", + "setEncoding", "write", "end", "destroy", "cork", "uncork", +}; + +/* A checked-dynamic call of one of a native stream's members (borrowed + * stream): the result, +1, with `this` results answering `self`, or the + * current `this` when self is NULL. */ +static ScrDyn *scr_stream_dyn_call(ScrStream *stream, ScrDyn *self, const char *method, + ScrDyn *const *args, size_t argc) { ScrDyn *result = NULL; ScrDyn *first = argc ? args[0] : scr_dyn_undefined(); if (!strcmp(method, "on") || !strcmp(method, "once") || !strcmp(method, "addListener") || @@ -6501,21 +6655,72 @@ static ScrDyn *scr_stream_view_call(ScrClosure *cb, ScrDyn *const *args, size_t scr_error_release(error); } else if (!strcmp(method, "cork")) scr_stream_cork(stream); else if (!strcmp(method, "uncork")) scr_stream_uncork(stream); - if (!result && !scr_exc_pending()) result = scr_dyn_this_get(); + if (!result && !scr_exc_pending()) result = self ? scr_dyn_retain(self) : scr_dyn_this_get(); + return result; +} + +static ScrDyn *scr_stream_view_call(ScrClosure *cb, ScrDyn *const *args, size_t argc) { + ScrStream *stream = scr_box_get_ref(cb->caps[0]); + ScrStr *name = scr_box_get_ref(cb->caps[1]); + ScrDyn *result = scr_stream_dyn_call(stream, NULL, name->data, args, argc); scr_str_release(name); scr_stream_release(stream); return result; } +/* ScrHandleStreamHooks.call: a member a native stream answers to + * checked-dynamic calls, on the handle's Transform. Members that answer + * the stream answer `self`. */ +static bool scr_handle_stream_call(const ScrDynHandleOps *ops, void *h, ScrDyn *self, + const char *method, ScrDyn *const *args, size_t argc, + ScrDyn **result) { + for (size_t i = 0; i < sizeof scr_stream_dyn_methods / sizeof *scr_stream_dyn_methods; i++) { + if (strcmp(method, scr_stream_dyn_methods[i]) != 0) continue; + ScrStream *stream = scr_handle_transform(ops, h); + *result = scr_stream_dyn_call(stream, self, method, args, argc); + scr_stream_release(stream); + return true; + } + return false; +} + +/* The stream properties a checked-dynamic read answers. */ +static const char *const scr_stream_dyn_properties[] = { + "readable", "readableEnded", "readableLength", "readableHighWaterMark", + "writable", "writableEnded", "writableFinished", "writableLength", "writableHighWaterMark", + "writableNeedDrain", "destroyed", "closed", +}; + +static ScrDyn *scr_stream_dyn_property(ScrStream *stream, const char *name) { + double value = scr_stream_prop(stream, name); + bool number = strstr(name, "Length") || strstr(name, "HighWaterMark"); + return number ? scr_dyn_new_num(value) : scr_dyn_new_bool(value != 0); +} + +/* ScrHandleStreamHooks.get: a stream property of the handle's Transform. */ +static ScrDyn *scr_handle_stream_get(const ScrDynHandleOps *ops, void *h, const char *key, + size_t key_len) { + for (size_t i = 0; i < sizeof scr_stream_dyn_properties / sizeof *scr_stream_dyn_properties; i++) { + const char *name = scr_stream_dyn_properties[i]; + if (strlen(name) != key_len || memcmp(name, key, key_len) != 0) continue; + ScrStream *stream = scr_handle_transform(ops, h); + ScrDyn *value = scr_stream_dyn_property(stream, name); + scr_stream_release(stream); + return value; + } + return NULL; +} + +static const ScrHandleStreamHooks scr_handle_stream_hooks_impl = { + scr_handle_stream_call, + scr_handle_stream_get, +}; + ScrDyn *scr_stream_dyn_view(ScrStream *stream) { ScrDyn *view = scr_dyn_new_obj(); ScrDyn *prototype = scr_dyn_new_obj(); - static const char *const methods[] = { - "on", "once", "addListener", "prependListener", "prependOnceListener", - "off", "removeListener", "pipe", "unpipe", "read", "pause", "resume", "isPaused", - "setEncoding", "write", "end", "destroy", "cork", "uncork", - }; - for (size_t i = 0; i < sizeof methods / sizeof *methods; i++) { + const char *const *methods = scr_stream_dyn_methods; + for (size_t i = 0; i < sizeof scr_stream_dyn_methods / sizeof *scr_stream_dyn_methods; i++) { ScrClosure *cb = scr_closure_new((void *)scr_stream_view_call, 2); cb->caps[0] = scr_box_new_obj(scr_stream_retain_v, scr_stream_release_v, scr_stream_trace); cb->caps[1] = scr_box_new(SCR_BOX_STR); @@ -6524,16 +6729,9 @@ ScrDyn *scr_stream_dyn_view(ScrStream *stream) { scr_dyn_obj_set(prototype, methods[i], strlen(methods[i]), scr_dyn_new_func(cb, scr_stream_view_call, 0, "native stream method", methods[i])); } - static const char *const properties[] = { - "readable", "readableEnded", "readableLength", "readableHighWaterMark", - "writable", "writableEnded", "writableFinished", "writableLength", "writableHighWaterMark", - "writableNeedDrain", "destroyed", "closed", - }; - for (size_t i = 0; i < sizeof properties / sizeof *properties; i++) { - double value = scr_stream_prop(stream, properties[i]); - bool number = strstr(properties[i], "Length") || strstr(properties[i], "HighWaterMark"); - scr_dyn_obj_set(view, properties[i], strlen(properties[i]), - number ? scr_dyn_new_num(value) : scr_dyn_new_bool(value != 0)); + for (size_t i = 0; i < sizeof scr_stream_dyn_properties / sizeof *scr_stream_dyn_properties; i++) { + const char *name = scr_stream_dyn_properties[i]; + scr_dyn_obj_set(view, name, strlen(name), scr_stream_dyn_property(stream, name)); } if (stream->st->native && stream->st->native_ops && stream->st->native_ops->view) { ScrClosure *cb = scr_closure_new((void *)scr_stream_view_call, 2); @@ -6712,17 +6910,13 @@ ScrStream *scr_stream_pipe_general(ScrStream *src, ScrStream *dst, bool end) { /* A pipeline() stage that arrives as a checked-dynamic value (the * stream.unboxDyn libCall): a boxed native stream unwraps by identity - * (+1); any other value throws the compiled program's not-supported - * message (Node also takes iterables and functions there). */ + * (+1), a stream handle (a Cipheriv or Decipheriv) answers its stream, and + * any other value throws the compiled program's not-supported message + * (Node also takes iterables and functions there). */ ScrStream *scr_stream_unbox_dyn(const ScrDyn *value, const ScrStr *message) { - if (value->kind == SCR_DYN_TYPED_REF && value->v.typed_ref.type_key_len > 7 && - !memcmp(value->v.typed_ref.type_key, "object:", 7)) { - ScrStream *stream = scr_dyn_typed_ref_unbox(value); - const ScrVt *vt = scr_stream_vt_of(stream); - if (vt && ((vt->pre >= scr_readable_vt.pre && vt->pre <= scr_readable_vt.post) || - (vt->pre >= scr_writable_vt.pre && vt->pre <= scr_writable_vt.post))) return stream; - value->v.typed_ref.release(stream); - } + ScrStream *stream = scr_stream_of_handle(value); + if (!stream) stream = scr_stream_boxed(value); + if (stream) return stream; scr_throw_error_msg_code(SCR_ERR_ERROR, message->data, message->len, "SC2020"); return NULL; } diff --git a/packages/runtime/src/scr_stream.h b/packages/runtime/src/scr_stream.h index 99f58331c..1e5b789e7 100644 --- a/packages/runtime/src/scr_stream.h +++ b/packages/runtime/src/scr_stream.h @@ -143,6 +143,13 @@ ScrStream *scr_stream_pipe_general(ScrStream *src, ScrStream *dst, bool end); /* A boxed native stream out of a checked-dynamic pipeline stage (+1), or * NULL with `message` thrown as the not-supported Error. */ ScrStream *scr_stream_unbox_dyn(const struct ScrDyn *value, const ScrStr *message); +/* Handles that are Transforms (ScrDynHandleOps.transform: Cipheriv and + * Decipheriv). scr_stream_of_handle: the handle's Transform (+1), built on + * first use, or NULL when the value is not such a handle. + * stream.handleStream: the same, throwing the coded SC2020 error for any + * other value. */ +ScrStream *scr_stream_of_handle(const struct ScrDyn *value); +ScrStream *scr_stream_handle(const struct ScrDyn *value); /* Checked-dynamic stream slots (scr_stream_dyn.c): the native stream a * value boxes with the requested side (1 readable, 2 writable; +1, or * NULL), and readable.pipe()/unpipe() for a destination that is not a @@ -197,35 +204,34 @@ ScrStream *scr_stream_pipeline(double n, ScrStream **streams, ScrClosure *cb, Sc * mustCall wrappers; borrowed). */ ScrClosure *scr_stream_finished_dyn(ScrStream *s, struct ScrDyn *cb); ScrStream *scr_stream_pipeline_dyn(double n, ScrStream **streams, struct ScrDyn *cb); -/* pipeline(...[, cb]) with one run of function stages (pipeline.js): the - * run starts at streams[at] (count functions). flags bit 0: the first - * stage is a source function, called with only its options; bits 1-2: - * the last stage is a function destination (2: an async function whose - * promise, boxed through box, carries the value to the callback), and - * pipeline() answers a PassThrough. Other - * stages get the previous stream itself or the previous stage's async - * generator. mask bit i: stage i takes { signal } (the pipeline's own - * AbortSignal, aborted through abort when the pipeline finishes). The - * last generator before a stream or a PassThrough is pumped - * (pumpToNode) through its helpers: next/ret answer the resume's - * promise, read maps its IteratorResult to the value or - * scr_stream_from_done. Streams and closures borrowed (helpers, box, - * signal and abort NULL when unused); cb moves. */ -ScrStream *scr_stream_pipeline_stages(double n, ScrStream **streams, double at, double count, - ScrClosure **stages, double flags, double mask, - ScrClosure *next, ScrClosure *read, ScrClosure *ret, - ScrClosure *box, struct ScrDyn *signal, ScrClosure *abort, - ScrClosure *cb, ScrStreamErrInv inv); -ScrStream *scr_stream_pipeline_stages_dyn(double n, ScrStream **streams, double at, double count, - ScrClosure **stages, double flags, double mask, - ScrClosure *next, ScrClosure *read, ScrClosure *ret, - ScrClosure *box, struct ScrDyn *signal, - ScrClosure *abort, struct ScrDyn *cb); -struct ScrPromise *scr_sp_pipeline_stages(double n, ScrStream **streams, double at, double count, - ScrClosure **stages, double flags, double mask, - ScrClosure *next, ScrClosure *read, ScrClosure *ret, - ScrClosure *box, struct ScrDyn *signal, - ScrClosure *abort); +/* pipeline(...[, cb]) with function stages (pipeline.js): positions bit j + * marks the j-th of the n + count stages a function. A function at + * position 0 is a source function, called with only its options. flags + * bits 1-2: the last stage is a function destination (2: an async + * function whose promise, boxed through box, carries the value to the + * callback), and pipeline() answers a PassThrough. Other stages get the + * previous stream itself or the previous stage's async generator. mask + * bit i: function i takes { signal } (the pipeline's own AbortSignal, + * aborted through abort when the pipeline finishes). The last generator + * of each run that a stream follows is pumped (pumpToNode) through its + * three helpers in `helpers`: next/ret answer the resume's promise, read + * maps its IteratorResult to the value or scr_stream_from_done. Streams + * and closures borrowed (helpers, box, signal and abort NULL when + * unused); cb moves. */ +ScrStream *scr_stream_pipeline_stages(double n, ScrStream **streams, double count, + ScrClosure **stages, double positions, double flags, + double mask, ScrClosure **helpers, ScrClosure *box, + struct ScrDyn *signal, ScrClosure *abort, ScrClosure *cb, + ScrStreamErrInv inv); +ScrStream *scr_stream_pipeline_stages_dyn(double n, ScrStream **streams, double count, + ScrClosure **stages, double positions, double flags, + double mask, ScrClosure **helpers, ScrClosure *box, + struct ScrDyn *signal, ScrClosure *abort, + struct ScrDyn *cb); +struct ScrPromise *scr_sp_pipeline_stages(double n, ScrStream **streams, double count, + ScrClosure **stages, double positions, double flags, + double mask, ScrClosure **helpers, ScrClosure *box, + struct ScrDyn *signal, ScrClosure *abort); /* stream.compose(s1..sn): a pipeline over the streams and a Duplex that * writes to the first and reads from the last (compose.js). One stream * answers itself (Duplex.from of a duplex). +1, or NULL with diff --git a/packages/runtime/src/scr_stream_dyn.c b/packages/runtime/src/scr_stream_dyn.c index 11dcf6ee2..5cef9d536 100644 --- a/packages/runtime/src/scr_stream_dyn.c +++ b/packages/runtime/src/scr_stream_dyn.c @@ -17,8 +17,11 @@ static bool scr_stream_dyn_within(const ScrVt *vt, const ScrVt *cls) { /* The native stream a checked-dynamic value boxes (+1), when it has the * requested side (1 readable, 2 writable; Duplex, Transform and PassThrough - * have both), or NULL for any other value. Never throws. */ + * have both), or the stream of a stream handle, or NULL for any other + * value. Never throws. */ ScrStream *scr_stream_dyn_native(const ScrDyn *value, double side) { + /* A Cipheriv or Decipheriv is a Transform. */ + if (value->kind == SCR_DYN_HANDLE) return scr_stream_of_handle(value); if (value->kind != SCR_DYN_TYPED_REF || value->v.typed_ref.type_key_len <= 7 || memcmp(value->v.typed_ref.type_key, "object:", 7) != 0) return NULL; diff --git a/packages/runtime/src/scr_zlib_stream.c b/packages/runtime/src/scr_zlib_stream.c index 4ed717d53..54ffacf02 100644 --- a/packages/runtime/src/scr_zlib_stream.c +++ b/packages/runtime/src/scr_zlib_stream.c @@ -104,7 +104,8 @@ static void scr_zs_finish(ScrZlibStream *zs, ScrStream *s) { zs->flushing = false; scr_stream_flush_done(s, NULL, NULL, NULL); } else { - scr_stream_transform_done(s, NULL, NULL, NULL); + /* scr_zs_complete has applied Transform._write's callback rule. */ + scr_stream_write_done(s, NULL); } } diff --git a/tests/corpus/7055-cipher-streams.ts b/tests/corpus/7055-cipher-streams.ts new file mode 100644 index 000000000..8befd2339 --- /dev/null +++ b/tests/corpus/7055-cipher-streams.ts @@ -0,0 +1,182 @@ +// Cipheriv and Decipheriv as Transforms: pipeline() and stream/promises +// pipeline stages, pipe() chains, for await, write()/end() with 'data' +// events, setEncoding(), finished(), compose(), addAbortSignal(), +// stream/consumers, the stream properties, and the cipher's state after +// its stream flushed. A Hash reads through stream/consumers and for await +// the same way. +import * as crypto from "node:crypto"; +import { + PassThrough, + Readable, + Writable, + addAbortSignal, + compose, + finished, + pipeline, +} from "node:stream"; +import { text } from "node:stream/consumers"; +import { pipeline as pipelinePromise } from "node:stream/promises"; + +const key = Buffer.alloc(32, 7); +const iv = Buffer.alloc(16, 3); + +function collector(into: Buffer[]): Writable { + return new Writable({ + write(chunk: Buffer, _enc: string, cb: () => void) { + into.push(chunk); + cb(); + }, + }); +} + +async function viaCallbackPipeline(): Promise { + const out: Buffer[] = []; + await new Promise((resolve) => { + pipeline( + Readable.from(["hello ", "cipher ", "streams"]), + crypto.createCipheriv("aes-256-cbc", key, iv), + crypto.createDecipheriv("aes-256-cbc", key, iv), + collector(out), + (err: Error | null) => { + console.log("callback pipeline:", err ?? null, Buffer.concat(out).toString()); + resolve(); + }, + ); + }); +} + +async function viaPromisePipeline(): Promise { + const hash = crypto.createHash("sha256"); + await pipelinePromise( + Readable.from([Buffer.from("abc"), Buffer.from("def")]), + crypto.createCipheriv("aes-256-ctr", key, iv), + new PassThrough(), + hash, + ); + console.log("promise pipeline digest:", hash.digest("hex")); +} + +async function viaPipeAndForAwait(): Promise { + const cipher = crypto.createCipheriv("aes-256-cbc", key, iv); + const decipher = crypto.createDecipheriv("aes-256-cbc", key, iv); + const chain = Readable.from(["piped ", "through ", "two ciphers"]).pipe(cipher).pipe(decipher); + let text = ""; + for await (const chunk of chain) text += chunk; + console.log("pipe + for await:", text); +} + +async function viaEvents(): Promise { + const cipher = crypto.createCipheriv("aes-128-cbc", Buffer.alloc(16, 1), Buffer.alloc(16, 2)); + cipher.setEncoding("hex"); + let hex = ""; + cipher.on("data", (chunk: string) => { + hex += chunk; + }); + await new Promise((resolve) => { + cipher.on("end", () => { + console.log("events:", typeof hex, hex); + resolve(); + }); + console.log("write returns:", cipher.write("first block of text")); + cipher.end("and the rest"); + }); + // The stream's flush ran final(): the cipher is finished. + try { + cipher.update("more"); + } catch (e) { + console.log("update after the stream:", (e as Error).message); + } + try { + cipher.final(); + } catch (e) { + console.log("final after the stream:", (e as NodeJS.ErrnoException).code, (e as Error).message); + } +} + +async function gcm(): Promise { + const gcmKey = Buffer.alloc(32, 9); + const nonce = Buffer.alloc(12, 4); + const enc = crypto.createCipheriv("aes-256-gcm", gcmKey, nonce); + const parts: Buffer[] = []; + enc.on("data", (chunk: Buffer) => parts.push(chunk)); + await new Promise((resolve) => { + enc.on("end", () => resolve()); + enc.end("authenticated message"); + }); + const tag = enc.getAuthTag(); + console.log("gcm tag length:", tag.length); + const dec = crypto.createDecipheriv("aes-256-gcm", gcmKey, nonce); + dec.setAuthTag(tag); + let plain = ""; + for await (const chunk of Readable.from([Buffer.concat(parts)]).pipe(dec)) plain += chunk; + console.log("gcm round trip:", plain); +} + +async function viaFinishedAndProperties(): Promise { + const c = crypto.createDecipheriv("aes-256-ctr", key, iv); + console.log("before:", c.readable, c.writable, c.writableEnded, c.destroyed, c.writableLength); + const done = new Promise((resolve) => { + finished(c, (err?: NodeJS.ErrnoException | null) => { + console.log("finished:", err ?? null, c.readableEnded, c.writableFinished); + resolve(); + }); + }); + const parts: string[] = []; + const reading = (async () => { + for await (const chunk of c) parts.push(chunk.toString("hex")); + })(); + c.write(Buffer.from("00112233", "hex")); + c.end(Buffer.from("44556677", "hex")); + console.log("ended:", c.writableEnded, c.writable); + await reading; + await done; + console.log("for await over the cipher:", parts.join("")); + await new Promise((resolve) => setImmediate(resolve)); + console.log("after close:", c.destroyed, c.closed); +} + +async function viaCompositions(): Promise { + const both = compose( + crypto.createCipheriv("aes-256-cbc", key, iv), + crypto.createDecipheriv("aes-256-cbc", key, iv), + ); + let composed = ""; + both.on("data", (chunk: Buffer) => { + composed += chunk.toString(); + }); + await new Promise((resolve) => { + both.on("end", () => resolve()); + both.end("composed"); + }); + console.log("compose:", composed); + + const ac = new AbortController(); + const aborted = addAbortSignal(ac.signal, crypto.createCipheriv("aes-256-ctr", key, iv)); + aborted.on("error", (e: Error) => console.log("addAbortSignal:", e.name)); + const closed = new Promise((resolve) => aborted.on("close", () => resolve())); + ac.abort(); + await closed; + console.log("aborted cipher destroyed:", aborted.destroyed); + + const sealed = crypto.createCipheriv("aes-256-ctr", key, iv).update("through consumers"); + const plain = await text(Readable.from([sealed]).pipe(crypto.createDecipheriv("aes-256-ctr", key, iv))); + console.log("consumers text:", plain); + + const hash = crypto.createHash("sha1"); + hash.setEncoding("hex"); + console.log("hash text:", await text(Readable.from(["abc"]).pipe(hash))); + const md5 = crypto.createHash("md5"); + md5.end("xyz"); + for await (const digest of md5) console.log("hash for await:", (digest as Buffer).toString("hex")); +} + +async function main(): Promise { + await viaFinishedAndProperties(); + await viaCompositions(); + await viaCallbackPipeline(); + await viaPromisePipeline(); + await viaPipeAndForAwait(); + await viaEvents(); + await gcm(); +} +main(); diff --git a/tests/corpus/7056-cipher-stream-errors.ts b/tests/corpus/7056-cipher-stream-errors.ts new file mode 100644 index 000000000..e3e653091 --- /dev/null +++ b/tests/corpus/7056-cipher-stream-errors.ts @@ -0,0 +1,111 @@ +// Cipher stream errors, close ordering and backpressure: a bad decrypt +// through pipeline() destroys every stage with the same error, a failed +// GCM authentication reaches 'error' after the data, destroy() closes the +// cipher's stream, and write() reports the high-water mark with 'drain'. +import * as crypto from "node:crypto"; +import { Readable, Writable, pipeline } from "node:stream"; + +const key = Buffer.alloc(32, 5); +const iv = Buffer.alloc(16, 6); + +function sealed(text: string): Buffer { + const c = crypto.createCipheriv("aes-256-cbc", key, iv); + return Buffer.concat([c.update(text), c.final()]); +} + +async function badDecrypt(): Promise { + const log: string[] = []; + const source = Readable.from([sealed("secret message")]); + const decipher = crypto.createDecipheriv("aes-256-cbc", Buffer.alloc(32, 8), iv); + const sink = new Writable({ + write(_chunk: Buffer, _enc: string, cb: () => void) { + cb(); + }, + }); + source.on("close", () => log.push("source close")); + decipher.on("error", (e: NodeJS.ErrnoException) => log.push(`decipher error ${e.code}`)); + decipher.on("close", () => log.push("decipher close")); + sink.on("error", (e: NodeJS.ErrnoException) => log.push(`sink error ${e.code}`)); + sink.on("close", () => log.push("sink close")); + await new Promise((resolve) => { + pipeline(source, decipher, sink, (err: NodeJS.ErrnoException | null) => { + log.push(`callback ${err?.code} ${err?.message}`); + setImmediate(() => { + console.log(log.join("\n")); + console.log("destroyed:", source.destroyed, decipher.destroyed, sink.destroyed); + resolve(); + }); + }); + }); +} + +async function gcmWithoutTag(): Promise { + const d = crypto.createDecipheriv("aes-256-gcm", key, Buffer.alloc(12)); + await new Promise((resolve) => { + d.on("data", (chunk: Buffer) => console.log("gcm data", chunk.length)); + d.on("error", (e: Error) => console.log("gcm error:", e.message)); + d.on("close", () => { + console.log("gcm close"); + resolve(); + }); + d.end(Buffer.alloc(5)); + }); +} + +async function destroyMidway(): Promise { + const c = crypto.createCipheriv("aes-256-ctr", key, iv); + await new Promise((resolve) => { + c.on("data", (chunk: Buffer) => { + console.log("ctr data", chunk.length); + c.destroy(new Error("stop")); + }); + c.on("error", (e: Error) => console.log("ctr error:", e.message)); + c.on("close", () => { + console.log("ctr close, destroyed:", c.destroyed); + resolve(); + }); + c.write("0123456789"); + c.write("abcdef"); + }); +} + +async function backpressure(): Promise { + const c = crypto.createCipheriv("aes-256-ctr", key, iv); + let received = 0; + const slow = new Writable({ + highWaterMark: 1024, + write(chunk: Buffer, _enc: string, cb: () => void) { + received += chunk.length; + setImmediate(cb); + }, + }); + c.pipe(slow); + const block = Buffer.alloc(4096, 1); + let writes = 0; + let fullAt = -1; + while (writes < 64) { + writes++; + if (!c.write(block)) { + fullAt = writes; + break; + } + } + console.log("write returned false at", fullAt); + await new Promise((resolve) => c.once("drain", () => resolve())); + console.log("drained"); + await new Promise((resolve) => { + slow.on("finish", () => { + console.log("received", received); + resolve(); + }); + c.end(); + }); +} + +async function main(): Promise { + await badDecrypt(); + await gcmWithoutTag(); + await destroyMidway(); + await backpressure(); +} +main(); diff --git a/tests/corpus/7057-cipher-streams.mjs b/tests/corpus/7057-cipher-streams.mjs new file mode 100644 index 000000000..2173d29aa --- /dev/null +++ b/tests/corpus/7057-cipher-streams.mjs @@ -0,0 +1,85 @@ +// Cipheriv and Decipheriv streams from JavaScript: pipeline() and pipe() +// with cipher stages, a cipher passed through untyped helpers that call +// its stream members dynamically, and cipher stages that reach pipeline() +// through an untyped parameter. +import crypto from "node:crypto"; +import { pipeline, Readable, Writable } from "node:stream"; + +const key = Buffer.alloc(24, 1); +const iv = Buffer.alloc(16, 2); + +function attach(stream, label, log) { + stream.on("data", (chunk) => log.push(`${label} data ${chunk.length}`)); + stream.on("end", () => log.push(`${label} end`)); + stream.on("close", () => log.push(`${label} close`)); + return stream; +} + +function feed(stream, parts) { + for (const part of parts) stream.write(part); + stream.end(); +} + +function state(stream) { + return [stream.readable, stream.writable, stream.writableEnded, stream.destroyed, stream.readableLength].join(","); +} + +const log = []; +const cipher = attach(crypto.createCipheriv("aes-192-cbc", key, iv), "cipher", log); +const collected = []; +cipher.on("data", (chunk) => collected.push(chunk)); +cipher.on("end", () => { + console.log("state at end:", state(cipher)); + const sealed = Buffer.concat(collected); + const out = []; + pipeline( + Readable.from([sealed.subarray(0, 7), sealed.subarray(7)]), + crypto.createDecipheriv("aes-192-cbc", key, iv), + new Writable({ + write(chunk, enc, cb) { + out.push(chunk); + cb(); + }, + }), + (err) => { + console.log(log.join("\n")); + console.log("pipeline:", err ?? null, Buffer.concat(out).toString()); + const piped = Readable.from([sealed]).pipe(crypto.createDecipheriv("aes-192-cbc", key, iv)); + piped.setEncoding("utf8"); + let text = ""; + piped.on("data", (s) => (text += s)); + piped.on("end", () => { + console.log("pipe:", text); + throughHelper(sealed); + }); + }, + ); +}); + +// The stages arrive as untyped values. +function run(stages, done) { + const out = []; + pipeline( + stages[0], + stages[1], + new Writable({ + write(chunk, enc, cb) { + out.push(chunk); + cb(); + }, + }), + (err) => done(err, Buffer.concat(out)), + ); +} + +function throughHelper(sealed) { + run([Readable.from([sealed]), crypto.createDecipheriv("aes-192-cbc", key, iv)], (err, plain) => { + console.log("untyped stage:", err ?? null, plain.toString()); + run([Readable.from([sealed]), crypto.createDecipheriv("aes-192-cbc", Buffer.alloc(24, 9), iv)], (err2) => { + console.log("untyped stage, wrong key:", err2.code, err2.message); + }); + }); +} +console.log("state before:", state(cipher)); +feed(cipher, ["one ", "two ", "three"]); +console.log("state after end():", state(cipher)); diff --git a/tests/corpus/7058-pipeline-function-stages.mjs b/tests/corpus/7058-pipeline-function-stages.mjs new file mode 100644 index 000000000..f0a61e7c2 --- /dev/null +++ b/tests/corpus/7058-pipeline-function-stages.mjs @@ -0,0 +1,180 @@ +// pipeline() function stages from JavaScript: runs of async generator +// stages between streams, stages named by a function declaration and a +// const, sources that are arrays, strings, synchronous generators and +// generator functions (with the ticks Node's async-from-sync iteration +// takes), the string and Buffer methods of generator-fed chunks, and an +// error thrown in a stage after two streams. +import { pipeline, PassThrough, Transform, Writable } from "node:stream"; +import { pipeline as pipelinePromise } from "node:stream/promises"; +import zlib from "node:zlib"; + +function sink(into) { + return new Writable({ + write(chunk, enc, cb) { + into.push(chunk.toString()); + cb(); + }, + }); +} + +async function* upper(source) { + for await (const chunk of source) yield chunk.toString().toUpperCase(); +} + +const bracket = async function* (source) { + for await (const chunk of source) yield `[${chunk.trim().replaceAll("\n", "|")}]`; +}; + +function* letters() { + yield "a"; + yield "b"; + yield "c"; +} + +// Stages on both sides of two zlib streams, and a named stage. +{ + const out = []; + await pipelinePromise( + async function* () { + for (let i = 0; i < 3; i++) yield `line ${i}\n`; + }, + zlib.createGzip(), + zlib.createGunzip(), + upper, + bracket, + sink(out), + ); + console.log("split runs:", out.join("")); +} + +// Iterable sources, and the destination value that reaches the callback. +await new Promise((resolve) => { + pipeline( + ["x", "y", "z"], + zlib.createDeflate(), + zlib.createInflate(), + async function (source) { + let text = ""; + for await (const chunk of source) text += chunk; + return text; + }, + (err, value) => { + console.log("array source:", err ?? null, value); + resolve(); + }, + ); +}); +{ + const out = []; + await pipelinePromise("str", upper, sink(out)); + console.log("string source:", out.join(",")); + out.length = 0; + await pipelinePromise(letters, upper, sink(out)); + console.log("generator function source:", out.join(",")); + out.length = 0; + await pipelinePromise(letters(), new PassThrough(), sink(out)); + console.log("generator source:", out.join(",")); +} + +// Generator-fed chunks keep their string and Buffer methods. +{ + const out = []; + await pipelinePromise( + async function* () { + yield Buffer.from("buf"); + yield Buffer.from("fers"); + }, + async function* (source) { + for await (const chunk of source) yield chunk.toString("hex"); + }, + async function* (source) { + for await (const chunk of source) yield chunk.padStart(10, "-"); + }, + sink(out), + ); + console.log("chunk methods:", out.join(" ")); +} + +// The ticks of iterable sources match Node's for-await over them. +function ticker(log) { + let on = true; + (async () => { + for (let i = 0; on && i < 12; i++) { + log.push(`t${i}`); + await null; + } + })(); + return () => { + on = false; + }; +} +{ + const log = []; + const stop = ticker(log); + await new Promise((resolve) => + pipeline(["a", "b"], new PassThrough(), sink(log), () => { + stop(); + resolve(); + }), + ); + console.log("array ticks:", log.join(" ")); +} +{ + const log = []; + const stop = ticker(log); + await new Promise((resolve) => + pipeline( + letters, + async function* (source) { + for await (const chunk of source) { + log.push(`stage ${chunk}`); + yield chunk; + } + }, + sink(log), + () => { + stop(); + resolve(); + }, + ), + ); + console.log("generator function ticks:", log.join(" ")); +} + +// An error thrown in a stage after two streams destroys both with it. +{ + const log = []; + const first = new PassThrough(); + const second = new Transform({ + transform(chunk, enc, cb) { + cb(null, chunk.toString().toUpperCase()); + }, + }); + first.on("close", () => log.push("first close")); + second.on("close", () => log.push("second close")); + second.on("error", (e) => log.push(`second error ${e.name}`)); + await new Promise((resolve) => { + pipeline( + async function* () { + yield "one\n"; + yield "two\n"; + }, + first, + second, + async function* (source) { + for await (const chunk of source) { + log.push(`stage got ${JSON.stringify(chunk)}`); + if (chunk.length > 100) yield chunk; + throw new Error("stage failed"); + } + }, + sink([]), + (err) => { + log.push(`callback ${err.message}`); + setImmediate(resolve); + }, + ); + }); + console.log(log.join("\n")); + console.log("destroyed:", first.destroyed, second.destroyed); +} diff --git a/tests/corpus/7059-transform-backpressure.ts b/tests/corpus/7059-transform-backpressure.ts new file mode 100644 index 000000000..d19a41eb8 --- /dev/null +++ b/tests/corpus/7059-transform-backpressure.ts @@ -0,0 +1,90 @@ +// Transform write backpressure and typed pipeline stages. A Transform +// holds a write's callback while its readable side is at the high-water +// mark, so write() returns false once both buffers fill and 'drain' comes +// after the reader catches up. Typed stages: named async generator +// functions on both sides of zlib streams, an array source, and an async +// function destination whose value resolves the promise pipeline. +import { PassThrough, Transform, Writable } from "node:stream"; +import { pipeline } from "node:stream/promises"; +import * as zlib from "node:zlib"; + +class Copy extends Transform { + override _transform(chunk: Buffer, _enc: BufferEncoding, cb: (err?: Error | null) => void): void { + this.push(chunk); + cb(); + } +} + +async function measure(label: string, t: Transform): Promise { + let received = 0; + const slow = new Writable({ + highWaterMark: 1024, + write(chunk: Buffer, _enc: string, cb: () => void) { + received += chunk.length; + setImmediate(cb); + }, + }); + t.pipe(slow); + const block = Buffer.alloc(4096, 120); + let falseAt = -1; + for (let i = 1; i <= 64; i++) { + if (!t.write(block)) { + falseAt = i; + break; + } + } + console.log(label, "write() false at", falseAt, t.writableLength, t.readableLength); + await new Promise((resolve) => t.once("drain", () => resolve())); + console.log(label, "drained", t.writableLength); + await new Promise((resolve) => { + slow.on("finish", () => resolve()); + t.end(); + }); + console.log(label, "received", received); +} + +async function* numbered(): AsyncGenerator { + for (let i = 0; i < 4; i++) yield `row ${i}\n`; +} + +async function* shout(source: AsyncIterable): AsyncGenerator { + for await (const chunk of source) yield chunk.toString().toUpperCase(); +} + +async function count(source: AsyncIterable): Promise { + let n = 0; + for await (const chunk of source) n += chunk.split("\n").length - 1; + return n; +} + +async function main(): Promise { + await measure("passthrough", new PassThrough()); + await measure("subclass", new Copy()); + await measure( + "options", + new Transform({ + transform(chunk: Buffer, _enc: string, cb: (err?: Error | null, data?: Buffer) => void) { + cb(null, chunk); + }, + }), + ); + + const out: string[] = []; + await pipeline( + numbered, + zlib.createGzip(), + zlib.createGunzip(), + shout, + new Writable({ + write(chunk: Buffer, _enc: string, cb: () => void) { + out.push(chunk.toString()); + cb(); + }, + }), + ); + console.log("named stages:", JSON.stringify(out.join(""))); + + const rows = await pipeline(["a\n", "b\n", "c\n"], zlib.createGzip(), zlib.createGunzip(), shout, count); + console.log("array source, rows:", rows); +} +main(); diff --git a/tests/harness/corpus-costs/7055-cipher-streams.ts.json b/tests/harness/corpus-costs/7055-cipher-streams.ts.json new file mode 100644 index 000000000..7dda3fe10 --- /dev/null +++ b/tests/harness/corpus-costs/7055-cipher-streams.ts.json @@ -0,0 +1 @@ +{ "plain": 4395, "san": 6600 } diff --git a/tests/harness/corpus-costs/7056-cipher-stream-errors.ts.json b/tests/harness/corpus-costs/7056-cipher-stream-errors.ts.json new file mode 100644 index 000000000..b3e89de2a --- /dev/null +++ b/tests/harness/corpus-costs/7056-cipher-stream-errors.ts.json @@ -0,0 +1 @@ +{ "plain": 2558, "san": 4456 } diff --git a/tests/harness/corpus-costs/7057-cipher-streams.mjs.json b/tests/harness/corpus-costs/7057-cipher-streams.mjs.json new file mode 100644 index 000000000..1edb6e866 --- /dev/null +++ b/tests/harness/corpus-costs/7057-cipher-streams.mjs.json @@ -0,0 +1 @@ +{ "plain": 2780, "san": 5329 } diff --git a/tests/harness/corpus-costs/7058-pipeline-function-stages.mjs.json b/tests/harness/corpus-costs/7058-pipeline-function-stages.mjs.json new file mode 100644 index 000000000..d4709c5d6 --- /dev/null +++ b/tests/harness/corpus-costs/7058-pipeline-function-stages.mjs.json @@ -0,0 +1 @@ +{ "plain": 3703, "san": 7279 } diff --git a/tests/harness/corpus-costs/7059-transform-backpressure.ts.json b/tests/harness/corpus-costs/7059-transform-backpressure.ts.json new file mode 100644 index 000000000..cc34bea0e --- /dev/null +++ b/tests/harness/corpus-costs/7059-transform-backpressure.ts.json @@ -0,0 +1 @@ +{ "plain": 2394, "san": 5656 } diff --git a/tests/harness/self-hosting-native-toolchain.test.ts b/tests/harness/self-hosting-native-toolchain.test.ts index 7da7626b7..c01b52595 100644 --- a/tests/harness/self-hosting-native-toolchain.test.ts +++ b/tests/harness/self-hosting-native-toolchain.test.ts @@ -105,7 +105,7 @@ for (const backend of ["llvm"] as const) { object_format: target.objectFormat, minimum_os: target.minimumOs, }, - runtime_abi: { version: 24, marker: "scr_runtime_abi_v24" }, + runtime_abi: { version: 25, marker: "scr_runtime_abi_v25" }, compiler: { command: "fixture", identity: "fixture", target: target.llvmTriple }, macros: { executable: [], excluded: [], sanitizer: "external-toolchain-required" }, flavors: { From f00428ccb1aa6a7e0b3a384cb71b53bd4fe28217 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 22 Oct 2026 02:53:03 -0500 Subject: [PATCH 2/2] Keep pipe() answering its destination and evaluate pipeline() arguments in source order --- .../frontend/lowering/builtins/hash-stream.ts | 16 +++--- .../src/frontend/lowering/lower-stream.ts | 52 +++++++++++++------ tests/corpus/7055-cipher-streams.ts | 40 ++++++++++++++ 3 files changed, 87 insertions(+), 21 deletions(-) diff --git a/packages/compiler/src/frontend/lowering/builtins/hash-stream.ts b/packages/compiler/src/frontend/lowering/builtins/hash-stream.ts index f2f3625dd..88f6c2d95 100644 --- a/packages/compiler/src/frontend/lowering/builtins/hash-stream.ts +++ b/packages/compiler/src/frontend/lowering/builtins/hash-stream.ts @@ -82,7 +82,10 @@ export function lowerHashStreamCall( if (!isValueStream(lowerer, access.expression)) return null; const info = lowerer.classes.get(TRANSFORM_CLASS); if (!info) return null; - return withValueStreamReceiver(lowerer, access.expression, locOf(call), () => + // pipe(dest) answers its destination, which may be a Transform too; + // the other members typed as the Transform answer `this` + const answersThis = name !== "pipe"; + return withValueStreamReceiver(lowerer, access.expression, locOf(call), answersThis, () => streamMember ? lowerStreamMethodCall(lowerer, call, access, info) : lowerEmitterMethodCall(lowerer, call, access, info), @@ -98,19 +101,20 @@ export function lowerValueStreamProperty( if (expr.questionDotToken || !isValueStream(lowerer, expr.expression)) return null; const info = lowerer.classes.get(TRANSFORM_CLASS); if (!info) return null; - return withValueStreamReceiver(lowerer, expr.expression, locOf(expr), () => + return withValueStreamReceiver(lowerer, expr.expression, locOf(expr), true, () => lowerStreamProperty(lowerer, expr, info), ); } /** Evaluates the value `receiver` names once into a hidden local and runs - * `lower` with the receiver lowering to that value's Transform. A result - * typed as the Transform (a member that answers `this`) answers the value - * itself. */ + * `lower` with the receiver lowering to that value's Transform. When + * `answersThis`, a result typed as the Transform (the member answered + * `this`) answers the value itself. */ function withValueStreamReceiver( lowerer: Lowerer, receiver: ts.Expression, loc: SrcLoc, + answersThis: boolean, lower: () => IrExpr | null, ): IrExpr | null { const value = lowerer.lowerExpr(receiver); @@ -124,7 +128,7 @@ function withValueStreamReceiver( } if (!lowered) return null; const answersStream = - lowered.type.kind === "object" && lowered.type.className === TRANSFORM_CLASS; + answersThis && lowered.type.kind === "object" && lowered.type.className === TRANSFORM_CLASS; const result: IrExpr = answersStream ? { kind: "seqExpr", diff --git a/packages/compiler/src/frontend/lowering/lower-stream.ts b/packages/compiler/src/frontend/lowering/lower-stream.ts index ab21b8a7e..d6f89dc11 100644 --- a/packages/compiler/src/frontend/lowering/lower-stream.ts +++ b/packages/compiler/src/frontend/lowering/lower-stream.ts @@ -2935,6 +2935,10 @@ function lowerPipelineStages( // What the next function stage receives: null before the first stage, // else the previous stream's type or the previous stage's generator. let srcT: IrType | null = null; + // The streams and stages in argument order: the head lists all streams + // before all stages, so when a stage precedes a stream the arguments + // are bound to locals first, keeping JavaScript's evaluation order. + const argOrder: { list: IrExpr[]; index: number }[] = []; for (let j = 0; j < stageArgs.length; j++) { const arg = stageArgs[j]!; const fn = pipelineStageFunction(lowerer, arg); @@ -2948,6 +2952,7 @@ function lowerPipelineStages( const source = lowerIterableSource(lowerer, arg, syncSourceFn ? fn : null, loc); if (source === null) lowerer.noLowering("pipeline over this source", arg); positions += 1; + argOrder.push({ list: stages, index: stages.length }); stages.push(source!.closure); srcT = source!.out; continue; @@ -2974,6 +2979,7 @@ function lowerPipelineStages( if (e instanceof StreamArgTypeThrow) lowerer.noLowering("pipeline over this value", arg); throw e; } + argOrder.push({ list: streams, index: streams.length }); streams.push(stream); srcT = stream.type; continue; @@ -3022,6 +3028,7 @@ function lowerPipelineStages( mask |= 1 << stages.length; } positions += 2 ** j; + argOrder.push({ list: stages, index: stages.length }); stages.push(lowered.closure); if (lowered.out.kind === "promise") { dest = 2; @@ -3048,6 +3055,20 @@ function lowerPipelineStages( } helperArgs.push(pipelineBoxHelper(lowerer, destPromise!, loc)); } + const argDecls: IrStmt[] = []; + const firstStage = argOrder.findIndex((a) => a.list === stages); + if (firstStage >= 0 && argOrder.slice(firstStage).some((a) => a.list === streams)) { + for (const a of argOrder) { + const init = a.list[a.index]!; + const local = lowerer.declareHiddenLocal("%pipelineArg", init.type); + argDecls.push({ kind: "varDecl", localId: local.id, init, loc: init.loc }); + a.list[a.index] = varRef(local.id, init.type, init.loc); + } + } + const bindArgs = (call: IrExpr): IrExpr => + argDecls.length === 0 + ? call + : { kind: "seqExpr", stmts: argDecls, result: call, type: call.type, loc }; // SCR_PIPE_* flags: bits 1-2 the destination, bit 5 the signal and abort. const flags = (dest << 1) | (mask !== 0 ? 32 : 0); const head: IrExpr[] = [ @@ -3062,7 +3083,7 @@ function lowerPipelineStages( ]; const last: IrType = dest === 2 ? { kind: "object", className: "%PassThrough" } : streams[streams.length - 1]!.type; - if (mask === 0) return { args: head, last, dest, destPromise, wrap: (call) => call }; + if (mask === 0) return { args: head, last, dest, destPromise, wrap: bindArgs }; // const ac = new AbortController(): its signal goes to the stages, and // finishImpl's ac.abort() runs through the abort closure. const ac = lowerer.declareHiddenLocal("%pipelineAbort", DYN); @@ -3079,20 +3100,21 @@ function lowerPipelineStages( last, dest, destPromise, - wrap: (call) => ({ - kind: "seqExpr", - stmts: [ - { - kind: "varDecl", - localId: ac.id, - init: { kind: "libCall", fn: "fetch.abortControllerNew", args: [], type: DYN, loc }, - loc, - }, - ], - result: call, - type: call.type, - loc, - }), + wrap: (call) => + bindArgs({ + kind: "seqExpr", + stmts: [ + { + kind: "varDecl", + localId: ac.id, + init: { kind: "libCall", fn: "fetch.abortControllerNew", args: [], type: DYN, loc }, + loc, + }, + ], + result: call, + type: call.type, + loc, + }), }; } diff --git a/tests/corpus/7055-cipher-streams.ts b/tests/corpus/7055-cipher-streams.ts index 8befd2339..8f9e0d1ea 100644 --- a/tests/corpus/7055-cipher-streams.ts +++ b/tests/corpus/7055-cipher-streams.ts @@ -8,6 +8,7 @@ import * as crypto from "node:crypto"; import { PassThrough, Readable, + Transform, Writable, addAbortSignal, compose, @@ -65,6 +66,43 @@ async function viaPipeAndForAwait(): Promise { console.log("pipe + for await:", text); } +async function viaPipeIntoTransform(): Promise { + const cipher = crypto.createCipheriv("aes-256-cbc", key, iv); + const upper = new Transform({ + transform(chunk: Buffer, _enc: string, cb: (err: Error | null, data?: Buffer) => void) { + cb(null, Buffer.from(chunk.toString("hex").toUpperCase())); + }, + }); + const out = cipher.pipe(upper); + console.log("pipe answers its destination:", out === upper); + cipher.end("into a transform"); + let hex = ""; + for await (const chunk of out) hex += chunk; + console.log("pipe into transform:", hex); +} + +async function viaPipelineArgumentOrder(): Promise { + const order: string[] = []; + const source = (): string[] => { + order.push("source"); + return ["x", "y"]; + }; + const cipherStage = (): crypto.Cipheriv => { + order.push("cipher"); + return crypto.createCipheriv("aes-256-ctr", key, iv); + }; + const hash = crypto.createHash("sha256"); + await pipelinePromise( + source(), + async function* (src: AsyncIterable) { + for await (const chunk of src) yield chunk.toUpperCase(); + }, + cipherStage(), + hash, + ); + console.log("pipeline argument order:", order.join(" "), hash.digest("hex")); +} + async function viaEvents(): Promise { const cipher = crypto.createCipheriv("aes-128-cbc", Buffer.alloc(16, 1), Buffer.alloc(16, 2)); cipher.setEncoding("hex"); @@ -176,6 +214,8 @@ async function main(): Promise { await viaCallbackPipeline(); await viaPromisePipeline(); await viaPipeAndForAwait(); + await viaPipeIntoTransform(); + await viaPipelineArgumentOrder(); await viaEvents(); await gcm(); }