From 0d016b7554ebdeb9c537ee768a5a89c5fd482704 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 22 Oct 2026 19:57:46 -0500 Subject: [PATCH 01/44] util: runtime format strings, inspect.custom and colors, untyped promisify --- .../limitations/builtin-function-values.mdx | 2 +- .../console-and-process-streams.mdx | 7 + .../src/generated/node-v24-compatibility.json | 12 +- .../generated/node-v24-internal.json | 35 +- internal/compatibility/static-support.json | 8 +- .../ambient/scriptc-node-fallback.d.ts | 57 +- .../compiler/src/backend/llvm/lib-shared.ts | 16 + .../lowering/builtins/promisify-callback.ts | 26 +- .../src/frontend/lowering/call-arguments.ts | 159 ++- .../lowering/expressions/object-literals.ts | 20 +- .../lowering/expressions/property-access.ts | 11 + .../src/frontend/lowering/lower-calls.ts | 30 + .../src/frontend/lowering/lower-classes.ts | 13 + .../src/frontend/lowering/lower-console.ts | 38 +- .../src/frontend/lowering/lower-inspect.ts | 779 +++++++++++- .../src/frontend/lowering/lower-modules.ts | 13 +- .../compiler/src/frontend/lowering/lowerer.ts | 4 + .../src/frontend/lowering/surfaces.ts | 3 +- .../src/frontend/lowering/symbol-fields.ts | 29 + packages/compiler/src/frontend/type-mapper.ts | 24 + packages/compiler/src/ir/builtin-effects.ts | 11 + .../compiler/src/ir/builtin-signatures.ts | 37 +- .../corpus/7520-util-format-runtime.ts.json | 6 + .../corpus/7521-util-format-runtime.cjs.json | 6 + .../corpus/7522-promisify-untyped.cjs.json | 6 + .../corpus/7523-promisify-untyped.ts.json | 6 + .../tests/corpus/7524-inspect-custom.ts.json | 6 + .../tests/corpus/7525-inspect-custom.cjs.json | 6 + .../7526-inspect-options-runtime.ts.json | 6 + .../diagnostics/inspect-custom-fences.ts.json | 6 + packages/runtime/src/scr_console_native.c | 6 +- packages/runtime/src/scr_dyn.h | 4 + packages/runtime/src/scr_inspect.c | 1123 ++++++++++++++++- packages/runtime/src/scr_inspect.h | 71 +- packages/runtime/src/scr_json.c | 18 + packages/runtime/src/scr_symbol.c | 5 + packages/runtime/src/scr_util_callable.c | 166 +-- tests/corpus/7520-util-format-runtime.ts | 110 ++ tests/corpus/7521-util-format-runtime.cjs | 64 + tests/corpus/7522-promisify-untyped.cjs | 64 + tests/corpus/7523-promisify-untyped.ts | 38 + tests/corpus/7524-inspect-custom.ts | 91 ++ tests/corpus/7525-inspect-custom.cjs | 75 ++ tests/corpus/7526-inspect-options-runtime.ts | 62 + tests/diagnostics/inspect-custom-fences.ts | 44 + .../inspect-custom-fences.ts.txt | 49 + .../7520-util-format-runtime.ts.json | 1 + .../7521-util-format-runtime.cjs.json | 1 + .../7522-promisify-untyped.cjs.json | 1 + .../7523-promisify-untyped.ts.json | 1 + .../corpus-costs/7524-inspect-custom.ts.json | 1 + .../corpus-costs/7525-inspect-custom.cjs.json | 1 + .../7526-inspect-options-runtime.ts.json | 1 + tests/harness/errors.test.ts | 47 + 54 files changed, 3101 insertions(+), 325 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7520-util-format-runtime.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7521-util-format-runtime.cjs.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7522-promisify-untyped.cjs.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7523-promisify-untyped.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7524-inspect-custom.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7525-inspect-custom.cjs.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7526-inspect-options-runtime.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/diagnostics/inspect-custom-fences.ts.json create mode 100644 tests/corpus/7520-util-format-runtime.ts create mode 100644 tests/corpus/7521-util-format-runtime.cjs create mode 100644 tests/corpus/7522-promisify-untyped.cjs create mode 100644 tests/corpus/7523-promisify-untyped.ts create mode 100644 tests/corpus/7524-inspect-custom.ts create mode 100644 tests/corpus/7525-inspect-custom.cjs create mode 100644 tests/corpus/7526-inspect-options-runtime.ts create mode 100644 tests/diagnostics/inspect-custom-fences.ts create mode 100644 tests/harness/__snapshots__/inspect-custom-fences.ts.txt create mode 100644 tests/harness/corpus-costs/7520-util-format-runtime.ts.json create mode 100644 tests/harness/corpus-costs/7521-util-format-runtime.cjs.json create mode 100644 tests/harness/corpus-costs/7522-promisify-untyped.cjs.json create mode 100644 tests/harness/corpus-costs/7523-promisify-untyped.ts.json create mode 100644 tests/harness/corpus-costs/7524-inspect-custom.ts.json create mode 100644 tests/harness/corpus-costs/7525-inspect-custom.cjs.json create mode 100644 tests/harness/corpus-costs/7526-inspect-options-runtime.ts.json diff --git a/docs/content/limitations/builtin-function-values.mdx b/docs/content/limitations/builtin-function-values.mdx index 0fa62e558..efd45af2f 100644 --- a/docs/content/limitations/builtin-function-values.mdx +++ b/docs/content/limitations/builtin-function-values.mdx @@ -14,7 +14,7 @@ A class can extend `require("events")` directly. Other immutable aliases of table-backed Node builtins support direct calls, but cannot be passed elsewhere. `util.promisify` supports compile-time conversions of `child_process.execFile`, `child_process.exec`, `setTimeout` and `setImmediate` (to the `timers/promises` functions, as in Node), `stream.finished` and `stream.pipeline` (to the `stream/promises` functions, as in Node), the `zlib` callback one-shots `deflate`, `inflate`, `deflateRaw`, `inflateRaw`, `gzip`, `gunzip` and `unzip` (with or without options), the `crypto` callback functions `randomBytes`, `pbkdf2`, `scrypt`, `hkdf`, `randomFill`, `randomInt` and `generateKey`, `dns.lookup` (settling with `{ address, family }`, or the address array with `all: true`) and `dns.lookupService` (settling with `{ hostname, service }`), and the `fs` callback functions that have an `fs/promises` counterpart (`access`, `appendFile`, `chmod`, `chown`, `copyFile`, `lchown`, `link`, `lstat`, `lutimes`, `mkdir`, `mkdtemp`, `readFile`, `readdir`, `readlink`, `realpath`, `rename`, `rm`, `rmdir`, `stat`, `statfs`, `symlink`, `truncate`, `unlink`, `utimes`, and `writeFile`); calls through the converted function lower as the `fs/promises` call. -`util.promisify` over an untyped (`any`) function returns a native wrapper that follows Node's callback rules and carries the original's `name` and `length`. +`util.promisify` over an untyped function (a value typed `any`, a function whose parameters are untyped, or any other target in JavaScript) returns a native wrapper that follows Node's callback rules and carries the original's `name` and `length`, and throws Node's `ERR_INVALID_ARG_TYPE` for values that are not functions. Such a function with other own properties, `util.promisify.custom` among them, throws `SC2020` where `util.promisify` runs, and the wrapper does not report Node's DEP0174 warning when the original returns a promise. `util.styleText` supports standard formats and aliases, `#RGB` and `#RRGGBB` hex colors, arrays of formats, nested ANSI resets, and explicit color enabling with `{ validateStream: false }`. diff --git a/docs/content/limitations/console-and-process-streams.mdx b/docs/content/limitations/console-and-process-streams.mdx index c62b82f0c..946cb8061 100644 --- a/docs/content/limitations/console-and-process-streams.mdx +++ b/docs/content/limitations/console-and-process-streams.mdx @@ -1,6 +1,13 @@ `console.log`, `info`, and `debug` write to stdout; `error` and `warn` write to stderr. They accept `unknown`, printing strings verbatim and other values through native `util.inspect`. Direct calls support global `console` and `node:console` imports. +Console output, `util.format`, and `util.formatWithOptions` with a format string known only at run time, spread arguments, or an options object format their arguments at run time. +There, `%s` of a function or of a class instance whose class defines `toString` and `%o` of an object throw `SC2020`. +`util.inspect` with `colors`, with option values known only at run time, or in the `inspect(value, showHidden, depth, colors)` form renders the value's untyped view in the runtime with Node's styles. +Colored output of errors, promises, regular expressions, `DataView`, `ArrayBuffer`, weak collections, iterators, and other native objects throws `SC2020`. +Inspect options other than `depth`, `colors`, `customInspect`, `compact` (a number or `false`), `breakLength`, and `sorted` (a boolean) throw `SC2020` when they are set to a value other than Node's default. +`util.inspect` calls `[util.inspect.custom]` methods of classes and objects with Node's depth, options, and `util.inspect` arguments, and `console.dir` and `customInspect: false` skip them. +A class instance with such a method that is held as an untyped value throws `SC2020` when it is inspected, and a class whose subclasses declare the method without declaring it itself is refused. Stored global console values and detached output methods retain identity and accept ordinary arguments. Stored methods do not support format specifiers with substitution arguments. Other stored console APIs, custom `Console` instances, and console mutation are unsupported. diff --git a/docs/src/generated/node-v24-compatibility.json b/docs/src/generated/node-v24-compatibility.json index c341da852..b94ac47be 100644 --- a/docs/src/generated/node-v24-compatibility.json +++ b/docs/src/generated/node-v24-compatibility.json @@ -73027,9 +73027,9 @@ "inherited": true }, "static": { - "status": "not-implemented", - "detail": "No static compiler lowering is registered for this API yet.", - "verification": "registry-gap" + "status": "partial", + "detail": "Implemented for the documented scriptc module-loader subset.", + "verification": "test-backed" } }, { @@ -73050,9 +73050,9 @@ "inherited": true }, "static": { - "status": "not-implemented", - "detail": "No static compiler lowering is registered for this API yet.", - "verification": "registry-gap" + "status": "partial", + "detail": "Implemented for the documented scriptc module-loader subset.", + "verification": "test-backed" } }, { diff --git a/internal/compatibility/generated/node-v24-internal.json b/internal/compatibility/generated/node-v24-internal.json index 9393fcaa8..f95cf3aa7 100644 --- a/internal/compatibility/generated/node-v24-internal.json +++ b/internal/compatibility/generated/node-v24-internal.json @@ -101265,6 +101265,13 @@ "tests/corpus/5835-util-format-integer-specifiers.ts", "tests/corpus/5884-inspect-layout-options.ts", "tests/corpus/6428-promisify-callback-functions.ts", + "tests/corpus/7520-util-format-runtime.ts", + "tests/corpus/7521-util-format-runtime.cjs", + "tests/corpus/7522-promisify-untyped.cjs", + "tests/corpus/7523-promisify-untyped.ts", + "tests/corpus/7524-inspect-custom.ts", + "tests/corpus/7525-inspect-custom.cjs", + "tests/corpus/7526-inspect-options-runtime.ts", "tests/corpus/node24-deep-equality-properties.js", "tests/corpus/node24-deep-equality.cjs", "tests/corpus/node24-deep-equality.ts", @@ -102369,7 +102376,9 @@ "tests": [ "tests/corpus/1636-util-format.ts", "tests/corpus/2440-console-inspect-args.ts", - "tests/corpus/5835-util-format-integer-specifiers.ts" + "tests/corpus/5835-util-format-integer-specifiers.ts", + "tests/corpus/7520-util-format-runtime.ts", + "tests/corpus/7521-util-format-runtime.cjs" ] }, "anchorSource": "exact" @@ -102397,7 +102406,9 @@ "tests": [ "tests/corpus/1636-util-format.ts", "tests/corpus/2440-console-inspect-args.ts", - "tests/corpus/5835-util-format-integer-specifiers.ts" + "tests/corpus/5835-util-format-integer-specifiers.ts", + "tests/corpus/7520-util-format-runtime.ts", + "tests/corpus/7521-util-format-runtime.cjs" ] }, "anchorSource": "exact" @@ -102588,7 +102599,8 @@ "tests/corpus/1632-inspect-records.ts", "tests/corpus/1637-inspect-dyn.ts", "tests/corpus/2485-inspect-circular-refs.ts", - "tests/corpus/5884-inspect-layout-options.ts" + "tests/corpus/5884-inspect-layout-options.ts", + "tests/corpus/7526-inspect-options-runtime.ts" ] }, "anchorSource": "exact" @@ -102611,8 +102623,11 @@ "inherited": true }, "static": { - "status": "not-implemented", - "evidence": "compiler-unmatched:util" + "status": "partial", + "evidence": "compiler-feature:util.util.inspect", + "tests": [ + "tests/corpus/7526-inspect-options-runtime.ts" + ] }, "anchorSource": "exact" }, @@ -102634,8 +102649,12 @@ "inherited": true }, "static": { - "status": "not-implemented", - "evidence": "compiler-unmatched:util" + "status": "partial", + "evidence": "compiler-feature:util.custom", + "tests": [ + "tests/corpus/7524-inspect-custom.ts", + "tests/corpus/7525-inspect-custom.cjs" + ] }, "anchorSource": "ancestor" }, @@ -102952,6 +102971,8 @@ "tests/fixtures/node-types/promisify-dns.ts", "tests/corpus/6428-promisify-callback-functions.ts", "tests/fixtures/node-types/promisify-callback-functions.ts", + "tests/corpus/7522-promisify-untyped.cjs", + "tests/corpus/7523-promisify-untyped.ts", "tests/harness/project-config.test.ts" ] }, diff --git a/internal/compatibility/static-support.json b/internal/compatibility/static-support.json index 06aad26cf..b5211c967 100644 --- a/internal/compatibility/static-support.json +++ b/internal/compatibility/static-support.json @@ -244,9 +244,11 @@ {"chapter": "worker_threads", "symbols": ["resourceLimits"], "status": "partial", "evidence": ["tests/corpus/5806-worker-resource-limits.ts"]}, {"chapter": "worker_threads", "symbols": ["BroadcastChannel extends EventTarget", "broadcastChannel.close", "broadcastChannel.postMessage", "broadcastChannel.ref", "broadcastChannel.unref", "onmessage"], "status": "partial", "evidence": ["tests/corpus/5800-broadcast-channel.ts", "tests/corpus/5801-broadcast-channel.cjs", "tests/corpus/5802-broadcast-channel-workers.ts"]}, {"chapter": "util", "symbols": ["util.types.isAnyArrayBuffer", "util.types.isArrayBuffer", "util.types.isArrayBufferView", "util.types.isDataView", "util.types.isTypedArray", "util.types.isUint8Array", "util.types.isUint8ClampedArray", "util.types.isUint16Array", "util.types.isUint32Array", "util.types.isInt8Array", "util.types.isInt16Array", "util.types.isInt32Array", "util.types.isFloat32Array", "util.types.isFloat64Array", "util.types.isMap", "util.types.isSet", "util.types.isWeakMap", "util.types.isWeakSet", "util.types.isRegExp", "util.types.isNativeError", "util.types.isPromise", "util.types.isProxy", "util.types.isAsyncFunction", "util.types.isGeneratorFunction", "util.types.isGeneratorObject"], "status": "partial", "evidence": ["tests/corpus/node24-util-types.ts", "tests/corpus/node24-util-types.js", "tests/corpus/node24-util-types.cjs", "tests/fixtures/node-types/util-types.ts"]}, - {"chapter": "util", "symbols": ["util.format", "util.formatWithOptions"], "status": "partial", "evidence": ["tests/corpus/1636-util-format.ts", "tests/corpus/2440-console-inspect-args.ts", "tests/corpus/5835-util-format-integer-specifiers.ts"]}, + {"chapter": "util", "symbols": ["util.format", "util.formatWithOptions"], "status": "partial", "evidence": ["tests/corpus/1636-util-format.ts", "tests/corpus/2440-console-inspect-args.ts", "tests/corpus/5835-util-format-integer-specifiers.ts", "tests/corpus/7520-util-format-runtime.ts", "tests/corpus/7521-util-format-runtime.cjs"]}, {"chapter": "util", "symbols": ["util.types.isAnyArrayBuffer", "util.types.isArrayBuffer", "util.types.isArrayBufferView", "util.types.isDataView", "util.types.isTypedArray", "util.types.isUint8Array", "util.types.isUint8ClampedArray", "util.types.isUint16Array", "util.types.isUint32Array", "util.types.isInt8Array", "util.types.isInt16Array", "util.types.isInt32Array", "util.types.isFloat32Array", "util.types.isFloat64Array", "util.types.isMap", "util.types.isSet", "util.types.isWeakMap", "util.types.isWeakSet", "util.types.isRegExp", "util.types.isNativeError", "util.types.isPromise", "util.types.isProxy", "util.types.isAsyncFunction", "util.types.isGeneratorFunction", "util.types.isGeneratorObject", "util.types.isDate", "util.types.isSharedArrayBuffer", "util.types.isMapIterator", "util.types.isSetIterator", "util.types.isKeyObject", "util.types.isNumberObject", "util.types.isStringObject", "util.types.isBooleanObject", "util.types.isBigIntObject", "util.types.isSymbolObject", "util.types.isBoxedPrimitive", "util.types.isBigInt64Array", "util.types.isBigUint64Array", "util.types.isFloat16Array", "util.types.isCryptoKey", "util.types.isExternal"], "status": "partial", "evidence": ["tests/corpus/5831-util-types-brands.ts", "tests/corpus/5832-util-types-brands-dynamic.js", "tests/corpus/node24-util-types.ts", "tests/corpus/node24-util-types.js", "tests/corpus/node24-util-types.cjs", "tests/fixtures/node-types/util-types.ts"]}, - { "chapter": "util", "signatures": ["`util.inspect(object[, options])`"], "status": "partial", "evidence": ["tests/corpus/1630-inspect-scalars.ts", "tests/corpus/1632-inspect-records.ts", "tests/corpus/1637-inspect-dyn.ts", "tests/corpus/2485-inspect-circular-refs.ts", "tests/corpus/5884-inspect-layout-options.ts"] }, + { "chapter": "util", "signatures": ["`util.inspect(object[, options])`"], "status": "partial", "evidence": ["tests/corpus/1630-inspect-scalars.ts", "tests/corpus/1632-inspect-records.ts", "tests/corpus/1637-inspect-dyn.ts", "tests/corpus/2485-inspect-circular-refs.ts", "tests/corpus/5884-inspect-layout-options.ts", "tests/corpus/7526-inspect-options-runtime.ts"] }, + { "chapter": "util", "signatures": ["`util.inspect(object[, showHidden[, depth[, colors]]])`"], "status": "partial", "evidence": ["tests/corpus/7526-inspect-options-runtime.ts"] }, + { "chapter": "util", "signatures": ["`custom` Type: {symbol} that can be used to declare custom inspect functions."], "status": "partial", "evidence": ["tests/corpus/7524-inspect-custom.ts", "tests/corpus/7525-inspect-custom.cjs"] }, { "chapter": "util", "symbols": ["util.styleText"], "status": "partial", "evidence": ["tests/corpus/node24-style-text.ts", "tests/corpus/node24-style-text-validation.js", "tests/corpus/node24-style-text-environment.js", "tests/corpus/node24-style-text.cjs", "tests/fixtures/node-types/style-text.ts"] }, { "chapter": "util", "symbols": ["util.isDeepStrictEqual"], "status": "partial", "evidence": ["tests/corpus/node24-deep-equality.ts", "tests/corpus/node24-deep-equality.cjs", "tests/corpus/node24-deep-equality-properties.js", "tests/fixtures/node-types/deep-equality.ts", "packages/compiler/test/deep-equality.test.ts"] }, { "chapter": "deprecations", "symbols": ["util.toUSVString"], "status": "not-applicable" }, @@ -400,7 +402,7 @@ { "chapter": "test", "parent": "Class: `TestContext`", "symbols": ["context.before", "context.after", "context.beforeEach", "context.afterEach", "context.plan", "context.fullName"], "status": "partial", "evidence": ["tests/fixtures/node-test/cases/ctx-hooks.ts", "tests/fixtures/node-test/cases/ctx-plan.ts", "tests/harness/node-test.test.ts"] }, { "chapter": "test", "symbols": ["getTestContext"], "status": "partial", "evidence": ["tests/fixtures/node-test/cases/get-test-context.ts", "tests/fixtures/node-test/cases/get-test-context-cancelled.ts", "tests/harness/node-test.test.ts"] }, { "chapter": "test", "symbols": ["describe.skip", "describe.todo", "describe.only", "suite.skip", "suite.todo", "suite.only", "it.skip", "it.todo", "it.only"], "status": "partial", "evidence": ["tests/fixtures/node-test/cases/suite-variants.ts", "tests/fixtures/node-test/cases/only-nested.ts", "tests/fixtures/node-test/cases/todo-suites.ts", "tests/harness/node-test.test.ts"] }, - { "chapter": "util", "signatures": ["`util.promisify(original)`"], "status": "partial", "evidence": ["tests/corpus/1473-promisify-execfile.ts", "tests/corpus/1573-promisify-execfile-env-spread.ts", "tests/corpus/2912-promisify-fs-readfile.ts", "tests/corpus/5625-promisify-exec.ts", "tests/corpus/5626-promisify-forms.ts", "tests/fixtures/node-types/promisify-fs.ts", "tests/corpus/5628-promisify-timers.ts", "tests/fixtures/node-types/promisify-timers.ts", "tests/corpus/5629-promisify-stream.ts", "tests/corpus/5630-promisify-zlib.ts", "tests/fixtures/node-types/promisify-zlib.ts", "tests/corpus/5631-promisify-crypto.ts", "tests/fixtures/node-types/promisify-crypto.ts", "tests/corpus/5632-promisify-dns.ts", "tests/fixtures/node-types/promisify-dns.ts", "tests/corpus/6428-promisify-callback-functions.ts", "tests/fixtures/node-types/promisify-callback-functions.ts", "tests/harness/project-config.test.ts"] }, + { "chapter": "util", "signatures": ["`util.promisify(original)`"], "status": "partial", "evidence": ["tests/corpus/1473-promisify-execfile.ts", "tests/corpus/1573-promisify-execfile-env-spread.ts", "tests/corpus/2912-promisify-fs-readfile.ts", "tests/corpus/5625-promisify-exec.ts", "tests/corpus/5626-promisify-forms.ts", "tests/fixtures/node-types/promisify-fs.ts", "tests/corpus/5628-promisify-timers.ts", "tests/fixtures/node-types/promisify-timers.ts", "tests/corpus/5629-promisify-stream.ts", "tests/corpus/5630-promisify-zlib.ts", "tests/fixtures/node-types/promisify-zlib.ts", "tests/corpus/5631-promisify-crypto.ts", "tests/fixtures/node-types/promisify-crypto.ts", "tests/corpus/5632-promisify-dns.ts", "tests/fixtures/node-types/promisify-dns.ts", "tests/corpus/6428-promisify-callback-functions.ts", "tests/fixtures/node-types/promisify-callback-functions.ts", "tests/corpus/7522-promisify-untyped.cjs", "tests/corpus/7523-promisify-untyped.ts", "tests/harness/project-config.test.ts"] }, { "chapter": "typescript", "symbols": ["type"], "status": "supported", "evidence": ["tests/corpus/2702-type-import-link-static/main.ts", "tests/corpus/2703-type-import-link-static-reexport/main.ts", "tests/corpus/2704-type-import-link-static-success/main.ts", "tests/corpus/2707-type-import-link-static-default-reexport/main.ts", "tests/harness/errors.test.ts"] }, { "chapter": "zlib", "signatures": ["Class: `zlib.Deflate`", "Class: `zlib.Inflate`", "Class: `zlib.Gzip`", "Class: `zlib.Gunzip`", "Class: `zlib.DeflateRaw`", "Class: `zlib.InflateRaw`", "Class: `zlib.Unzip`", "Class: `zlib.ZlibBase`", "`zlib.close([callback])`", "`zlib.flush([kind, ]callback)`", "`zlib.params(level, strategy, callback)`", "`zlib.reset()`", "`bytesWritten` Type: {number}", "Class: `Options`"], "status": "partial", "evidence": ["tests/corpus/5370-zlib-stream-roundtrip.ts", "tests/corpus/5371-zlib-stream-chunking.cjs", "tests/corpus/5372-zlib-stream-errors.cjs", "tests/corpus/5373-zlib-stream-options.cjs", "tests/corpus/5374-zlib-stream-flush-params.ts", "tests/corpus/5375-zlib-stream-promises.ts", "tests/corpus/5377-zlib-stream-backpressure.cjs"] }, { "chapter": "zlib", "signatures": ["`zlib.constants`"], "status": "partial", "evidence": ["tests/corpus/5376-zlib-constants.cjs", "tests/corpus/5374-zlib-stream-flush-params.ts"] }, diff --git a/packages/compiler/ambient/scriptc-node-fallback.d.ts b/packages/compiler/ambient/scriptc-node-fallback.d.ts index b195c3fbb..80054b4fa 100644 --- a/packages/compiler/ambient/scriptc-node-fallback.d.ts +++ b/packages/compiler/ambient/scriptc-node-fallback.d.ts @@ -3838,7 +3838,13 @@ declare module "util" { export type TextDecoder = globalThis.TextDecoder; export var TextEncoder: typeof globalThis.TextEncoder; export var TextDecoder: typeof globalThis.TextDecoder; - /* stream.pipeline's overload comes first: its __promisify__ member keeps + /* An untyped function: an `any` value, or a function whose parameters + * are untyped (JavaScript's `(value, callback) => ...`), the only + * parameters that accept this object type. The promisified function is + * untyped too. It comes first because such functions match every other + * overload. */ + export function promisify(fn: (...args: { readonly "scriptc:untyped": true }[]) => unknown): any; + /* stream.pipeline's overload comes next: its __promisify__ member keeps * every other target out of it. */ export function promisify( fn: typeof import("node:stream").pipeline, @@ -3959,15 +3965,56 @@ declare module "util" { ): (address: string, port: number) => Promise<{ hostname: string; service: string }>; /* Declared surface without full lowering (the type-level tolerance * story: guarded/diagnostic-path code must TYPECHECK; a reached use - * without a lowering fences at its site). inspect's options carry the - * documented knobs through an index signature. */ + * without a lowering fences at its site). */ + export interface InspectOptions { + showHidden?: boolean | undefined; + depth?: number | null | undefined; + colors?: boolean | undefined; + customInspect?: boolean | undefined; + showProxy?: boolean | undefined; + maxArrayLength?: number | null | undefined; + maxStringLength?: number | null | undefined; + breakLength?: number | undefined; + compact?: boolean | number | undefined; + sorted?: boolean | ((a: string, b: string) => number) | undefined; + getters?: "get" | "set" | boolean | undefined; + numericSeparator?: boolean | undefined; + } + export type Style = + | "special" + | "number" + | "bigint" + | "boolean" + | "undefined" + | "null" + | "string" + | "symbol" + | "date" + | "regexp" + | "module"; + export interface InspectOptionsStylized extends InspectOptions { + stylize(text: string, styleType: Style): string; + } + export type CustomInspectFunction = ( + depth: number, + options: InspectOptionsStylized, + inspect: typeof import("node:util").inspect, + ) => any; export function inspect( object: unknown, - options?: { depth?: number | null; colors?: boolean; [k: string]: unknown }, + showHidden?: boolean, + depth?: number | null, + color?: boolean, ): string; + export function inspect(object: unknown, options?: InspectOptions): string; + export namespace inspect { + /* The key of an object's custom inspect function: + * Symbol.for('nodejs.util.inspect.custom'). */ + const custom: unique symbol; + } export function format(format?: unknown, ...args: unknown[]): string; export function formatWithOptions( - inspectOptions: { depth?: number | null; colors?: boolean; [k: string]: unknown }, + inspectOptions: InspectOptions, format?: unknown, ...args: unknown[] ): string; diff --git a/packages/compiler/src/backend/llvm/lib-shared.ts b/packages/compiler/src/backend/llvm/lib-shared.ts index 0b6041034..b02cf44f3 100644 --- a/packages/compiler/src/backend/llvm/lib-shared.ts +++ b/packages/compiler/src/backend/llvm/lib-shared.ts @@ -1034,6 +1034,22 @@ export const LIB_FN_SYMS: Record = { "dyn.generatorThis": "scr_gen_receiver", "insp.dyn": "scr_insp_dyn", "insp.dynS": "scr_insp_dyn_s", + "insp.format": "scr_insp_format_pack", + "insp.formatPrimitives": "scr_insp_format_primitives", + "insp.formatWithOptions": "scr_insp_format_with_options", + "insp.inspect": "scr_insp_inspect", + "insp.customEnabled": "scr_insp_custom_enabled", + "insp.registerCustom": "scr_insp_register_custom", + "insp.customInspect": "scr_insp_custom_inspect", + "insp.depthKind": "scr_insp_depth_kind", + "insp.customDepth": "scr_insp_custom_depth", + "insp.customOptions": "scr_insp_custom_options", + "insp.inspectFunction": "scr_insp_inspect_function", + "insp.customText": "scr_insp_custom_text", + "insp.customIsSelf": "scr_insp_custom_is_self", + "insp.customResult": "scr_insp_custom_result", + "insp.formatSpecifies": "scr_insp_format_specifies", + "insp.formatSpecifiesDyn": "scr_insp_format_specifies_dyn", "fs.readFileSyncDyn": "scr_fs_read_file_sync_dyn", "fs.readFileEncoded": "scr_fs_read_file_encoded", "fs.readFdEncoded": "scr_fs_read_fd_encoded", diff --git a/packages/compiler/src/frontend/lowering/builtins/promisify-callback.ts b/packages/compiler/src/frontend/lowering/builtins/promisify-callback.ts index b4dbbf513..d3378c8ec 100644 --- a/packages/compiler/src/frontend/lowering/builtins/promisify-callback.ts +++ b/packages/compiler/src/frontend/lowering/builtins/promisify-callback.ts @@ -1,7 +1,8 @@ import { keyedHelperName } from "../stable-names.js"; import * as ts from "../../ts7/adapter.js"; import { type Lowerer } from "../lowerer.js"; -import { varRef } from "../../../ir/build.js"; +import { dynUndefinedExpr, varRef } from "../../../ir/build.js"; +import { isJsSourceFile } from "../../program.js"; import { BOOL, DYN, @@ -92,12 +93,22 @@ function callbackShape( return { fnT, resultT, args, callbackT, callbackParamT: callbackParamT!, inner, error }; } -/** Whether `promisify(fn)`'s one argument is typed `any`: the function is - * only known at run time, so the promisified function is built there. */ +/** Whether `promisify(fn)`'s one argument is only known at run time, so + * the promisified function is built there: a value typed `any`, a function + * with untyped parameters (the declarations type that call `any`), or any + * argument in JavaScript, whose inferred types do not describe the + * function Node receives (it validates the argument and reads its own + * properties). */ export function untypedPromisifyArgument(lowerer: Lowerer, call: ts.CallExpression): boolean { - if (call.questionDotToken || call.arguments.nodes.length !== 1) return false; - const argNode = call.arguments.nodes[0]!; - return !ts.isSpreadElement(argNode) && (lowerer.typeOf(argNode).flags & ts.TypeFlags.Any) !== 0; + if (call.questionDotToken || call.arguments.nodes.length > 1) return false; + const argNode = call.arguments.nodes[0]; + if (argNode !== undefined && ts.isSpreadElement(argNode)) return false; + if (isJsSourceFile(ts.getSourceFileOfNode(call))) return true; + if (argNode === undefined) return false; + return ( + (lowerer.typeOf(argNode).flags & ts.TypeFlags.Any) !== 0 || + (lowerer.typeOf(call).flags & ts.TypeFlags.Any) !== 0 + ); } /** The callback shape of a `promisify(fn)` call, or null. */ @@ -144,10 +155,11 @@ export function lowerCallbackPromisify( if (shape === null) { // An untyped (`any`) function: Node's promisify at run time. if (!untypedPromisifyArgument(lowerer, call)) return null; + const argNode = call.arguments.nodes[0]; return { kind: "libCall", fn: "util.promisifyDyn", - args: [lowerer.lowerExprExpecting(call.arguments.nodes[0]!, DYN)], + args: [argNode ? lowerer.lowerExprExpecting(argNode, DYN) : dynUndefinedExpr(loc)], type: DYN, loc, }; diff --git a/packages/compiler/src/frontend/lowering/call-arguments.ts b/packages/compiler/src/frontend/lowering/call-arguments.ts index 060ffe9d2..7dc302365 100644 --- a/packages/compiler/src/frontend/lowering/call-arguments.ts +++ b/packages/compiler/src/frontend/lowering/call-arguments.ts @@ -56,6 +56,95 @@ function emptyTupleSpread(lowerer: Lowerer, node: ts.Expression): boolean { return restTupleElementTypes(lowerer.typeOf(operand), lowerer.typeCtx)?.length === 0; } +/** A call's arguments as one checked-dynamic array, built in source order + * into a fresh hidden local: each argument converts to dyn, and a spread + * contributes the values its iteration yields (holes as undefined). + * `lowerSpread` lowers a spread's operand; the default lowers it here. */ +export function dynArgumentPack( + lowerer: Lowerer, + argNodes: readonly ts.Expression[], + loc: SrcLoc, + options: { + /** Pre-lowered values pushed before the arguments. */ + leading?: readonly IrExpr[] | undefined; + blame?: ts.Node | undefined; + lowerSpread?: (arg: ts.SpreadElement) => IrExpr; + } = {}, +): { stmts: IrStmt[]; pack: () => IrExpr } { + const blame = options.blame ?? argNodes[0]!; + const lowerSpread = + options.lowerSpread ?? ((arg: ts.SpreadElement) => lowerer.lowerExpr(arg.expression)); + const pack = lowerer.declareHiddenLocal("%spreadArgs", DYN); + const ref = (): IrExpr => ({ kind: "varRef", localId: pack.id, type: DYN, loc }); + const stmts: IrStmt[] = [ + { + kind: "varDecl", + localId: pack.id, + init: { kind: "dynArrLit", elems: [], type: DYN, loc }, + loc, + }, + ]; + const push = (value: IrExpr, fn: "dyn.packPush" | "dyn.packPushSpreadIter") => { + stmts.push({ + kind: "exprStmt", + expr: { kind: "libCall", fn, args: [ref(), value], type: VOID, loc: value.loc }, + loc: value.loc, + }); + }; + for (const value of options.leading ?? []) + push(lowerer.coerceInto(blame, value, DYN), "dyn.packPush"); + const spreads = argNodes.filter(ts.isSpreadElement).length; + argNodes.forEach((arg, index) => { + if (!ts.isSpreadElement(arg)) { + push(lowerer.lowerExprExpecting(arg, DYN), "dyn.packPush"); + return; + } + let value = lowerSpread(arg); + if (value.type.kind === "set") { + value = { + kind: "setIntrinsic", + method: "toArray", + receiver: value, + args: [], + type: arrayOf(value.type.elem), + loc: value.loc, + }; + } else if (value.type.kind === "object") { + value = lowerer.classIteratorDrainCall(value, locOf(arg)) ?? value; + } + if (value.type.kind === "array") { + // Argument iteration materializes holes as present undefined + // before conversion to the dense checked-dynamic argument pack. + value = { + kind: "arrayLit", + elems: [value], + spreads: [0], + type: value.type, + loc: value.loc, + }; + } + value = checkedIterableSpread(lowerer, value, ts.getTextOfNode(arg.expression), loc); + if (spreads === 1 && index === argNodes.length - 1) { + stmts.push({ + kind: "exprStmt", + expr: { + kind: "libCall", + fn: "dyn.packPushSpread", + args: [ + ref(), + value, + { kind: "strLit", value: ts.getTextOfNode(arg.expression), type: STRING, loc }, + ], + type: VOID, + loc, + }, + loc, + }); + } else push(value, "dyn.packPushSpreadIter"); + }); + return { stmts, pack: ref }; +} + export function completeArgs( lowerer: Lowerer, argNodes: readonly ts.Expression[], @@ -96,72 +185,10 @@ export function completeArgs( // Runtime-length spreads determine the complete argument list before // any parameter default runs. Build it once in source order, then // extract the fixed native ABI slots; missing elements are undefined. - const pack = lowerer.declareHiddenLocal("%spreadArgs", DYN); - const ref = (): IrExpr => ({ kind: "varRef", localId: pack.id, type: DYN, loc }); - const stmts: IrStmt[] = [ - { - kind: "varDecl", - localId: pack.id, - init: { kind: "dynArrLit", elems: [], type: DYN, loc }, - loc, - }, - ]; - const push = (value: IrExpr, fn: "dyn.packPush" | "dyn.packPushSpreadIter") => { - stmts.push({ - kind: "exprStmt", - expr: { kind: "libCall", fn, args: [ref(), value], type: VOID, loc: value.loc }, - loc: value.loc, - }); - }; - for (const value of leading ?? []) push(lowerer.coerceInto(blame, value, DYN), "dyn.packPush"); - const spreads = argNodes.filter(ts.isSpreadElement).length; - argNodes.forEach((arg, index) => { - if (!ts.isSpreadElement(arg)) { - push(lowerer.lowerExprExpecting(arg, DYN), "dyn.packPush"); - return; - } - let value = lowerSpread(arg); - if (value.type.kind === "set") { - value = { - kind: "setIntrinsic", - method: "toArray", - receiver: value, - args: [], - type: arrayOf(value.type.elem), - loc: value.loc, - }; - } else if (value.type.kind === "object") { - value = lowerer.classIteratorDrainCall(value, locOf(arg)) ?? value; - } - if (value.type.kind === "array") { - // Argument iteration materializes holes as present undefined - // before conversion to the dense checked-dynamic argument pack. - value = { - kind: "arrayLit", - elems: [value], - spreads: [0], - type: value.type, - loc: value.loc, - }; - } - value = checkedIterableSpread(lowerer, value, ts.getTextOfNode(arg.expression), loc); - if (spreads === 1 && index === argNodes.length - 1) { - stmts.push({ - kind: "exprStmt", - expr: { - kind: "libCall", - fn: "dyn.packPushSpread", - args: [ - ref(), - value, - { kind: "strLit", value: ts.getTextOfNode(arg.expression), type: STRING, loc }, - ], - type: VOID, - loc, - }, - loc, - }); - } else push(value, "dyn.packPushSpreadIter"); + const { stmts, pack: ref } = dynArgumentPack(lowerer, argNodes, loc, { + leading, + blame, + lowerSpread, }); const args = shapes.map((shape, index): IrExpr => { if (shape.mode === "arguments") return ref(); diff --git a/packages/compiler/src/frontend/lowering/expressions/object-literals.ts b/packages/compiler/src/frontend/lowering/expressions/object-literals.ts index 4d04be30a..43322246c 100644 --- a/packages/compiler/src/frontend/lowering/expressions/object-literals.ts +++ b/packages/compiler/src/frontend/lowering/expressions/object-literals.ts @@ -33,7 +33,11 @@ import { } from "../../type-mapper.js"; import { isSafeToRepeat } from "./evaluation-safety.js"; import { tryLowerExpression } from "./try-lower-expression.js"; -import { fenceSymbolFieldCopy } from "../symbol-fields.js"; +import { + classSymbolKeyOf, + fenceSymbolFieldCopy, + symbolFieldDisplayName, +} from "../symbol-fields.js"; import { planUnionRetag } from "../union-retag.js"; import { jsHoldsDynObject, runtimeOptionalStorageOperand } from "../lower-exprs.js"; import { builtinGlobalValue } from "./property-access.js"; @@ -194,7 +198,19 @@ function propNameText(lowerer: Lowerer, name: ts.PropertyName): string { * (NamedEvaluation): an identifier, string or numeric key, or a computed * key that folds to one string. Null for other keys. */ function propertyFunctionName(lowerer: Lowerer, name: ts.PropertyName): string | null { - if (ts.isComputedPropertyName(name)) return literalComputedKey(lowerer, name); + if (ts.isComputedPropertyName(name)) { + const literal = literalComputedKey(lowerer, name); + if (literal !== null) return literal; + // A symbol key names the function `[description]`. + let key = name.expression; + while (ts.isParenthesizedExpression(key)) key = key.expression; + const symbol = classSymbolKeyOf(lowerer, key); + if (symbol) return `[${symbolFieldDisplayName(symbol.fieldName).slice("Symbol(".length, -1)}]`; + const wellKnown = ts.isPropertyAccessExpression(key) + ? lowerer.stdlibGlobalMember(key, "Symbol") + : null; + return wellKnown ? `[Symbol.${wellKnown}]` : null; + } if (ts.isNumericLiteral(name)) return String(Number(name.text)); if (ts.isIdentifier(name) || ts.isStringLiteral(name)) return name.text; return null; diff --git a/packages/compiler/src/frontend/lowering/expressions/property-access.ts b/packages/compiler/src/frontend/lowering/expressions/property-access.ts index 5df3c7fe0..9157c551d 100644 --- a/packages/compiler/src/frontend/lowering/expressions/property-access.ts +++ b/packages/compiler/src/frontend/lowering/expressions/property-access.ts @@ -5,6 +5,7 @@ import * as ts from "../../ts7/adapter.js"; import { lowerHistogramProperty } from "../builtins/performance.js"; import { staticFunctionNameRead } from "../static-function-names.js"; import { typedArgumentsArity } from "../typed-arguments.js"; +import { INSPECT_CUSTOM_DESCRIPTION, isInspectCustomAccess } from "../symbol-fields.js"; import { isThisParameter } from "../call-signatures.js"; import { checkedPrivateValue, @@ -310,6 +311,16 @@ export function lowerPropertyAccessExpr( const navigator = lowerNavigatorMember(lowerer, expr, NODE_COMPAT_VERSION); if (navigator) return navigator; } + // util.inspect.custom: Symbol.for('nodejs.util.inspect.custom'). + if (isInspectCustomAccess(lowerer, expr)) { + return { + kind: "libCall", + fn: "sym.for", + args: [{ kind: "strLit", value: INSPECT_CUSTOM_DESCRIPTION, type: STRING, loc }], + type: SYMBOL_T, + loc, + }; + } // A dgram socket's method read as a value in JavaScript (`emit: // socket.emit.bind(socket)`): the socket crosses as its checked-dynamic // handle, whose methods are function values that dispatch on their diff --git a/packages/compiler/src/frontend/lowering/lower-calls.ts b/packages/compiler/src/frontend/lowering/lower-calls.ts index c0406e77b..711cd7379 100644 --- a/packages/compiler/src/frontend/lowering/lower-calls.ts +++ b/packages/compiler/src/frontend/lowering/lower-calls.ts @@ -267,6 +267,28 @@ import { codeGenerationCalleeOf, lowerCodeGenerationCall } from "./code-generati const ZLIB_STREAM_METHODS: ReadonlySet = new Set(["flush", "params", "close", "reset"]); +/** The node:util member a `(0, member)` callee names, or null. */ +function indirectUtilCalleeOf( + lowerer: Lowerer, + callee: ts.Expression, +): { module: string; member: string } | null { + let e = callee; + while (ts.isParenthesizedExpression(e)) e = e.expression; + if ( + !ts.isBinaryExpression(e) || + e.operatorToken.kind !== ts.SyntaxKind.CommaToken || + !ts.isNumericLiteral(e.left) + ) + return null; + const target = e.right; + const bi = ts.isIdentifier(target) + ? lowerer.builtinImportOf(target) + : ts.isPropertyAccessExpression(target) && !target.questionDotToken + ? lowerer.builtinMemberOf(target) + : null; + return bi?.module === "util" ? bi : null; +} + export function lowerCall(lowerer: Lowerer, expr: ts.CallExpression): IrExpr { const loc = locOf(expr); // Object's checker return type is `any`. Claim known object inputs @@ -295,6 +317,14 @@ export function lowerCall(lowerer: Lowerer, expr: ts.CallExpression): IrExpr { const codeGeneration = codeGenerationCalleeOf(lowerer, expr.expression); if (codeGeneration !== null) return lowerCodeGenerationCall(lowerer, expr, codeGeneration); } + // `(0, util_1.inspect)(value)`: TypeScript's CommonJS output calls a + // named import through a comma expression, which leaves the call's + // receiver undefined. node:util's functions do not read their receiver, + // so the call lowers as the direct one. + { + const util = indirectUtilCalleeOf(lowerer, expr.expression); + if (util) return lowerBuiltinCall(lowerer, expr, util, loc); + } // A package shim's runtime intrinsic (a bodiless `declare function` // named scriptcUnixDgram* in a shipped shim): a direct runtime call. if (ts.isIdentifier(expr.expression) && isShimIntrinsicName(expr.expression.text)) { diff --git a/packages/compiler/src/frontend/lowering/lower-classes.ts b/packages/compiler/src/frontend/lowering/lower-classes.ts index b23697cab..d4875c9df 100644 --- a/packages/compiler/src/frontend/lowering/lower-classes.ts +++ b/packages/compiler/src/frontend/lowering/lower-classes.ts @@ -155,6 +155,7 @@ import { } from "./lower-event-emitter.js"; import { declSymbolOf } from "./lower-modules.js"; import { + INSPECT_CUSTOM_MEMBER, classSymbolKeyOf, classSymbolKeyOfSymbol, symbolFieldDisplayName, @@ -3208,6 +3209,15 @@ export function collectClassShapeInner( symbolMethods.set(key.identity, mName); } } + // Error values held as untyped values render through their error + // encoding, which keeps no compiled methods to call. + if (mName === INSPECT_CUSTOM_MEMBER && errorRooted) { + lowerer.unsupported( + "SC1090", + member.name, + "[util.inspect.custom] methods on Error subclasses", + ); + } // Generator calls use their spawn wrapper. Public methods are // direct calls only when no subclass overrides the method; // override chains involving generators fence below. @@ -4422,6 +4432,9 @@ export function collectClassShapeInner( ...(methods.size > 0 ? { methods: [...methods.keys()] } : {}), ...((): { prototypeNames?: string[] } => { const names = classPrototypeNames(decl); + // A [util.inspect.custom] method, by its reserved slot name: the + // checked-dynamic inspect walk refuses instances that have one. + if (methods.has(INSPECT_CUSTOM_MEMBER)) names.push(INSPECT_CUSTOM_MEMBER); return names.length > 0 ? { prototypeNames: names } : {}; })(), ...(abstractClass ? { abstract: true as const } : {}), diff --git a/packages/compiler/src/frontend/lowering/lower-console.ts b/packages/compiler/src/frontend/lowering/lower-console.ts index eec308342..db4d39e70 100644 --- a/packages/compiler/src/frontend/lowering/lower-console.ts +++ b/packages/compiler/src/frontend/lowering/lower-console.ts @@ -14,7 +14,14 @@ import { boolLit, dynUndefinedExpr, numLit, strLit, varRef } from "../../ir/buil import { locOf } from "../program.js"; import { type ConsoleMethodMember } from "./builtins/console.js"; import { jsFuncNameOf, type Lowerer } from "./lowerer.js"; -import { lowerConsoleInspectArg, lowerFormatCall, lowerInspectValue } from "./lower-inspect.js"; +import { + heldFormatArgs, + lowerConsoleInspectArg, + lowerFormatCall, + lowerInspectValue, + lowerRuntimeFormat, + runtimeFormatString, +} from "./lower-inspect.js"; import { streamSidesOf } from "./lower-stream.js"; /** node:console Console instances: a native handle (the checked-dynamic @@ -357,9 +364,32 @@ export function lowerConsoleOutput( ) { return consoleWrite(stdout, [lowerFormatCall(lowerer, expr, loc, false)], loc); } - const args = expr.arguments.nodes.map((a) => - lowerConsoleArg(lowerer, a, lowerer.lowerExpr(a), surface, loc), - ); + const nodes = expr.arguments.nodes; + // Spread arguments and format strings that may hold a substitution at + // run time go through the runtime formatter. A string first argument + // without a '%' keeps the space-joined write below. + if (nodes.some(ts.isSpreadElement)) + return consoleWrite(stdout, [lowerRuntimeFormat(lowerer, nodes, loc)], loc); + const runtime = + nodes.length > 1 && !isStringLiteralLike(first) ? runtimeFormatString(lowerer, first!) : null; + if (runtime === "other") + return consoleWrite(stdout, [lowerRuntimeFormat(lowerer, nodes, loc)], loc); + if (runtime !== null) { + const held = heldFormatArgs(lowerer, nodes, loc); + if (!held.joinable) return seq(held.stmts, consoleWrite(stdout, [held.format()], loc), loc); + const joined = held.args.map((arg) => + lowerConsoleArg(lowerer, arg.node, arg.ref(), surface, loc), + ); + const write: IrStmt = { + kind: "if", + cond: held.specifies, + then: [{ kind: "exprStmt", expr: consoleWrite(stdout, [held.format()], loc), loc }], + else_: [{ kind: "exprStmt", expr: consoleWrite(stdout, joined, loc), loc }], + loc, + }; + return seq([...held.stmts, write], noop(loc), loc); + } + const args = nodes.map((a) => lowerConsoleArg(lowerer, a, lowerer.lowerExpr(a), surface, loc)); return consoleWrite(stdout, args, loc); } diff --git a/packages/compiler/src/frontend/lowering/lower-inspect.ts b/packages/compiler/src/frontend/lowering/lower-inspect.ts index 47f490c7b..dd950a2a2 100644 --- a/packages/compiler/src/frontend/lowering/lower-inspect.ts +++ b/packages/compiler/src/frontend/lowering/lower-inspect.ts @@ -14,13 +14,15 @@ import { InternalCompilerError } from "../../errors.js"; import * as ts from "../ts7/adapter.js"; import { unsignedHex } from "../../format-integer.js"; import type { Lowerer } from "./lowerer.js"; -import { isJsSourceFile } from "../program.js"; +import { isJsSourceFile, locOf } from "../program.js"; +import { noLoweringDiag } from "../../diagnostics/diagnostic.js"; import { BOOL, DYN, F64, type IrExpr, type IrFunction, + type IrLocal, type IrStmt, type IrStrIntrinsicMethod, type IrType, @@ -32,6 +34,7 @@ import { type SrcLoc, canConvertToDyn, canDynCheckTo, + funcOf, CLASS_FIELD_ATTRIBUTES_PREFIX, classDynViewSupported, DYN_CLASS_PROPERTIES, @@ -42,15 +45,16 @@ import { shapeHasAccessorSlots, typeKey, } from "../../ir/ir.js"; -import type { ClassInfo } from "./lower-classes.js"; +import { accessorCall, type ClassInfo } from "./lower-classes.js"; import { isSafeToDiscard, isSafeToRepeat } from "./expressions/evaluation-safety.js"; -import { symbolFieldDisplayName } from "./symbol-fields.js"; +import { INSPECT_CUSTOM_MEMBER, symbolFieldDisplayName } from "./symbol-fields.js"; import { eventTargetRootOf } from "./lower-event-target.js"; import { dateRootOf } from "./builtins/date-classes.js"; import { lowerSystemErrorMap } from "./lower-system-errors.js"; import { isPerfEntryType, perfEntryJson } from "./builtins/performance.js"; import { recordMayKeepSource, recordSourceTest } from "./record-sources.js"; import { nullProtoRecordShape, recordNullProtoTest } from "./null-prototype.js"; +import { dynArgumentPack } from "./call-arguments.js"; /* ── IR construction shorthand ───────────────────────────────────────── */ function concatAll(parts: IrExpr[], loc: SrcLoc): IrExpr { @@ -805,6 +809,8 @@ function inspectSupport( case "object": { const info = lowerer.classes.get(t.className); if (!info) return `class '${t.className}' has no inspect lowering`; + const custom = customInspectReason(lowerer, info); + if (custom !== null) return custom; // EventTarget and Event subclasses render through Node's inspectors // for those classes (the runtime's capsule view), and so do classes // declared `extends Date`. @@ -900,6 +906,8 @@ export function inspectExpr( recurse: IrExpr, depth: IrExpr, loc: SrcLoc, + /** The instance's own rendering, without its custom inspect method. */ + ownRendering = false, ): IrExpr { // A Dirent row renders through its dyn view: Node's `Dirent { ... }` // with the Symbol(type) key. @@ -960,6 +968,15 @@ export function inspectExpr( loc, }; case "object": + if (!ownRendering && customInspectMethodOf(lowerer, t.className)) { + return { + kind: "call", + callee: customInspectHelper(lowerer, t, loc), + args: [value, recurse, depth], + type: STRING, + loc, + }; + } if ( eventTargetRootOf(lowerer.classes.get(t.className)) !== null || dateRootOf(lowerer.classes.get(t.className)) !== null @@ -1139,6 +1156,258 @@ function classInspectsAddedProperties(lowerer: Lowerer, info: ClassInfo): boolea ); } +/* ── custom inspect methods ([util.inspect.custom]) ──────────────────── */ + +/** The `[util.inspect.custom]` method a class's instances have, declared + * on the class or a base; null when they have none. */ +function customInspectMethodOf(lowerer: Lowerer, className: string) { + const info = lowerer.classes.get(className); + return info ? (lowerer.findMethodOn(info, INSPECT_CUSTOM_MEMBER) ?? null) : null; +} + +/** Why instances of a class with a custom inspect method have no static + * rendering, or null. Node calls the method with the remaining depth, the + * options and util.inspect; the method's parameters receive them as + * checked-dynamic values, and its result renders in the instance's place. */ +function customInspectReason(lowerer: Lowerer, info: ClassInfo): string | null { + const found = lowerer.findMethodOn(info, INSPECT_CUSTOM_MEMBER); + if (!found) { + const below = (c: ClassInfo): boolean => + c.subclasses.some((s) => s.methods.has(INSPECT_CUSTOM_MEMBER) || below(s)); + return below(info) + ? "classes whose subclasses declare [util.inspect.custom] have no inspect lowering yet (declare the method on the class itself)" + : null; + } + if (found.sig.params.length > 3) + return "[util.inspect.custom] methods with more than three parameters have no inspect lowering"; + for (const param of found.sig.params) { + if ( + param.type.kind !== "dyn" && + !canDynCheckTo( + param.type, + (id) => lowerer.shapes.get(id), + (id) => lowerer.unions.get(id), + ) + ) + return `[util.inspect.custom] methods with a '${lowerer.fmt(param.type)}' parameter have no inspect lowering`; + } + const ret = found.sig.ret; + if (ret.kind === "object") + return "[util.inspect.custom] methods that return a class instance have no inspect lowering"; + if ( + ret.kind !== "string" && + ret.kind !== "dyn" && + ret.kind !== "void" && + inspectSupport(lowerer, ret, new Set(), { recursive: false }) !== null + ) + return `[util.inspect.custom] methods returning '${lowerer.fmt(ret)}' have no inspect lowering`; + return null; +} + +/** Whether values of `t` can hold a checked-dynamic value, whose own custom + * inspect functions the runtime walk calls. */ +function typeReachesDyn(lowerer: Lowerer, t: IrType): boolean { + const seen = new Set(); + const walk = (u: IrType): boolean => { + if (u.kind === "dyn") return true; + const k = typeKey(u); + if (seen.has(k)) return false; + seen.add(k); + return typeConstituents(lowerer, u).some(walk); + }; + return walk(t); +} + +/** Whether values of `t` can hold an instance of a class with a custom + * inspect method. */ +export function typeReachesCustomInspect(lowerer: Lowerer, t: IrType): boolean { + const seen = new Set(); + const walk = (u: IrType): boolean => { + const k = typeKey(u); + if (seen.has(k)) return false; + seen.add(k); + if (u.kind === "object" && customInspectMethodOf(lowerer, u.className)) return true; + return typeConstituents(lowerer, u).some(walk); + }; + return walk(t); +} + +/** `%util.inspCustom.(v, r, d): string` — formatValue's custom hook + * over a class whose instances have a `[util.inspect.custom]` method: + * with ctx.customInspect set, the method's result (a string re-indented + * at the current level, the instance itself rendered normally, any other + * value rendered in its place); otherwise the instance's own rendering. */ +function customInspectHelper( + lowerer: Lowerer, + t: IrType & { kind: "object" }, + loc: SrcLoc, +): string { + const key = `inspCustom:${typeKey(t)}`; + const existing = lowerer.inspectHelpers.get(key); + if (existing) return existing; + const name = keyedHelperName(lowerer, "%util.inspCustom", key); + lowerer.inspectHelpers.set(key, name); + const found = customInspectMethodOf(lowerer, t.className)!; + const v = (): IrExpr => varRef("v.0", t, loc); + const r = (): IrExpr => varRef("r.0", F64, loc); + const d = (): IrExpr => varRef("d.0", F64, loc); + const ret = (value: IrExpr): IrStmt => ({ kind: "return", value, loc }); + const normal = (): IrExpr => inspectExpr(lowerer, t, v(), r(), d(), loc, true); + const runtimeArgs: IrExpr[] = [ + { kind: "libCall", fn: "insp.customDepth", args: [r(), d()], type: DYN, loc }, + { kind: "libCall", fn: "insp.customOptions", args: [d()], type: DYN, loc }, + { kind: "libCall", fn: "insp.inspectFunction", args: [], type: DYN, loc }, + ]; + const args = found.sig.params.map((param, i) => + lowerer.coerceToExpected(runtimeArgs[i]!, param.type), + ); + const resultT = found.sig.ret; + const call = accessorCall(lowerer, t.className, INSPECT_CUSTOM_MEMBER, v(), args, resultT, loc); + const locals: IrLocal[] = [ + { id: "v.0", name: "v", type: t, mutable: false }, + { id: "r.0", name: "r", type: F64, mutable: false }, + { id: "d.0", name: "d", type: F64, mutable: false }, + ]; + const render: IrStmt[] = []; + if (resultT.kind === "string") { + render.push(ret({ kind: "libCall", fn: "insp.customText", args: [call], type: STRING, loc })); + } else if (resultT.kind === "dyn") { + locals.push({ id: "res.0", name: "res", type: DYN, mutable: false }); + const res = (): IrExpr => varRef("res.0", DYN, loc); + render.push( + { kind: "varDecl", localId: "res.0", init: call, loc }, + { + kind: "if", + cond: { + kind: "libCall", + fn: "insp.customIsSelf", + args: [res(), { kind: "dynFrom", value: v(), liveRef: true, type: DYN, loc }], + type: BOOL, + loc, + }, + then: [ret(normal())], + else_: null, + loc, + }, + ret({ kind: "libCall", fn: "insp.customResult", args: [res(), r(), d()], type: STRING, loc }), + ); + } else if (resultT.kind === "void") { + render.push({ kind: "exprStmt", expr: call, loc }, ret(strLit("undefined", loc))); + } else { + render.push(ret(inspectExpr(lowerer, resultT, call, r(), d(), loc))); + } + lowerer.liftedFns.push({ + name, + params: [ + { localId: "v.0", name: "v", type: t }, + { localId: "r.0", name: "r", type: F64 }, + { localId: "d.0", name: "d", type: F64 }, + ], + returnType: STRING, + locals, + body: [ + { + kind: "if", + cond: { kind: "libCall", fn: "insp.customEnabled", args: [], type: BOOL, loc }, + then: render, + else_: null, + loc, + }, + ret(normal()), + ], + loc, + }); + return name; +} + +/** `%util.inspCustomDyn.(self, depth, options, inspect): dyn` — a + * class's custom inspect method for its instances held as checked-dynamic + * values: the runtime's inspect walk calls it with the instance's typed + * ref and the method's three arguments. */ +function customInspectDynHelper( + lowerer: Lowerer, + t: IrType & { kind: "object" }, + loc: SrcLoc, +): string { + const name = keyedHelperName(lowerer, "%util.inspCustomDyn", typeKey(t)); + const found = customInspectMethodOf(lowerer, t.className)!; + const param = (id: string): IrExpr => varRef(id, DYN, loc); + const ids = ["depth.0", "options.0", "inspect.0"]; + const call = accessorCall( + lowerer, + t.className, + INSPECT_CUSTOM_MEMBER, + varRef("v.0", t, loc), + found.sig.params.map((p, i) => lowerer.coerceToExpected(param(ids[i]!), p.type)), + found.sig.ret, + loc, + ); + const body: IrStmt[] = [ + { kind: "varDecl", localId: "v.0", init: lowerer.coerceToExpected(param("self.0"), t), loc }, + ]; + if (found.sig.ret.kind === "void") { + body.push( + { kind: "exprStmt", expr: call, loc }, + { kind: "return", value: dynUndefinedExpr(loc), loc }, + ); + } else { + body.push({ kind: "return", value: lowerer.coerceToExpected(call, DYN), loc }); + } + const params = ["self.0", ...ids]; + lowerer.liftedFns.push({ + name, + params: params.map((id) => ({ localId: id, name: id.slice(0, -2), type: DYN })), + returnType: DYN, + locals: [ + ...params.map((id) => ({ id, name: id.slice(0, -2), type: DYN, mutable: false })), + { id: "v.0", name: "v", type: t, mutable: false }, + ], + body, + loc, + }); + return name; +} + +/** The statements %main runs first, in every thread: each class whose + * instances have a custom inspect method registers its compiled call + * under its instances' typed-ref type key (scr_insp_register_custom). */ +export function customInspectRegistrations(lowerer: Lowerer, loc: SrcLoc): IrStmt[] { + const stmts: IrStmt[] = []; + for (const info of lowerer.classes.values()) { + if (info.def.runtime || info.def.abstract || isErrorClass(lowerer, info.def.name)) continue; + if (!lowerer.findMethodOn(info, INSPECT_CUSTOM_MEMBER)) continue; + if (customInspectReason(lowerer, info) !== null) continue; + const t = { kind: "object" as const, className: info.def.name }; + const fn = customInspectDynHelper(lowerer, t, loc); + stmts.push({ + kind: "exprStmt", + expr: { + kind: "libCall", + fn: "insp.registerCustom", + args: [ + strLit(typeKey(t), loc), + { + kind: "dynFrom", + value: { + kind: "closure", + fnName: fn, + captures: [], + type: funcOf([DYN, DYN, DYN, DYN], DYN), + loc, + }, + type: DYN, + loc, + }, + ], + type: VOID, + loc, + }, + loc, + }); + } + return stmts; +} + /* ── the interned per-type helper ────────────────────────────────────── */ /** `%util.insp.(v, r, d): string` — formatValue over the static type: @@ -2213,13 +2482,22 @@ function isDataViewTypeName(tname: string): boolean { /* ── the callsite: options, direct function/class names ──────────────── */ /** The parsed options literal: the resolved depth (numeric; Infinity for - * `null`), breakLength (numeric; default 80) and compact (numeric, 0 for + * `null`), breakLength (numeric; default 80) and compact (numeric, -1 for * false; default 3), with every other non-default knob fenced by name. */ interface InspectOptions { depth: number; breakLength: number; compact: number; sorted: boolean; + /** Set by the `atRunTime` parse: colors or a computed knob value, which + * the runtime reads. */ + runtime: boolean; + /** How the depth option was spelled, for custom inspect methods: 0 a + * number (also when absent), 1 null, 2 undefined. */ + depthKind: number; + /** customInspect: false skips [util.inspect.custom] methods (console.dir's + * default). */ + customInspect: boolean; } function parseInspectOptions( @@ -2227,9 +2505,18 @@ function parseInspectOptions( node: ts.Expression | undefined, recursive: boolean, dir = false, + atRunTime = false, ): InspectOptions { const surface = dir ? "console.dir" : "util.inspect"; - const options: InspectOptions = { depth: 2, breakLength: 80, compact: 3, sorted: false }; + const options: InspectOptions = { + depth: 2, + breakLength: 80, + compact: 3, + sorted: false, + runtime: false, + depthKind: 0, + customInspect: !dir, + }; if (!node || (ts.isIdentifier(node) && node.text === "undefined")) return options; if (!ts.isObjectLiteralExpression(node)) { lowerer.noLowering( @@ -2240,6 +2527,11 @@ function parseInspectOptions( } for (const prop of node.properties.nodes) { if (!ts.isPropertyAssignment(prop) || !ts.isIdentifier(prop.name)) { + // `{ depth }` and `{ ...base }` hold values known at run time. + if (atRunTime && (ts.isShorthandPropertyAssignment(prop) || ts.isSpreadAssignment(prop))) { + options.runtime = true; + continue; + } lowerer.noLowering( `${surface} options in this form`, prop, @@ -2269,8 +2561,17 @@ function parseInspectOptions( // bounded by the (finite) object graph — exactly Node. void recursive; options.depth = Infinity; - } else if (!(ts.isIdentifier(value) && value.text === "undefined")) { + options.depthKind = value.kind === ts.SyntaxKind.NullKeyword ? 1 : 0; + } else if (ts.isIdentifier(value) && value.text === "undefined") { + // Node's ctx.depth is undefined: no depth limit. + options.depth = Infinity; + options.depthKind = 2; + } else { const n = numeric(); + if (n === null && atRunTime) { + options.runtime = true; + continue; + } if (n === null) { lowerer.noLowering( `${surface} with a non-literal depth option`, @@ -2281,7 +2582,8 @@ function parseInspectOptions( options.depth = n; } } else if (key === "colors") { - if (value.kind !== ts.SyntaxKind.FalseKeyword) { + if (value.kind !== ts.SyntaxKind.FalseKeyword && atRunTime) options.runtime = true; + else if (value.kind !== ts.SyntaxKind.FalseKeyword) { lowerer.noLowering( `${surface} with colors: true`, value, @@ -2293,6 +2595,10 @@ function parseInspectOptions( // compact of at least 1; false and smaller numbers break every // composite onto its own lines. compact: true is the older layout. const n = value.kind === ts.SyntaxKind.FalseKeyword ? 0 : numeric(); + if (n === null && atRunTime && value.kind !== ts.SyntaxKind.TrueKeyword) { + options.runtime = true; + continue; + } if (n === null || Number.isNaN(n)) { lowerer.noLowering( `${surface} with this compact option`, @@ -2300,9 +2606,22 @@ function parseInspectOptions( "compact must be false or a numeric literal (compact: true has no lowering)", ); } - options.compact = Math.max(n, 0); + // false is -1: the same layout as 0, and custom inspect functions + // read it back as false. + options.compact = value.kind === ts.SyntaxKind.FalseKeyword ? -1 : Math.max(n, 0); } else if (key === "sorted") { - if (value.kind !== ts.SyntaxKind.TrueKeyword && value.kind !== ts.SyntaxKind.FalseKeyword) { + const sortedLiteral = + value.kind === ts.SyntaxKind.TrueKeyword || value.kind === ts.SyntaxKind.FalseKeyword; + if ( + !sortedLiteral && + atRunTime && + !ts.isArrowFunction(value) && + !ts.isFunctionExpression(value) + ) { + options.runtime = true; + continue; + } + if (!sortedLiteral) { lowerer.noLowering( `${surface} with this sorted option`, value, @@ -2312,6 +2631,10 @@ function parseInspectOptions( options.sorted = value.kind === ts.SyntaxKind.TrueKeyword; } else if (key === "breakLength") { const n = ts.isIdentifier(value) && value.text === "Infinity" ? Infinity : numeric(); + if (n === null && atRunTime) { + options.runtime = true; + continue; + } if (n === null || Number.isNaN(n)) { lowerer.noLowering( `${surface} with a non-literal breakLength option`, @@ -2320,6 +2643,12 @@ function parseInspectOptions( ); } options.breakLength = n; + } else if ( + !dir && + key === "customInspect" && + (value.kind === ts.SyntaxKind.TrueKeyword || value.kind === ts.SyntaxKind.FalseKeyword) + ) { + options.customInspect = value.kind === ts.SyntaxKind.TrueKeyword; } else if (dir && key === "customInspect") { // console.dir's own default; only that value is lowered. if (value.kind !== ts.SyntaxKind.FalseKeyword) { @@ -2417,16 +2746,86 @@ function directCallableInspect(lowerer: Lowerer, node: ts.Expression, loc: SrcLo return null; } -/** util.inspect(value[, options]) — the spoke's inspect entry. */ +/** util.inspect(value[, options]) and inspect(value, showHidden, depth, + * colors) — the spoke's inspect entry. Options the compiler resolves + * render through the static helpers; colors, option values known only at + * run time and the legacy forms render the value's checked-dynamic view + * in the runtime (insp.inspect). */ function lowerInspectCall(lowerer: Lowerer, expr: ts.CallExpression, loc: SrcLoc): IrExpr { - if (expr.arguments.nodes.length < 1 || expr.arguments.nodes.length > 2) { + const args = expr.arguments.nodes; + if (args.length < 1 || args.length > 4 || args.some(ts.isSpreadElement)) { lowerer.noLowering( - `util.inspect with ${expr.arguments.nodes.length} arguments`, + `util.inspect with ${args.some(ts.isSpreadElement) ? "spread" : args.length} arguments`, expr, - "the supported forms are inspect(value) and inspect(value, options)", + "the supported forms are inspect(value), inspect(value, options) and inspect(value, showHidden, depth, colors)", ); } - return lowerInspectValue(lowerer, expr.arguments.nodes[0]!, expr.arguments.nodes[1], loc, false); + if (args.length <= 2 && !inspectOptionsAtRunTime(lowerer, args[1])) + return lowerInspectValue(lowerer, args[0]!, args[1], loc, false); + return lowerRuntimeInspect(lowerer, args, loc); +} + +/** Whether an options argument needs the runtime reader: an options value + * that is not an object literal, or a literal with colors or a computed + * depth, compact, breakLength or sorted value. Literal options otherwise + * keep the static parse, which also reports the literal options neither + * path lowers. */ +function inspectOptionsAtRunTime(lowerer: Lowerer, node: ts.Expression | undefined): boolean { + if (!node || (ts.isIdentifier(node) && node.text === "undefined")) return false; + if (!ts.isObjectLiteralExpression(node)) return true; + return parseInspectOptions(lowerer, node, false, false, true).runtime; +} + +/** util.inspect over the value's checked-dynamic view, with every argument + * evaluated in order and read in the runtime as Node's inspect reads it. */ +function lowerRuntimeInspect( + lowerer: Lowerer, + args: readonly ts.Expression[], + loc: SrcLoc, +): IrExpr { + const valueNode = args[0]!; + if ( + directCallableInspect(lowerer, valueNode, loc) !== null || + ts.isArrowFunction(valueNode) || + ts.isFunctionExpression(valueNode) + ) { + lowerer.noLowering( + "util.inspect of a function or class with these options", + valueNode, + "pass the options as a literal without colors", + ); + } + const value = lowerer.lowerExpr(valueNode); + if (value.type.kind !== "dyn") { + const reason = inspectSupport(lowerer, value.type, new Set(), { recursive: false }); + if (reason !== null) + lowerer.noLowering(`util.inspect of '${lowerer.fmt(value.type)}' values`, valueNode, reason); + if (!lowerer.dynConvertible(value.type)) + lowerer.noLowering( + `util.inspect with these options of '${lowerer.fmt(value.type)}' values`, + valueNode, + "the options are read at run time, where the value renders through its checked-dynamic view", + ); + } + const stmts: IrStmt[] = []; + const held = (init: IrExpr): IrExpr => { + const local = lowerer.declareHiddenLocal("%inspectArg", DYN); + stmts.push({ kind: "varDecl", localId: local.id, init, loc }); + return varRef(local.id, DYN, loc); + }; + const operands: IrExpr[] = [held(lowerer.coerceToExpected(value, DYN))]; + for (let i = 1; i < 4; i++) { + const node = args[i]; + operands.push(node ? held(lowerer.lowerExprExpecting(node, DYN)) : dynUndefinedExpr(loc)); + } + operands.push(numLit(args.length, loc)); + return { + kind: "seqExpr", + stmts, + result: { kind: "libCall", fn: "insp.inspect", args: operands, type: STRING, loc }, + type: STRING, + loc, + }; } /** `inspect(value, options)` over one value node: the shared body of @@ -2512,6 +2911,46 @@ export function lowerInspectValue( } const options = parseInspectOptions(lowerer, optionsNode, walk.recursive, dir); const depth = options.depth; + // Custom inspect functions read the options as Node spelled them: an + // unbounded depth given as Infinity or undefined, and customInspect: + // false (console.dir's default), apply around the render when the value + // can reach a custom method; the previous settings return afterwards. + const reachesCustom = + typeReachesCustomInspect(lowerer, value.type) || typeReachesDyn(lowerer, value.type); + const depthContext = reachesCustom && depth === Infinity && options.depthKind !== 1; + const customContext = reachesCustom && !options.customInspect; + if (depthContext || customContext) { + const stmts: IrStmt[] = []; + const restore: IrStmt[] = []; + const setting = (fn: "insp.customInspect" | "insp.depthKind", arg: IrExpr, type: IrType) => { + const previous = lowerer.declareHiddenLocal("%inspectContext", type); + const call = (a: IrExpr): IrExpr => ({ kind: "libCall", fn, args: [a], type, loc }); + stmts.push({ kind: "varDecl", localId: previous.id, init: call(arg), loc }); + restore.push({ kind: "exprStmt", expr: call(varRef(previous.id, type, loc)), loc }); + }; + if (customContext) setting("insp.customInspect", boolLit(false, loc), BOOL); + if (depthContext) setting("insp.depthKind", numLit(options.depthKind, loc), F64); + const text = lowerer.declareHiddenLocal("%inspectText", STRING); + stmts.push({ + kind: "varDecl", + localId: text.id, + init: withInspectLayout( + lowerer, + options, + value, + (input) => inspectExpr(lowerer, value.type, input, numLit(0, loc), numLit(depth, loc), loc), + loc, + ), + loc, + }); + return { + kind: "seqExpr", + stmts: [...stmts, ...restore.reverse()], + result: varRef(text.id, STRING, loc), + type: STRING, + loc, + }; + } if (dir && value.type.kind === "dyn") { return withInspectLayout( lowerer, @@ -2542,7 +2981,17 @@ export function lowerInspectValue( * formatNumber, strings verbatim, objects through inspect at the given * depth). Null = fence with the given reason. */ function formatSArg(lowerer: Lowerer, node: ts.Expression, depth: number, loc: SrcLoc): IrExpr { - const value = lowerer.lowerExpr(node); + return formatSValue(lowerer, node, lowerer.lowerExpr(node), depth, loc); +} + +/** formatSArg over the already-lowered value of `node`. */ +function formatSValue( + lowerer: Lowerer, + node: ts.Expression, + value: IrExpr, + depth: number, + loc: SrcLoc, +): IrExpr { const t = value.type; if (t.kind === "string") return value; if (t.kind === "f64") @@ -2626,6 +3075,13 @@ function formatOArg(lowerer: Lowerer, node: ts.Expression, depth: number, loc: S node, reason, ); + if (depth === 4 && typeReachesCustomInspect(lowerer, value.type)) { + lowerer.noLowering( + "util.format %o over values with [util.inspect.custom] methods", + node, + "%o passes showHidden to the methods — use %O", + ); + } if (depth === 4 && typeTreeHasArray(lowerer, value.type, new Set())) { lowerer.noLowering( "util.format %o over arrays", @@ -2708,8 +3164,9 @@ function formatIntegerFixed(value: IrExpr, spec: "%d" | "%i", loc: SrcLoc): IrEx /** util.format(...) / util.formatWithOptions({}, ...): the compile-time * %-substitution over a literal format string (formatWithOptionsInternal * ported exactly — the arg cursor, the args-exhausted guard, %c's - * consume-and-drop), with per-static-type conversions. Non-literal - * format strings lower only in the substitution-free shapes. */ + * consume-and-drop), with per-static-type conversions. Format strings + * known only at run time, spread arguments and non-empty options lower to + * the runtime formatter (lowerRuntimeFormat, heldFormatArgs). */ export function lowerFormatCall( lowerer: Lowerer, expr: ts.CallExpression, @@ -2721,30 +3178,60 @@ export function lowerFormatCall( const argNodes = [...(argNodesOverride ?? expr.arguments.nodes)]; if (withOptions) { const opts = argNodes.shift(); + // The empty literal {} is the defaults: exactly util.format. if (!opts || !ts.isObjectLiteralExpression(opts) || opts.properties.nodes.length > 0) { - lowerer.noLowering( - "util.formatWithOptions with a non-empty options literal", - opts ?? expr, - "only the empty literal {} (the defaults — exactly util.format) is lowered", - ); + if (opts && ts.isSpreadElement(opts)) { + lowerer.noLowering( + "util.formatWithOptions with a spread options argument", + opts, + "pass the options object as the first argument", + ); + } + return lowerRuntimeFormat(lowerer, argNodes, loc, opts ?? null); } } if (argNodes.length === 0) return strLit("", loc); const first = argNodes[0]!; const firstIsLiteral = ts.isStringLiteral(first) || ts.isNoSubstitutionTemplateLiteral(first); + if (argNodes.some(ts.isSpreadElement)) { + if (literalPrefix !== "" && !firstIsLiteral) + throw new InternalCompilerError("a format prefix needs a literal format string"); + return lowerRuntimeFormat(lowerer, argNodes, loc, undefined, literalPrefix); + } /** The rest-args tail: ` ` + (strings verbatim, everything else * inspected at the default depth). */ const restArg = (node: ts.Expression): IrExpr => formatSArg(lowerer, node, 2, loc); if (!firstIsLiteral) { - const firstT = lowerer.mapTypeOf(lowerer.typeOf(first)); - if (firstT?.kind === "string" && argNodes.length > 1) { - lowerer.noLowering( - "util.format with a runtime format string and further arguments", - first, - "the %-substitution positions are runtime-dependent — use a string literal", - ); + const runtime = argNodes.length > 1 ? runtimeFormatString(lowerer, first) : null; + if (runtime === "other") return lowerRuntimeFormat(lowerer, argNodes, loc); + if (runtime !== null) { + // A string that substitutes formats at run time; any other string + // joins with the remaining arguments, as below. + const held = heldFormatArgs(lowerer, argNodes, loc); + if (!held.joinable) { + return { kind: "seqExpr", stmts: held.stmts, result: held.format(), type: STRING, loc }; + } + const joined: IrExpr[] = []; + held.args.forEach((arg, i) => { + if (i > 0) joined.push(strLit(" ", loc)); + joined.push(formatRestValue(lowerer, arg.node, arg.ref(), loc)); + }); + return { + kind: "seqExpr", + stmts: held.stmts, + result: { + kind: "ternary", + cond: held.specifies, + then: held.format(), + else_: concatAll(joined, loc), + type: STRING, + loc, + }, + type: STRING, + loc, + }; } // No substitutions possible: every argument joins with spaces. const parts: IrExpr[] = []; @@ -3030,6 +3517,240 @@ export function lowerFormatCall( return concatAll(parts, loc); } +/** One space-joined argument of util.format, as Node's formatter renders + * it after the format string: strings verbatim, everything else through + * inspect at the default depth. */ +function formatRestValue( + lowerer: Lowerer, + node: ts.Expression, + value: IrExpr, + loc: SrcLoc, +): IrExpr { + switch (value.type.kind) { + case "string": + return value; + case "f64": + return { kind: "libCall", fn: "insp.f64", args: [value], type: STRING, loc }; + case "bool": + return { kind: "toString", operand: value, type: STRING, loc }; + case "dyn": + return { kind: "libCall", fn: "insp.dynS", args: [value, numLit(2, loc)], type: STRING, loc }; + default: + return lowerConsoleInspectArg(lowerer, node, value, "util.format", loc); + } +} + +const UNCONVERTIBLE_HINT = + "the runtime formatter reads its arguments as checked-dynamic values — use a string literal format"; + +/** A string-valued expression that throws the SC2020 refusal where it runs + * (a JavaScript construct fence deferred to the branch that needs it). */ +function runtimeFenceValue( + lowerer: Lowerer, + surface: string, + node: ts.Node, + hint: string, + loc: SrcLoc, +): IrExpr { + const diag = noLoweringDiag(surface, locOf(node), hint); + lowerer.runtimeFences.push(diag); + const sf = ts.getSourceFileOfNode(node); + const line = ts.getLineAndCharacterOfPosition(sf, diag.loc.start).line + 1; + const slot = lowerer.declareHiddenLocal("%never", STRING); + slot.mutable = true; + return { + kind: "seqExpr", + stmts: [ + { kind: "varDecl", localId: slot.id, init: null, loc }, + { + kind: "runtimeFence", + code: diag.code, + message: `${diag.message} [${diag.code} at ${diag.loc.file}:${line}]`, + loc, + }, + ], + result: varRef(slot.id, STRING, loc), + diverges: true, + type: STRING, + loc, + }; +} + +/** How a non-literal format string is known: "string" and "dyn" values + * test for a substitution at run time, "other" types (unions with a string + * arm) format at run time, and null types never hold a string. */ +export function runtimeFormatString( + lowerer: Lowerer, + first: ts.Expression, +): "string" | "dyn" | "other" | null { + const t = lowerer.mapTypeOf(lowerer.typeOf(first)); + if (t === null) return "other"; + if (t.kind === "string") return "string"; + if (t.kind === "dyn") return "dyn"; + if (t.kind === "union") { + const arms = lowerer.unions.get(t.unionId)?.arms ?? []; + return arms.some((arm) => arm.kind === "string" || arm.kind === "dyn") ? "other" : null; + } + return t.kind === "jsval" || t.kind === "caught" ? "other" : null; +} + +/** The runtime formatter over a call's arguments: one checked-dynamic + * pack in source order (spreads contribute their elements), after the + * options when `options` is given (null: an omitted options argument, + * which Node's validation rejects). A literal prefix joins a literal + * format string. */ +export function lowerRuntimeFormat( + lowerer: Lowerer, + argNodes: readonly ts.Expression[], + loc: SrcLoc, + options?: ts.Expression | null, + literalPrefix = "", +): IrExpr { + const stmts: IrStmt[] = []; + let optionsRef: IrExpr | null = null; + if (options !== undefined) { + const held = lowerer.declareHiddenLocal("%formatOptions", DYN); + stmts.push({ + kind: "varDecl", + localId: held.id, + init: options === null ? dynUndefinedExpr(loc) : lowerer.lowerExprExpecting(options, DYN), + loc, + }); + optionsRef = varRef(held.id, DYN, loc); + } + let nodes = argNodes; + const leading: IrExpr[] = []; + if (literalPrefix !== "") { + const first = argNodes[0] as ts.StringLiteralLike; + leading.push(strLit(literalPrefix + first.text, loc)); + nodes = argNodes.slice(1); + } + const pack = dynArgumentPack(lowerer, nodes, loc, { + leading, + blame: argNodes[0] ?? options ?? undefined, + }); + stmts.push(...pack.stmts); + const result: IrExpr = + optionsRef === null + ? { kind: "libCall", fn: "insp.format", args: [pack.pack()], type: STRING, loc } + : { + kind: "libCall", + fn: "insp.formatWithOptions", + args: [optionsRef, pack.pack()], + type: STRING, + loc, + }; + return { kind: "seqExpr", stmts, result, type: STRING, loc }; +} + +/** The arguments of a call whose first argument is a string or dyn value + * known only at run time, each evaluated once into a hidden local: + * `specifies` tests that string for a '%' before its last character, and + * `format()` is the runtime formatter over the held values. The caller + * renders the other branch (the space-joined arguments) from `args` when + * every argument is `joinable` there. */ +export function heldFormatArgs( + lowerer: Lowerer, + argNodes: readonly ts.Expression[], + loc: SrcLoc, +): { + stmts: IrStmt[]; + args: { node: ts.Expression; ref: () => IrExpr }[]; + specifies: IrExpr; + format: () => IrExpr; + joinable: boolean; +} { + const stmts: IrStmt[] = []; + let unconvertible: { node: ts.Expression; type: IrType } | null = null; + const unconvertibleSurface = (t: IrType): string => + `util.format with a runtime format string over '${lowerer.fmt(t)}' values`; + const args = argNodes.map((node) => { + const value = lowerer.lowerExpr(node); + if (value.type.kind === "void") { + lowerer.unsupported( + "SC1090", + node, + "util.format of a void call result (call it as its own statement, then pass undefined)", + ); + } + if (value.type.kind !== "dyn" && !lowerer.dynConvertible(value.type)) { + // JavaScript keeps the space-joined branch, which printed these + // values before, and refuses only a string that substitutes. + if (!isJsSourceFile(ts.getSourceFileOfNode(node))) { + lowerer.noLowering(unconvertibleSurface(value.type), node, UNCONVERTIBLE_HINT); + } + unconvertible ??= { node, type: value.type }; + } + // Unit values have no local: their effects run here and a literal + // stands in. + const k = value.type.kind; + if (k === "undefinedT" || k === "nullT") { + if (value.kind !== "unitLit") stmts.push({ kind: "exprStmt", expr: value, loc }); + const unit = k === "nullT" ? ("null" as const) : ("undefined" as const); + return { node, ref: (): IrExpr => ({ kind: "unitLit", unit, type: value.type, loc }) }; + } + const held = lowerer.declareHiddenLocal("%formatArg", value.type); + stmts.push({ kind: "varDecl", localId: held.id, init: value, loc }); + return { node, ref: () => varRef(held.id, value.type, loc) }; + }); + const first = args[0]!.ref(); + const specifies: IrExpr = + first.type.kind === "string" + ? { kind: "libCall", fn: "insp.formatSpecifies", args: [first], type: BOOL, loc } + : { + kind: "libCall", + fn: "insp.formatSpecifiesDyn", + args: [lowerer.coerceToExpected(first, DYN)], + type: BOOL, + loc, + }; + const format = (): IrExpr => { + if (unconvertible !== null) { + const fenced = unconvertible; + return runtimeFenceValue( + lowerer, + unconvertibleSurface(fenced.type), + fenced.node, + UNCONVERTIBLE_HINT, + loc, + ); + } + // Strings, numbers, booleans, null and undefined format without the + // checked-dynamic walk other values need, which keeps it out of the + // executable. + const primitive = (t: IrType): boolean => + t.kind === "string" || + t.kind === "f64" || + t.kind === "bool" || + t.kind === "undefinedT" || + t.kind === "nullT" || + (t.kind === "union" && (lowerer.unions.get(t.unionId)?.arms.every(primitive) ?? false)); + return { + kind: "libCall", + fn: args.every((arg) => primitive(arg.ref().type)) ? "insp.formatPrimitives" : "insp.format", + args: [ + { + kind: "dynArrLit", + elems: args.map((arg) => lowerer.coerceToExpected(arg.ref(), DYN)), + type: DYN, + loc, + }, + ], + type: STRING, + loc, + }; + }; + // The space-joined branch renders each argument statically; arguments + // the static rendering refuses take the runtime formatter in both cases. + const joinable = args.every(({ ref }) => { + const t = ref().type; + if (t.kind === "func") return false; + if (t.kind === "bytes") return true; + return inspectSupport(lowerer, t, new Set(), { recursive: false }) === null; + }); + return { stmts, args, specifies, format, joinable }; +} + /** Arrays anywhere in the tree — the %o (showHidden) gate. */ function typeTreeHasArray(lowerer: Lowerer, t: IrType, visiting: Set): boolean { switch (t.kind) { diff --git a/packages/compiler/src/frontend/lowering/lower-modules.ts b/packages/compiler/src/frontend/lowering/lower-modules.ts index 5238333ad..8013b842f 100644 --- a/packages/compiler/src/frontend/lowering/lower-modules.ts +++ b/packages/compiler/src/frontend/lowering/lower-modules.ts @@ -93,6 +93,7 @@ import { createRequireProgramModuleDecl, } from "./builtins/module-bindings.js"; import { isPromisifyCall, registerBuiltinCallableAlias } from "./builtins/callable-aliases.js"; +import { customInspectRegistrations } from "./lower-inspect.js"; import { registerBuiltinClassAlias } from "./builtins/class-values.js"; import { stripTypeCasts } from "./builtins/arguments.js"; import { isWebCryptoMemberType } from "./builtins/webcrypto.js"; @@ -1947,18 +1948,21 @@ export function collectGlobals( // symbol registers HERE, before any use site can lower, and the // statement lowering skips the declaration by the same check. A // bad promisify target reports here (recorded once; the statement - // lowering poisons too, like every collectGlobals failure). + // lowering poisons too, like every collectGlobals failure). Targets + // that promisify to a function value (typed callback-last functions + // and untyped functions) keep an ordinary global. if ( ts.isIdentifier(decl.name) && decl.initializer && isPromisifyCall(lowerer, decl.initializer) ) { + let registered = true; try { - lowerer.registerPromisifiedBuiltinDecl(decl.name, decl.initializer); + registered = lowerer.registerPromisifiedBuiltinDecl(decl.name, decl.initializer); } catch (e) { if (!(e instanceof PoisonError)) throw e; } - continue; + if (registered) continue; } // `const inspect = require("util").inspect` at file scope: a // named import in const clothing (builtinImportOf resolves uses) @@ -3523,6 +3527,9 @@ export function buildMain(lowerer: Lowerer): IrFunction { loc, }); } + // Classes with custom inspect methods register their compiled calls for + // the runtime's inspect walk before any module runs, in every thread. + body.unshift(...customInspectRegistrations(lowerer, loc)); // Node's startup refusal (a resolution the graph carries that Node // rejects — preflight's Node-order resolution walk — or a module-LINK // SyntaxError from either named-import checker): the graph is refused diff --git a/packages/compiler/src/frontend/lowering/lowerer.ts b/packages/compiler/src/frontend/lowering/lowerer.ts index 93c6246d9..c2c60abdc 100644 --- a/packages/compiler/src/frontend/lowering/lowerer.ts +++ b/packages/compiler/src/frontend/lowering/lowerer.ts @@ -13677,6 +13677,10 @@ export class Lowerer { * are only dispatch sentinels for call-site-specific lowering. */ lowerBuiltinCallableValue(bi: { module: string; member: string }, loc: SrcLoc): IrExpr | null { if (bi.module === "util" && bi.member === "types") return utilTypesModuleValue(this, loc); + // util.inspect: the function custom inspect methods receive, one + // identity per thread. + if (bi.module === "util" && bi.member === "inspect") + return { kind: "libCall", fn: "insp.inspectFunction", args: [], type: DYN, loc }; if (bi.module === "util/types" && builtinModuleFnOf(this, bi.module, bi.member)) return lowerUtilTypeValue(this, bi.member, loc); if (bi.module === "util" && bi.member === "getSystemErrorMap") diff --git a/packages/compiler/src/frontend/lowering/surfaces.ts b/packages/compiler/src/frontend/lowering/surfaces.ts index 8b826304b..80da35ad3 100644 --- a/packages/compiler/src/frontend/lowering/surfaces.ts +++ b/packages/compiler/src/frontend/lowering/surfaces.ts @@ -1686,7 +1686,8 @@ export const BUILTIN_MODULE_FNS: Record< // lowerVarDecl / collectGlobals BEFORE any call lowering runs — and // inspect/format/formatWithOptions plus parseArgs through the util spoke // (lower-inspect.ts: per-type synthesized traversal helpers, - // compile-time format strings, checked-dynamic config/result for + // compile-time format strings and the runtime formatter for the others, + // checked-dynamic config/result for // parseArgs), which both dispatch paths try before the generic table. // parseArgs is tabled as well so the generated surface manifest records // the dedicated static member; the spoke owns its optional arity and diff --git a/packages/compiler/src/frontend/lowering/symbol-fields.ts b/packages/compiler/src/frontend/lowering/symbol-fields.ts index 56718615e..ea495d364 100644 --- a/packages/compiler/src/frontend/lowering/symbol-fields.ts +++ b/packages/compiler/src/frontend/lowering/symbol-fields.ts @@ -76,7 +76,36 @@ export function fenceSymbolFieldCopy( } } +/** The registry name util.inspect.custom denotes. */ +export const INSPECT_CUSTOM_DESCRIPTION = "nodejs.util.inspect.custom"; +/** The method slot of a class's `[util.inspect.custom]` member. */ +export const INSPECT_CUSTOM_MEMBER = `${FIELD_PREFIX}Symbol(${INSPECT_CUSTOM_DESCRIPTION})`; + +/** `inspect.custom` over node:util's inspect: a named import, `util.inspect` + * or the CommonJS `util_1.inspect`. */ +export function isInspectCustomAccess(lowerer: Lowerer, expr: ts.Expression): boolean { + while (ts.isParenthesizedExpression(expr)) expr = expr.expression; + if (!ts.isPropertyAccessExpression(expr) || expr.questionDotToken) return false; + if (!ts.isIdentifier(expr.name) || expr.name.text !== "custom") return false; + let target = expr.expression; + while (ts.isParenthesizedExpression(target)) target = target.expression; + const bi = ts.isIdentifier(target) + ? lowerer.builtinImportOf(target) + : ts.isPropertyAccessExpression(target) && !target.questionDotToken + ? lowerer.builtinMemberOf(target) + : null; + return bi?.module === "util" && bi.member === "inspect"; +} + export function classSymbolKeyOf(lowerer: Lowerer, key: ts.Expression): ClassSymbolKey | null { + // util.inspect.custom is the registered Symbol.for(INSPECT_CUSTOM_DESCRIPTION), + // one identity with the program's own Symbol.for calls of that name. + if (isInspectCustomAccess(lowerer, key)) { + let e = key; + while (ts.isParenthesizedExpression(e)) e = e.expression; + const sym = lowerer.checker.getSymbolAtLocation((e as ts.PropertyAccessExpression).name); + return sym ? symbolKeyOf(lowerer, sym, true, INSPECT_CUSTOM_DESCRIPTION) : null; + } if ( !ts.isIdentifier(key) && (!ts.isPropertyAccessExpression(key) || key.questionDotToken || !ts.isIdentifier(key.name)) diff --git a/packages/compiler/src/frontend/type-mapper.ts b/packages/compiler/src/frontend/type-mapper.ts index aef61c743..3db647f64 100644 --- a/packages/compiler/src/frontend/type-mapper.ts +++ b/packages/compiler/src/frontend/type-mapper.ts @@ -2955,6 +2955,30 @@ function mapTypeInner(type: ts.Type, ctx: TypeMapperCtx): IrType | null { // spawn consumes a presence-sensitive option bag, including optional // stdio and env values. Keep it checked-dynamic for native normalization. if (isStdlibInterface("SpawnOptions")) return DYN; + // util.inspect as a value (a custom inspect method's third parameter is + // `typeof inspect`): the runtime's inspect function, called through the + // checked-dynamic boundary. + if ( + psym !== undefined && + ts.symbolName(psym) === "inspect" && + psym.declarations.some( + (d) => + ts.isFunctionDeclaration(d) && + ctx.isStdlibFile(ts.getSourceFileOfNode(d)) && + isDeclaredInAmbientModule(d, "util"), + ) + ) + return DYN; + // util.inspect's options bag, which inspect and formatWithOptions read + // at run time (its sorted option can be a comparator function), and the + // options custom inspect functions receive. + if ( + (isStdlibInterface("InspectOptions") || isStdlibInterface("InspectOptionsStylized")) && + psym!.declarations.some( + (d) => ts.isInterfaceDeclaration(d) && isDeclaredInAmbientModule(d, "util"), + ) + ) + return DYN; // dns.LookupAllOptions (`all: true`, the overload whose callback takes // an address list): a lookup option bag dns.lookup reads at run time. // It rides checked-dynamic storage, so the lookup function type that diff --git a/packages/compiler/src/ir/builtin-effects.ts b/packages/compiler/src/ir/builtin-effects.ts index eb3f8c6ab..d676b05dd 100644 --- a/packages/compiler/src/ir/builtin-effects.ts +++ b/packages/compiler/src/ir/builtin-effects.ts @@ -180,6 +180,17 @@ export const MAY_THROW_LIB_FNS: ReadonlySet = new Set([ "num.toStringRadix", "num.toStringRadixF64", "insp.jsonDyn", + // The runtime formatter calls toString, valueOf and toJSON, and refuses + // renderings it has no lowering for. + "insp.format", + "insp.formatWithOptions", + "insp.inspect", + // The options object reads the caller's options (getters included), and + // the checked-dynamic walk calls custom inspect functions. + "insp.customOptions", + "insp.customResult", + "insp.dyn", + "insp.dynS", // diagnostics_channel: publish runs subscribers synchronously (a throw // propagates — the documented divergence from triggerUncaughtException); // the subscribe/unsubscribe forms throw Node's ERR_INVALID_ARG_TYPE for diff --git a/packages/compiler/src/ir/builtin-signatures.ts b/packages/compiler/src/ir/builtin-signatures.ts index 8edb223b3..6e07e1467 100644 --- a/packages/compiler/src/ir/builtin-signatures.ts +++ b/packages/compiler/src/ir/builtin-signatures.ts @@ -4370,8 +4370,43 @@ export const LIB_FN_SIGS = defineLibFnSignatures({ "insp.errorCause": { argTypes: [null, F64, F64], result: STRING }, "insp.dyn": { argTypes: [DYN, F64, F64], result: STRING }, "insp.dynS": { argTypes: [DYN, F64], result: STRING }, + /** util.format over a dyn array of its arguments, and + * util.formatWithOptions(options, pack): the runtime formatter for format + * strings known at run time and spread arguments. */ + "insp.format": { argTypes: [DYN], result: STRING }, + /** insp.format over arguments that are all strings, numbers, booleans, + * null or undefined, without the checked-dynamic walk. */ + "insp.formatPrimitives": { argTypes: [DYN], result: STRING }, + "insp.formatWithOptions": { argTypes: [DYN, DYN], result: STRING }, + /** util.inspect(value, options, depth, colors) with the call's argument + * count: the runtime reads the options and renders the checked-dynamic + * view. */ + "insp.inspect": { argTypes: [DYN, DYN, DYN, DYN, F64], result: STRING }, + /** A [util.inspect.custom] method's call: whether ctx.customInspect is + * set, its depth argument (the remaining depth at recursion r of depth + * d), its options object and util.inspect, then its result rendered: a + * string re-indented at the current level, or a checked-dynamic value + * (customIsSelf tests for the instance itself). customInspect sets the + * flag for the renders that follow and answers the previous value, and + * depthKind does the same for how an unbounded depth was given (0 + * Infinity, 1 null, 2 undefined). */ + "insp.customEnabled": { argTypes: [], result: BOOL }, + /** Registers a class's compiled custom inspect function under its + * instances' typed-ref type key, for the checked-dynamic walk. */ + "insp.registerCustom": { argTypes: [STRING, DYN], result: VOID }, + "insp.customInspect": { argTypes: [BOOL], result: BOOL }, + "insp.depthKind": { argTypes: [F64], result: F64 }, + "insp.customDepth": { argTypes: [F64, F64], result: DYN }, + "insp.customOptions": { argTypes: [F64], result: DYN }, + "insp.inspectFunction": { argTypes: [], result: DYN }, + "insp.customText": { argTypes: [STRING], result: STRING }, + "insp.customIsSelf": { argTypes: [DYN, DYN], result: BOOL }, + "insp.customResult": { argTypes: [DYN, F64, F64], result: STRING }, + /** Whether a runtime format string has a '%' before its last character. */ + "insp.formatSpecifies": { argTypes: [STRING], result: BOOL }, + "insp.formatSpecifiesDyn": { argTypes: [DYN], result: BOOL }, "insp.begin": { argTypes: [F64], result: VOID }, - /** util.inspect's breakLength, compact (0 for false) and sorted for the + /** util.inspect's breakLength, compact (-1 for false) and sorted for the * renders that follow; the call site restores 80, 3 and false afterwards. */ "insp.layout": { argTypes: [F64, F64, BOOL], result: VOID }, "insp.entry": { argTypes: [STRING, BOOL], result: VOID }, diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7520-util-format-runtime.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7520-util-format-runtime.ts.json new file mode 100644 index 000000000..825fd9882 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7520-util-format-runtime.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7520-util-format-runtime.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7521-util-format-runtime.cjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7521-util-format-runtime.cjs.json new file mode 100644 index 000000000..0714d5c08 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7521-util-format-runtime.cjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7521-util-format-runtime.cjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7522-promisify-untyped.cjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7522-promisify-untyped.cjs.json new file mode 100644 index 000000000..bcd6b54c6 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7522-promisify-untyped.cjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7522-promisify-untyped.cjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7523-promisify-untyped.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7523-promisify-untyped.ts.json new file mode 100644 index 000000000..20e96495a --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7523-promisify-untyped.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7523-promisify-untyped.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7524-inspect-custom.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7524-inspect-custom.ts.json new file mode 100644 index 000000000..4f5a3d007 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7524-inspect-custom.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7524-inspect-custom.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7525-inspect-custom.cjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7525-inspect-custom.cjs.json new file mode 100644 index 000000000..dddad2358 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7525-inspect-custom.cjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7525-inspect-custom.cjs" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7526-inspect-options-runtime.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7526-inspect-options-runtime.ts.json new file mode 100644 index 000000000..70b202903 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7526-inspect-options-runtime.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7526-inspect-options-runtime.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/diagnostics/inspect-custom-fences.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/diagnostics/inspect-custom-fences.ts.json new file mode 100644 index 000000000..dff7b7d6b --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/diagnostics/inspect-custom-fences.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/diagnostics/inspect-custom-fences.ts" + ], + "diags": [] +} diff --git a/packages/runtime/src/scr_console_native.c b/packages/runtime/src/scr_console_native.c index 8c7467127..492d57d82 100644 --- a/packages/runtime/src/scr_console_native.c +++ b/packages/runtime/src/scr_console_native.c @@ -523,7 +523,11 @@ ScrStr *scr_console_dir_dyn(ScrDyn *value, double depth) { scr_throw_error_msg_code(SCR_ERR_ERROR, message, sizeof message - 1, "SC2020"); return scr_str_new("", 0); } - return scr_insp_dyn(value, 0, depth); + /* console.dir is inspect with customInspect: false. */ + bool custom = scr_insp_custom_inspect(false); + ScrStr *out = scr_insp_dyn(value, 0, depth); + scr_insp_custom_inspect(custom); + return out; } /* ── console.table ─────────────────────────────────────────────────── diff --git a/packages/runtime/src/scr_dyn.h b/packages/runtime/src/scr_dyn.h index 74b710dc9..fdb114535 100644 --- a/packages/runtime/src/scr_dyn.h +++ b/packages/runtime/src/scr_dyn.h @@ -863,6 +863,10 @@ ScrDyn *scr_dyn_open_iterator(const ScrDyn *value, int *kind, bool generators); * for-of protocol reads it. +1, or NULL with the error pending. */ ScrDyn *scr_dyn_iterator_result_get(const ScrDyn *result, const char *key, size_t len); void scr_dyn_install_iterator_symbol(ScrDyn *key); +/* Symbol.for('nodejs.util.inspect.custom'): the symbol unit installs it + * when the registry creates it; NULL (borrowed) until then. */ +void scr_dyn_install_inspect_custom_symbol(ScrDyn *key); +ScrDyn *scr_dyn_inspect_custom_symbol(void); void scr_dyn_install_async_iterator_symbol(ScrDyn *key); void scr_dyn_install_async_dispose_symbol(ScrDyn *key); /* True when key is Symbol.asyncDispose (registered by the symbol unit when diff --git a/packages/runtime/src/scr_inspect.c b/packages/runtime/src/scr_inspect.c index a4d124ac3..2c7951103 100644 --- a/packages/runtime/src/scr_inspect.c +++ b/packages/runtime/src/scr_inspect.c @@ -152,6 +152,154 @@ static size_t insp_width(const char *data, size_t len) { return width; } +/* ── colors (stylizeWithColor) ───────────────────────────────────────── */ + +/* ctx.colors for the render in progress: the checked-dynamic walk wraps + * each styled leaf in inspect.colors' SGR codes for its inspect.styles + * style, and the layout measures entries without the codes. Static + * renderings never run with it set. */ +static SCR_TL bool g_colors; + +typedef enum { + INSP_SPECIAL, + INSP_NUMBER, + INSP_BIGINT, + INSP_BOOLEAN, + INSP_UNDEFINED, + INSP_NULL, + INSP_STRING, + INSP_SYMBOL, + INSP_DATE, +} InspStyle; + +/* inspect.styles mapped through inspect.colors: [open, close]. */ +static const unsigned char insp_style_codes[][2] = { + {36, 39}, /* special: cyan */ + {33, 39}, /* number: yellow */ + {33, 39}, /* bigint: yellow */ + {33, 39}, /* boolean: yellow */ + {90, 39}, /* undefined: grey */ + {1, 22}, /* null: bold */ + {32, 39}, /* string: green */ + {32, 39}, /* symbol: green */ + {35, 39}, /* date: magenta */ +}; + +static void insp_style_open(InspBuf *b, InspStyle style) { + char code[8]; + int n = snprintf(code, sizeof code, "\x1b[%um", insp_style_codes[style][0]); + ib_bytes(b, code, (size_t)n); +} + +static void insp_style_close(InspBuf *b, InspStyle style) { + char code[8]; + int n = snprintf(code, sizeof code, "\x1b[%um", insp_style_codes[style][1]); + ib_bytes(b, code, (size_t)n); +} + +/* ctx.stylize(text, style): +1; moves `text` in. */ +static ScrStr *insp_styled(ScrStr *text, InspStyle style) { + if (!g_colors) return text; + InspBuf b = {0}; + insp_style_open(&b, style); + ib_str(&b, text); + insp_style_close(&b, style); + scr_str_release(text); + return ib_take(&b); +} + +static ScrStr *insp_styled_lit(const char *text, size_t len, InspStyle style) { + return insp_styled(scr_str_new(text, len), style); +} + +/* The SC2020 refusal for colored renderings of values whose styles this + * runtime does not produce; answers an empty string. */ +static ScrStr *insp_colors_fence(const char *what) { + InspBuf b = {0}; + ib_cstr(&b, "colored inspect output of "); + ib_cstr(&b, what); + ib_cstr(&b, " has no lowering"); + ScrStr *message = ib_take(&b); + scr_throw_error_msg_code(SCR_ERR_ERROR, message->data, message->len, "SC2020"); + scr_str_release(message); + return scr_str_new("", 0); +} + +static ScrStr *insp_colors_fence_values(const char *cls) { + InspBuf b = {0}; + ib_cstr(&b, cls); + ib_cstr(&b, " values"); + ScrStr *what = ib_take(&b); + ScrStr *out = insp_colors_fence(what->data); + scr_str_release(what); + return out; +} + +/* The SGR sequence Node's removeColors strips (\x1b[<1 or 2 digits>m) at + * data[i]: its length, or 0. */ +static size_t insp_sgr_at(const char *data, size_t len, size_t i) { + if (data[i] != '\x1b' || i + 3 >= len || data[i + 1] != '[') return 0; + size_t j = i + 2, digits = 0; + while (j < len && digits < 2 && data[j] >= '0' && data[j] <= '9') j++, digits++; + return digits > 0 && j < len && data[j] == 'm' ? j + 1 - i : 0; +} + +/* A control sequence stripVTControlCharacters removes at data[i] (CSI + * sequences after ESC [ or C1 CSI, and OSC sequences ended by BEL): its + * length, or 0. */ +static size_t insp_vt_at(const char *data, size_t len, size_t i) { + const unsigned char *p = (const unsigned char *)data; + size_t j; + if (p[i] == 0x1b && i + 1 < len && p[i + 1] == '[') j = i + 2; + else if (p[i] == 0xc2 && i + 1 < len && p[i + 1] == 0x9b) j = i + 2; + else if (p[i] == 0x1b && i + 1 < len && p[i + 1] == ']') { + for (j = i + 2; j < len; j++) + if (p[j] == 0x07) return j + 1 - i; + return 0; + } else return 0; + while (j < len && ((p[j] >= '0' && p[j] <= '9') || p[j] == ';' || p[j] == '?')) j++; + if (j >= len) return 0; + unsigned char c = p[j]; + bool final = (c >= '0' && c <= '9') || (c >= 'A' && c <= 'P') || (c >= 'R' && c <= 'T') || + c == 'Z' || c == 'c' || (c >= 'f' && c <= 'n') || c == 'q' || c == 'r' || + c == 't' || c == 'y' || c == '=' || c == '>' || c == '<' || c == '~'; + return final ? j + 1 - i : 0; +} + +/* An entry's display width for the grid (getStringWidth(entry, + * ctx.colors)) and its UTF-16 length for the break-length test (Node's + * removeColors under ctx.colors). */ +static size_t insp_entry_width(const ScrStr *s) { + if (!g_colors) return insp_width(s->data, s->len); + size_t width = 0; + for (size_t i = 0; i < s->len;) { + size_t skip = insp_vt_at(s->data, s->len, i); + if (skip) { + i += skip; + continue; + } + uint32_t cp = insp_cp_at(s->data, s->len, &i); + if (insp_full_width(cp)) width += 2; + else if (!insp_zero_width(cp)) width += 1; + } + return width; +} + +static size_t insp_entry_len(const ScrStr *s) { + if (!g_colors) return insp_utf16_len(s->data, s->len); + size_t units = 0; + for (size_t i = 0; i < s->len;) { + size_t skip = insp_sgr_at(s->data, s->len, i); + if (skip) { + i += skip; + continue; + } + uint32_t cp = insp_cp_at(s->data, s->len, &i); + units += cp >= 0x10000 ? 2 : 1; + } + return units; +} + /* ── number rendering ────────────────────────────────────────────────── */ /* formatNumber: JS ToString EXCEPT -0 renders as '-0'. */ @@ -245,12 +393,12 @@ static SCR_TL size_t g_frames_cap; static SCR_TL size_t g_indent; /* ctx.indentationLvl */ static SCR_TL bool g_no_break; /* breakLength: Infinity (console.table cells) */ static SCR_TL double g_break_length = 80; /* ctx.breakLength */ -static SCR_TL double g_compact = 3; /* ctx.compact: a number, 0 for false */ +static SCR_TL double g_compact = 3; /* ctx.compact: a number, -1 for false */ static SCR_TL bool g_sorted; /* ctx.sorted === true */ static SCR_TL double g_cur_depth; /* ctx.currentDepth (last-entered composite) */ /* util.inspect's breakLength, compact and sorted for the renders that - * follow (compact false is 0); the call site restores 80, 3 and false + * follow (compact false is -1); the call site restores 80, 3 and false * after its render. */ void scr_insp_layout(double break_length, double compact, bool sorted) { g_break_length = break_length; @@ -369,7 +517,7 @@ void scr_insp_seen_push(const void *v) { ScrStr *scr_insp_circular(double id) { char buf[32]; int n = snprintf(buf, sizeof buf, "[Circular *%.0f]", id); - return scr_str_new(buf, (size_t)n); + return insp_styled_lit(buf, (size_t)n, INSP_SPECIAL); } /* Pops the value's seen entry after its frame closed. scr_insp_end already @@ -403,7 +551,7 @@ static void insp_group(InspFrame *f, bool trailing_more) { size_t *data_len = malloc((output_len ? output_len : 1) * sizeof(size_t)); if (!data_len) insp_oom(); for (size_t i = 0; i < output_len; i++) { - size_t w = insp_width(f->items[i]->data, f->items[i]->len); + size_t w = insp_entry_width(f->items[i]); data_len[i] = w; total_length += (double)w + 2; if ((double)w > max_length) max_length = (double)w; @@ -494,7 +642,7 @@ static bool insp_below_break_length(InspFrame *f, size_t start, const ScrStr *ba size_t total = f->len + start; if ((double)(total + f->len) > g_break_length) return false; for (size_t i = 0; i < f->len; i++) { - total += insp_utf16_len(f->items[i]->data, f->items[i]->len); + total += insp_entry_len(f->items[i]); if ((double)total > g_break_length) return false; } return base->len == 0 || !insp_contains_byte(base->data, base->len, '\n'); @@ -520,12 +668,16 @@ ScrStr *scr_insp_end(ScrStr *base, ScrStr *b0, ScrStr *b1, double recurse, bool int ref_id = f->seen ? insp_circ_id(f->seen) : 0; if (ref_id) { char buf[32]; - int n = snprintf(buf, sizeof buf, base->len ? " " : "", ref_id); - size_t len = (size_t)n + base->len; - ref_base = scr_str_alloc_raw(len, len); - memcpy(ref_base->data, buf, (size_t)n); - memcpy(ref_base->data + n, base->data, base->len); - ref_base->data[len] = '\0'; + int n = snprintf(buf, sizeof buf, "", ref_id); + InspBuf rb = {0}; + if (g_colors) insp_style_open(&rb, INSP_SPECIAL); + ib_bytes(&rb, buf, (size_t)n); + if (g_colors) insp_style_close(&rb, INSP_SPECIAL); + if (base->len) { + ib_char(&rb, ' '); + ib_str(&rb, base); + } + ref_base = ib_take(&rb); base = ref_base; } size_t entries = f->len; @@ -622,6 +774,19 @@ void scr_insp_nested_begin(void) { g_ncirc = 0; } +/* nested_end's restore alone: the caller's context returns. */ +void scr_insp_nested_restore(void) { + InspSaved *sv = &g_saved[--g_nsaved]; + free(g_seen); + g_indent = sv->indent; + g_cur_depth = sv->cur_depth; + g_seen = sv->seen; + g_nseen = sv->nseen; + g_seen_cap = sv->seen_cap; + memcpy(g_circ, sv->circ, sizeof g_circ); + g_ncirc = sv->ncirc; +} + ScrStr *scr_insp_nested_end(ScrStr *prefix, ScrStr *inner) { InspSaved *sv = &g_saved[--g_nsaved]; free(g_seen); @@ -717,13 +882,17 @@ ScrStr *scr_insp_str(ScrStr *s) { ib_cstr(&out, " +\n"); ib_spaces(&out, g_indent + 2); } + if (g_colors) insp_style_open(&out, INSP_STRING); insp_quote_into(&out, v + chunk_start, end - chunk_start); + if (g_colors) insp_style_close(&out, INSP_STRING); chunk_start = end; first = false; } } } else { + if (g_colors) insp_style_open(&out, INSP_STRING); insp_quote_into(&out, v, vlen); + if (g_colors) insp_style_close(&out, INSP_STRING); } if (trailer[0]) ib_cstr(&out, trailer); free(val.data); @@ -1007,7 +1176,9 @@ static void insp_key_into(InspBuf *b, const char *key, size_t len) { if (bare) { ib_bytes(b, key, len); } else { + if (g_colors) insp_style_open(b, INSP_STRING); insp_quote_into(b, key, len); + if (g_colors) insp_style_close(b, INSP_STRING); } } @@ -1330,7 +1501,7 @@ static ScrStr *scr_insp_dyn_collection(ScrDyn *d, double recurse, double depth, ib_char(&out, '['); ib_cstr(&out, label); ib_char(&out, ']'); - return ib_take(&out); + return insp_styled(ib_take(&out), INSP_SPECIAL); } ScrStr *base = ib_take(&head); scr_insp_begin(recurse + 1); @@ -1395,9 +1566,9 @@ static ScrStr *scr_insp_obj(ScrDyn *d, double recurse, double depth, const char : scr_str_new("{}", 2); } if (recurse > depth) { - if (cls) return scr_insp_class_tag(cls, NULL); - return d->null_proto ? scr_str_new("[Object: null prototype]", 24) - : scr_str_new("[Object]", 8); + if (cls) return insp_styled(scr_insp_class_tag(cls, NULL), INSP_SPECIAL); + return d->null_proto ? insp_styled_lit("[Object: null prototype]", 24, INSP_SPECIAL) + : insp_styled_lit("[Object]", 8, INSP_SPECIAL); } scr_insp_begin(recurse + 1); for (size_t i = 0; i < d->v.obj.len; i++) { @@ -1412,6 +1583,7 @@ static ScrStr *scr_insp_obj(ScrDyn *d, double recurse, double depth, const char ScrStr *entry = ib_take(&eb); scr_insp_entry(entry, false); scr_str_release(entry); + if (scr_exc_pending()) break; } for (size_t i = 0; i < nsymbols; i++) { ScrDynEntry *ent = scr_insp_symbol_entry(d, i); @@ -1455,12 +1627,12 @@ ScrStr *scr_insp_typed_array(ScrBytes *b, double recurse, double depth) { if (recurse > depth) { char tag[32]; int n = snprintf(tag, sizeof tag, "[%s]", scr_bytes_elem_name(b->elem)); - return scr_str_new(tag, (size_t)n); + return insp_styled_lit(tag, (size_t)n, INSP_SPECIAL); } scr_insp_begin(recurse + 1); size_t shown = b->len < 100 ? b->len : 100; for (size_t i = 0; i < shown; i++) { - ScrStr *s = scr_insp_f64(scr_bytes_get(b, (double)i)); + ScrStr *s = insp_styled(scr_insp_f64(scr_bytes_get(b, (double)i)), INSP_NUMBER); scr_insp_entry(s, true); scr_str_release(s); } @@ -1480,27 +1652,374 @@ ScrStr *scr_insp_typed_array(ScrBytes *b, double recurse, double depth) { return out; } +/* ── custom inspect functions (util.inspect.custom) ────────────────────── + * formatValue's hook: an object whose [util.inspect.custom] member is a + * function (other than util.inspect itself, on an object that is not its + * constructor's prototype) renders as that function's result. It runs + * before the circular and depth checks, with the remaining depth + * (ctx.depth - recurseTimes), the options object getUserOptions builds and + * util.inspect. A string result is re-indented at the current level; the + * object itself renders normally; any other value renders in its place. + * Class instances held as checked-dynamic values have compiled methods + * only, so one whose class defines the method throws SC2020. */ +static SCR_TL bool g_custom_inspect = true; /* ctx.customInspect */ +/* ctx.depth as the options gave it, for the custom functions' depth + * argument and options when the render's depth is unbounded: 0 the number + * g_depth_number (Infinity or NaN), 1 null, 2 undefined. */ +static SCR_TL int g_depth_kind = 1; +static SCR_TL double g_depth_number; +/* The options object that held keys inspect does not read: getUserOptions + * spreads it into the options the custom functions receive (borrowed for + * the render). */ +static SCR_TL const ScrDyn *g_user_options; + +double scr_insp_depth_kind(double kind) { + double previous = g_depth_kind; + g_depth_kind = (int)kind; + g_depth_number = INFINITY; + return previous; +} + +bool scr_insp_custom_inspect(bool enabled) { + bool previous = g_custom_inspect; + g_custom_inspect = enabled; + return previous; +} + +/* The style inspect.styles maps a styleType to, as [open, close] codes: + * the styles above plus regexp (red) and module (underline). false when + * the type has no style. */ +static bool insp_style_of(const ScrStr *type, unsigned *open, unsigned *close) { + static const struct { + const char *name; + unsigned open, close; + } styles[] = { + {"special", 36, 39}, {"number", 33, 39}, {"bigint", 33, 39}, {"boolean", 33, 39}, + {"undefined", 90, 39}, {"null", 1, 22}, {"string", 32, 39}, {"symbol", 32, 39}, + {"date", 35, 39}, {"regexp", 31, 39}, {"module", 4, 24}, + }; + for (size_t i = 0; i < sizeof styles / sizeof *styles; i++) { + if (type->len == strlen(styles[i].name) && memcmp(type->data, styles[i].name, type->len) == 0) { + *open = styles[i].open; + *close = styles[i].close; + return true; + } + } + return false; +} + +/* options.stylize: stylizeWithColor under colors (the text in its style's + * codes, or unchanged), stylizeNoColor otherwise (the argument itself). + * caps[0] holds whether colors apply. */ +static ScrDyn *insp_stylize_call(ScrClosure *closure, ScrDyn *const *args, size_t argc) { + ScrDyn *colors = scr_box_get_ref(closure->caps[0]); + bool colored = colors && scr_dyn_truthy(colors); + scr_dyn_release(colors); + ScrDyn *text = argc > 0 ? args[0] : (ScrDyn *)scr_dyn_undefined(); + if (!colored) return scr_dyn_retain(text); + ScrStr *str = scr_dyn_string_coerce_js(text); + if (!str) return NULL; + unsigned open, close; + ScrStr *type = argc > 1 && args[1]->kind == SCR_DYN_STR ? args[1]->v.str : NULL; + if (!type || !insp_style_of(type, &open, &close)) { + ScrDyn *out = scr_dyn_new_str(str); + scr_str_release(str); + return out; + } + InspBuf b = {0}; + char code[8]; + int n = snprintf(code, sizeof code, "\x1b[%um", open); + ib_bytes(&b, code, (size_t)n); + ib_str(&b, str); + n = snprintf(code, sizeof code, "\x1b[%um", close); + ib_bytes(&b, code, (size_t)n); + scr_str_release(str); + ScrStr *styled = ib_take(&b); + ScrDyn *out = scr_dyn_new_str(styled); + scr_str_release(styled); + return out; +} + +static ScrDyn *insp_inspect_call(ScrClosure *closure, ScrDyn *const *args, size_t argc); + +/* Whether `value` is the stylize function getUserOptions built for these + * colors. */ +static bool insp_own_stylize(const ScrDyn *value, bool colors) { + if (value->kind != SCR_DYN_FUNC || value->v.fn.thunk != insp_stylize_call) return false; + ScrDyn *flag = scr_box_get_ref(value->v.fn.clo->caps[0]); + bool colored = flag && scr_dyn_truthy(flag); + scr_dyn_release(flag); + return colored == colors; +} + +/* util.inspect as a function value (the custom functions' third + * argument): a fresh inspect context over the call's arguments, the + * caller's restored afterwards. One function per thread, so the argument + * compares equal to util.inspect. +1. */ +static SCR_TL ScrDyn *g_inspect_function; + +static void insp_inspect_function_cleanup(void) { + scr_dyn_release(g_inspect_function); + g_inspect_function = NULL; +} + +static ScrDyn *insp_inspect_value(void) { + if (!g_inspect_function) { + ScrClosure *closure = scr_closure_new(NULL, 0); + g_inspect_function = + scr_dyn_new_func(closure, insp_inspect_call, 2, "native:util.inspect", "inspect"); + scr_atexit(insp_inspect_function_cleanup); + } + return scr_dyn_retain(g_inspect_function); +} + +static void insp_opt_set(ScrDyn *options, const char *key, ScrDyn *value /* moves in */) { + scr_dyn_obj_set(options, key, strlen(key), value); +} + +/* getUserOptions(ctx) for the render in progress at `depth`. +1. */ +static ScrDyn *insp_user_options(double depth) { + ScrDyn *options = scr_dyn_new_obj(); + ScrClosure *closure = scr_closure_new(NULL, 1); + closure->caps[0] = scr_box_new_obj(scr_dyn_retain_v, scr_dyn_release_v, scr_dyn_trace_v); + scr_box_set_ref(closure->caps[0], scr_dyn_new_bool(g_colors)); + insp_opt_set(options, "stylize", + scr_dyn_new_func(closure, insp_stylize_call, 2, "native:util.stylize", + g_colors ? "stylizeWithColor" : "stylizeNoColor")); + insp_opt_set(options, "showHidden", scr_dyn_new_bool(false)); + insp_opt_set(options, "depth", + isfinite(depth) ? scr_dyn_new_num(depth) + : g_depth_kind == 0 ? scr_dyn_new_num(g_depth_number) + : g_depth_kind == 1 ? scr_dyn_new_null() + : scr_dyn_retain(scr_dyn_undefined())); + insp_opt_set(options, "colors", scr_dyn_new_bool(g_colors)); + insp_opt_set(options, "customInspect", scr_dyn_new_bool(true)); + insp_opt_set(options, "showProxy", scr_dyn_new_bool(false)); + insp_opt_set(options, "maxArrayLength", scr_dyn_new_num(100)); + insp_opt_set(options, "maxStringLength", scr_dyn_new_num(10000)); + insp_opt_set(options, "breakLength", scr_dyn_new_num(g_break_length)); + insp_opt_set(options, "compact", g_compact < 0 ? scr_dyn_new_bool(false) : scr_dyn_new_num(g_compact)); + insp_opt_set(options, "sorted", scr_dyn_new_bool(g_sorted)); + insp_opt_set(options, "getters", scr_dyn_new_bool(false)); + insp_opt_set(options, "numericSeparator", scr_dyn_new_bool(false)); + if (g_user_options && g_user_options->kind == SCR_DYN_OBJ) { + for (size_t i = 0; i < g_user_options->v.obj.len; i++) { + const ScrDynEntry *ent = &g_user_options->v.obj.entries[i]; + if (!ent->enumerable) continue; + ScrDyn *value = scr_dyn_obj_read((ScrDyn *)g_user_options, ent->key, ent->key_len); + if (!value) { + scr_dyn_release(options); + return NULL; + } + scr_dyn_obj_set(options, ent->key, ent->key_len, value); + } + } + return options; +} + +/* The custom inspect functions of the program's classes, by the type key + * of their instances' typed refs: compiled functions (self, depth, + * options, inspect) that call the instance's method and answer its + * result as a checked-dynamic value. The program registers them before + * it runs. */ +typedef struct { + char *type_key; + ScrDyn *fn; +} InspCustomClass; + +static SCR_TL InspCustomClass *g_custom_classes; +static SCR_TL size_t g_ncustom_classes; + +static void insp_custom_classes_cleanup(void) { + for (size_t i = 0; i < g_ncustom_classes; i++) { + free(g_custom_classes[i].type_key); + scr_dyn_release(g_custom_classes[i].fn); + } + free(g_custom_classes); + g_custom_classes = NULL; + g_ncustom_classes = 0; +} + +void scr_insp_register_custom(ScrStr *type_key, ScrDyn *fn) { + if (g_ncustom_classes == 0) scr_atexit(insp_custom_classes_cleanup); + InspCustomClass *grown = + realloc(g_custom_classes, (g_ncustom_classes + 1) * sizeof *g_custom_classes); + if (!grown) insp_oom(); + g_custom_classes = grown; + char *key = malloc(type_key->len + 1); + if (!key) insp_oom(); + memcpy(key, type_key->data, type_key->len); + key[type_key->len] = '\0'; + g_custom_classes[g_ncustom_classes].type_key = key; + g_custom_classes[g_ncustom_classes].fn = scr_dyn_retain(fn); + g_ncustom_classes++; +} + +static ScrDyn *insp_custom_class_fn(const ScrDyn *d) { + for (size_t i = 0; i < g_ncustom_classes; i++) + if (strcmp(g_custom_classes[i].type_key, d->v.typed_ref.type_key) == 0) + return g_custom_classes[i].fn; + return NULL; +} + +/* The [util.inspect.custom] result for an object kind, or NULL when the + * value renders normally (*thrown reports a pending exception). +1. */ +static ScrStr *insp_custom(ScrDyn *d, double recurse, double depth, bool *thrown) { + *thrown = false; + if (!g_custom_inspect) return NULL; + ScrDyn *fn; + bool compiled = d->kind == SCR_DYN_TYPED_REF; + if (compiled) { + /* A class instance: its class's compiled function, when it has a + * custom inspect method. A class whose method has no compiled call + * (the compiler refuses such classes where it renders them) refuses + * here too. */ + fn = insp_custom_class_fn(d); + if (!fn) { + ScrStr *name = scr_str_new("%symbol:Symbol(nodejs.util.inspect.custom)", 42); + bool declared = scr_dyn_typed_ref_class_has(d, name); + scr_str_release(name); + if (!declared) return NULL; + static const char message[] = + "util.inspect of this class instance's [util.inspect.custom] method has no lowering"; + scr_throw_error_msg_code(SCR_ERR_ERROR, message, sizeof message - 1, "SC2020"); + *thrown = true; + return NULL; + } + fn = scr_dyn_retain(fn); + } else { + ScrDyn *key = scr_dyn_inspect_custom_symbol(); + if (!key) return NULL; + if (d->kind != SCR_DYN_OBJ && d->kind != SCR_DYN_ARR && d->kind != SCR_DYN_FUNC) return NULL; + fn = scr_dyn_reflect_get(d, key, d); + if (!fn) { + *thrown = true; + return NULL; + } + } + if (fn->kind != SCR_DYN_FUNC || fn->v.fn.thunk == insp_inspect_call) { + scr_dyn_release(fn); + return NULL; + } + /* A constructor's prototype object renders normally. */ + ScrDyn *ctor = !compiled && d->kind == SCR_DYN_OBJ ? scr_dyn_obj_read(d, "constructor", 11) : NULL; + if (ctor && ctor->kind == SCR_DYN_FUNC) { + ScrDyn *prototype = scr_dyn_fn_get(ctor, "prototype", 9); + bool own_prototype = prototype == d; + scr_dyn_release(prototype); + if (own_prototype) { + scr_dyn_release(ctor); + scr_dyn_release(fn); + return NULL; + } + } + scr_dyn_release(ctor); + ScrDyn *args[3]; + /* ctx.depth - recurseTimes, or null for a null depth. */ + args[0] = scr_insp_custom_depth(recurse, depth); + args[1] = insp_user_options(depth); + if (!args[1]) { + scr_dyn_release(args[0]); + scr_dyn_release(fn); + *thrown = true; + return NULL; + } + args[2] = insp_inspect_value(); + ScrDyn *ret; + if (compiled) { + ScrDyn *call[4] = {d, args[0], args[1], args[2]}; + ret = scr_dyn_call(fn, call, 4, "[nodejs.util.inspect.custom]"); + } else { + scr_dyn_this_push_dyn(d); + ret = scr_dyn_call(fn, args, 3, "[nodejs.util.inspect.custom]"); + scr_dyn_this_pop(); + } + for (size_t i = 0; i < 3; i++) scr_dyn_release(args[i]); + scr_dyn_release(fn); + if (!ret) { + *thrown = true; + return NULL; + } + if (ret == d || scr_insp_custom_is_self(ret, d)) { + scr_dyn_release(ret); + return NULL; + } + ScrStr *out; + if (ret->kind == SCR_DYN_STR) out = scr_insp_custom_text(ret->v.str); + else out = scr_insp_dyn(ret, recurse, depth); + scr_dyn_release(ret); + if (scr_exc_pending()) { + scr_str_release(out); + *thrown = true; + return NULL; + } + return out; +} + +bool scr_insp_custom_enabled(void) { return g_custom_inspect; } + +/* ctx.depth - recurseTimes, or null for a null depth (NaN for undefined). */ +ScrDyn *scr_insp_custom_depth(double recurse, double depth) { + if (isfinite(depth)) return scr_dyn_new_num(depth - recurse); + if (g_depth_kind == 0) return scr_dyn_new_num(g_depth_number - recurse); + return g_depth_kind == 1 ? scr_dyn_new_null() : scr_dyn_new_num(NAN); +} + +ScrDyn *scr_insp_custom_options(double depth) { return insp_user_options(depth); } + +ScrDyn *scr_insp_inspect_function(void) { return insp_inspect_value(); } + +bool scr_insp_custom_is_self(ScrDyn *result, ScrDyn *self) { + return result->kind == SCR_DYN_TYPED_REF && self->kind == SCR_DYN_TYPED_REF && + result->v.typed_ref.ptr == self->v.typed_ref.ptr; +} + +ScrStr *scr_insp_custom_result(ScrDyn *result, double recurse, double depth) { + if (result->kind == SCR_DYN_STR) return scr_insp_custom_text(result->v.str); + return scr_insp_dyn(result, recurse, depth); +} + +/* A custom function's string result: each newline followed by the + * current indentation. +1. */ +ScrStr *scr_insp_custom_text(ScrStr *text) { + if (!memchr(text->data, '\n', text->len)) return scr_str_retain(text); + InspBuf out = {0}; + for (size_t i = 0; i < text->len; i++) { + ib_char(&out, text->data[i]); + if (text->data[i] == '\n') ib_spaces(&out, g_indent); + } + return ib_take(&out); +} + ScrStr *scr_insp_dyn(ScrDyn *d, double recurse, double depth) { if (d->kind == SCR_DYN_PROXY) return scr_insp_dyn(d->v.proxy.target, recurse, depth); + if ((d->kind == SCR_DYN_TYPED_REF ? g_ncustom_classes > 0 : scr_dyn_inspect_custom_symbol() != NULL) && + (d->kind == SCR_DYN_OBJ || d->kind == SCR_DYN_ARR || d->kind == SCR_DYN_FUNC || + d->kind == SCR_DYN_TYPED_REF)) { + bool thrown; + ScrStr *custom = insp_custom(d, recurse, depth, &thrown); + if (custom) return custom; + if (thrown) return scr_str_new("", 0); + } switch (d->kind) { case SCR_DYN_NULL: - return scr_str_new("null", 4); + return insp_styled_lit("null", 4, INSP_NULL); case SCR_DYN_UNDEF: - return scr_str_new("undefined", 9); + return insp_styled_lit("undefined", 9, INSP_UNDEFINED); case SCR_DYN_BOOL: - return d->v.b ? scr_str_new("true", 4) : scr_str_new("false", 5); + return d->v.b ? insp_styled_lit("true", 4, INSP_BOOLEAN) : insp_styled_lit("false", 5, INSP_BOOLEAN); case SCR_DYN_BIGINT: - return scr_bigint_inspect(d->v.bigint); + return insp_styled(scr_bigint_inspect(d->v.bigint), INSP_BIGINT); case SCR_DYN_SYMBOL: - return d->v.symbol.render(d->v.symbol.value); + return insp_styled(d->v.symbol.render(d->v.symbol.value), INSP_SYMBOL); case SCR_DYN_NUM: - return scr_insp_f64(d->v.num); + return insp_styled(scr_insp_f64(d->v.num), INSP_NUMBER); case SCR_DYN_STR: return scr_insp_str(d->v.str); case SCR_DYN_ARR: { size_t len = d->v.arr.len; if (len == 0) return scr_str_new("[]", 2); - if (recurse > depth) return scr_str_new("[Array]", 7); + if (recurse > depth) return insp_styled_lit("[Array]", 7, INSP_SPECIAL); scr_insp_begin(recurse + 1); /* Node's maxArrayLength (100) counts output entries; a run of holes * is one `` entry (formatSpecialArray). */ @@ -1512,7 +2031,7 @@ ScrStr *scr_insp_dyn(ScrDyn *d, double recurse, double depth) { while (next < len && !presence[next]) next++; char hole[48]; int hn = snprintf(hole, sizeof hole, "<%zu empty item%s>", next - i, next - i == 1 ? "" : "s"); - ScrStr *s = scr_str_new(hole, (size_t)hn); + ScrStr *s = insp_styled_lit(hole, (size_t)hn, INSP_UNDEFINED); scr_insp_entry(s, false); scr_str_release(s); i = next; @@ -1522,6 +2041,7 @@ ScrStr *scr_insp_dyn(ScrDyn *d, double recurse, double depth) { ScrStr *s = scr_insp_dyn(item, recurse + 1, depth); scr_insp_entry(s, item->kind == SCR_DYN_NUM); scr_str_release(s); + if (scr_exc_pending()) break; } bool more = i < len; if (more) { @@ -1542,7 +2062,10 @@ ScrStr *scr_insp_dyn(ScrDyn *d, double recurse, double depth) { /* An Error carried as a checked-dynamic value (a caught error, an * Error stored in an `unknown` slot): the error's own rendering, * never the encoding's internal marker. */ - if (scr_dyn_obj_get(d, "%error", 6) != NULL) return scr_insp_dyn_error(d, recurse, depth); + if (scr_dyn_obj_get(d, "%error", 6) != NULL) { + if (g_colors) return insp_colors_fence("Error values"); + return scr_insp_dyn_error(d, recurse, depth); + } /* Object.create(null)'s dictionary: Node prefixes the rendering * with "[Object: null prototype]" (formatValue's constructor-less * base), the empty form included, and the beyond-depth answer IS @@ -1562,9 +2085,12 @@ ScrStr *scr_insp_dyn(ScrDyn *d, double recurse, double depth) { } else { ib_cstr(&out, "[Function (anonymous)]"); } - return ib_take(&out); + return insp_styled(ib_take(&out), INSP_SPECIAL); } case SCR_DYN_HANDLE: { + if (g_colors && d->v.handle.tag != SCR_DYNH_SET && d->v.handle.tag != SCR_DYNH_MAP && + d->v.handle.tag != SCR_DYNH_DATE) + return insp_colors_fence_values(scr_dyn_handle_cls(d)); if (d->v.handle.tag == SCR_DYNH_ARRAY_BUFFER || d->v.handle.tag == SCR_DYNH_SHARED_ARRAY_BUFFER) { return scr_insp_array_buffer(d->v.handle.ptr, scr_dyn_handle_cls(d), recurse, depth); @@ -1574,7 +2100,8 @@ ScrStr *scr_insp_dyn(ScrDyn *d, double recurse, double depth) { if (d->v.handle.tag == SCR_DYNH_WEAK_MAP || d->v.handle.tag == SCR_DYNH_WEAK_SET) return scr_insp_weak_collection(d->v.handle.tag == SCR_DYNH_WEAK_MAP ? "WeakMap" : "WeakSet", recurse, depth); - if (d->v.handle.tag == SCR_DYNH_DATE) return scr_insp_date(scr_dyn_native_date_value(d)); + if (d->v.handle.tag == SCR_DYNH_DATE) + return insp_styled(scr_insp_date(scr_dyn_native_date_value(d)), INSP_DATE); if (d->v.handle.tag == SCR_DYNH_ITERATOR) { ScrStr *iterator = scr_insp_native_iterator(d, recurse, depth); if (iterator) return iterator; @@ -1623,9 +2150,14 @@ ScrStr *scr_insp_dyn(ScrDyn *d, double recurse, double depth) { } case SCR_DYN_TYPED_REF: { + if (g_colors && scr_insp_is_generator(d)) return insp_colors_fence("generator objects"); if (scr_insp_is_generator(d)) return scr_insp_generator(d); const char *cls = NULL; ScrDyn *view = scr_event_ref_hooks ? scr_event_ref_hooks->inspect_view(d, &cls) : NULL; + if (view && g_colors) { + scr_dyn_release(view); + return insp_colors_fence_values(cls); + } if (view) { /* The handle rule above: the class name past the depth limit. */ if (recurse > depth) { @@ -1655,6 +2187,11 @@ ScrStr *scr_insp_dyn(ScrDyn *d, double recurse, double depth) { double time; ScrStr *date_class = materialized->kind == SCR_DYN_OBJ && scr_dyn_date_class_time(d, &time) ? scr_dyn_date_class_name(d) : NULL; + if (date_class && g_colors) { + scr_str_release(date_class); + scr_dyn_release(materialized); + return insp_colors_fence("Date subclass instances"); + } if (date_class) { /* An instance of a class declared `extends Date`: the class name and * the time as Date.prototype.toISOString formats it, then its own @@ -1715,6 +2252,7 @@ ScrStr *scr_insp_dyn(ScrDyn *d, double recurse, double depth) { } case SCR_DYN_BYTES: { ScrBytes *b = d->v.bytes; + if (b->is_data_view && g_colors) return insp_colors_fence("DataView values"); if (b->is_data_view) return scr_insp_data_view(b, recurse, depth); if (d->buffer) { /* Buffer-flavored (a stream chunk): Node's form. */ @@ -1734,6 +2272,535 @@ ScrStr *scr_insp_dyn_s(ScrDyn *d, double depth) { return scr_insp_dyn(d, 0, depth); } +/* ── util.format over checked-dynamic values ───────────────────────────── + * formatWithOptionsInternal over argument values whose types are known + * only at run time: util.format and util.formatWithOptions with a runtime + * format string or spread arguments, console output with a runtime format + * string, and debuglog's lines. The %-substitutions (%s %d %i %f %j %o %O + * %c %%) follow Node's argument cursor, then the remaining arguments join + * with spaces, strings verbatim and everything else inspected under the + * options. With colors, primitives take inspect's default styles; colored + * object renderings have no lowering. These entries call program code + * (toString, valueOf and toJSON), so unlike the rest of this file they + * can throw: NULL with the exception pending. */ +static void insp_fmt_fence(const ScrInspFormatOptions *o, const char *what) { + InspBuf b = {0}; + ib_cstr(&b, o->surface); + ib_char(&b, ' '); + ib_cstr(&b, what); + ScrStr *message = ib_take(&b); + scr_throw_error_msg_code(SCR_ERR_ERROR, message->data, message->len, "SC2020"); + scr_str_release(message); +} + +static bool insp_fmt_object(const ScrDyn *value) { + switch (value->kind) { + case SCR_DYN_OBJ: + case SCR_DYN_ARR: + case SCR_DYN_FUNC: + case SCR_DYN_BYTES: + case SCR_DYN_HANDLE: + case SCR_DYN_PROMISE: + case SCR_DYN_TYPED_REF: + case SCR_DYN_PROXY: + return true; + default: + return false; + } +} + +void scr_insp_format_defaults(ScrInspFormatOptions *o, const char *surface) { + o->surface = surface; + o->depth = 2; + o->depth_kind = 0; + o->depth_number = 2; + o->colors = false; + o->custom_inspect = true; + o->break_length = 80; + o->compact = 3; + o->sorted = false; + o->user_options = NULL; +} + +/* inspect(value, options) at the given depth, compact and colors, with + * the options' breakLength and sorted; the previous layout returns + * afterwards. +1, or NULL with the exception pending. */ +static ScrStr *insp_fmt_inspect(ScrDyn *value, const ScrInspFormatOptions *o, double depth, + double compact, bool colors) { + double break_length = g_break_length, saved_compact = g_compact; + bool sorted = g_sorted, saved_colors = g_colors, saved_custom = g_custom_inspect; + int saved_depth_kind = g_depth_kind; + double saved_depth_number = g_depth_number; + const ScrDyn *saved_user = g_user_options; + scr_insp_layout(o->break_length, compact, o->sorted); + g_colors = colors; + g_custom_inspect = o->custom_inspect; + g_depth_kind = o->depth_kind; + g_depth_number = o->depth_number; + g_user_options = o->user_options; + ScrStr *text = scr_insp_dyn(value, 0, depth); + g_colors = saved_colors; + g_custom_inspect = saved_custom; + g_depth_kind = saved_depth_kind; + g_depth_number = saved_depth_number; + g_user_options = saved_user; + scr_insp_layout(break_length, saved_compact, sorted); + if (scr_exc_pending()) { + scr_str_release(text); + return NULL; + } + return text; +} + +/* hasBuiltInToString's negative answer: a toString the program defined on + * the object or its prototype chain (1). Symbol-keyed members (where + * Symbol.toPrimitive would also count) are reported as unknown (-1), and a + * class instance whose class defines toString as -2: its methods are + * compiled for the class's own calls, which this runtime cannot make. */ +static int insp_fmt_custom_to_string(const ScrDyn *value) { + if (value->kind == SCR_DYN_TYPED_REF) { + ScrStr *to_string = scr_str_new("toString", 8); + bool has = scr_dyn_typed_ref_class_has(value, to_string); + scr_str_release(to_string); + return has ? -2 : 0; + } + for (const ScrDyn *current = value; current && current->kind == SCR_DYN_OBJ; current = current->prototype) { + if (current->symbol_properties) return -1; + const ScrDyn *member = scr_dyn_obj_get(current, "toString", 8); + if (member) return member->kind == SCR_DYN_FUNC; + } + return 0; +} + +static ScrStr *insp_fmt_s(ScrDyn *value, const ScrInspFormatOptions *o) { + if (value->kind == SCR_DYN_STR) return scr_str_retain(value->v.str); + if (value->kind == SCR_DYN_NUM) return scr_insp_f64(value->v.num); + if (value->kind == SCR_DYN_BIGINT) return scr_bigint_inspect(value->v.bigint); + if (value->kind == SCR_DYN_FUNC) { + insp_fmt_fence(o, "%s of a function prints its source text, which a compiled binary does not keep"); + return NULL; + } + if ((value->kind == SCR_DYN_OBJ && scr_dyn_obj_get(value, "%error", 6) == NULL) || + value->kind == SCR_DYN_TYPED_REF) { + int custom = insp_fmt_custom_to_string(value); + if (custom == -2) { + insp_fmt_fence(o, "%s of a class instance whose class defines toString has no lowering"); + return NULL; + } + if (custom < 0) { + insp_fmt_fence(o, "%s of objects with symbol-keyed members has no lowering"); + return NULL; + } + if (custom) return scr_dyn_string_coerce_js(value); + } + if (!insp_fmt_object(value)) return scr_insp_dyn(value, 0, 0); + /* inspect(arg, { ...inspectOptions, depth: 0, colors: false, compact: 3 }) */ + return insp_fmt_inspect(value, o, 0, 3, false); +} + +static ScrStr *insp_fmt_number(ScrDyn *value, int spec) { + if (value->kind == SCR_DYN_BIGINT && spec != 'f') return scr_bigint_inspect(value->v.bigint); + if (value->kind == SCR_DYN_SYMBOL) return scr_str_new("NaN", 3); + double number; + if (spec == 'd') { + if (!scr_dyn_number_coerce_js(value, &number)) return NULL; + } else { + ScrStr *text = scr_dyn_string_coerce_js(value); + if (!text) return NULL; + number = spec == 'i' ? scr_parse_int(text, 0) : scr_parse_float(text); + scr_str_release(text); + } + return scr_insp_f64(number); +} + +/* One formatter's conversions: %s, %j, %d/%i/%f, %O/%o and the remaining + * arguments' inspect (depth -1: the options' depth). The loop references + * only the table it runs, so a program links only the conversions it uses. */ +typedef struct { + ScrStr *(*s)(ScrDyn *value, const ScrInspFormatOptions *o); + ScrStr *(*j)(const ScrDyn *value); + ScrStr *(*number)(ScrDyn *value, int spec); + ScrStr *(*inspect)(ScrDyn *value, const ScrInspFormatOptions *o, double depth); +} InspFmtOps; + +static ScrStr *insp_format_loop(ScrDyn *const *args, size_t argc, const ScrInspFormatOptions *o, + const InspFmtOps *ops) { + if (argc == 0) return scr_str_new("", 0); + InspBuf out = {0}; + size_t a = 0; + const char *join = ""; + if (args[0]->kind == SCR_DYN_STR) { + const ScrStr *first = args[0]->v.str; + if (argc == 1) return scr_str_retain((ScrStr *)first); + const char *text = first->data; + size_t last = 0; + for (size_t i = 0; first->len > 0 && i < first->len - 1; i++) { + if (text[i] != '%') continue; + int next = (unsigned char)text[++i]; + if (a + 1 != argc) { + ScrStr *piece = NULL; + switch (next) { + case 's': piece = ops->s(args[++a], o); break; + case 'j': piece = ops->j(args[++a]); break; + case 'd': + case 'i': + case 'f': piece = ops->number(args[++a], next); break; + case 'O': piece = ops->inspect(args[++a], o, o->depth); break; + case 'o': + if (insp_fmt_object(args[a + 1])) { + insp_fmt_fence(o, "%o of objects renders hidden properties, which have no lowering; use %O"); + break; + } + piece = ops->inspect(args[++a], o, 4); + break; + case 'c': + a += 1; + piece = scr_str_new("", 0); + break; + case '%': + ib_bytes(&out, text + last, i - last); + last = i + 1; + continue; + default: + continue; + } + if (!piece) { + free(out.data); + return NULL; + } + if (last != i - 1) ib_bytes(&out, text + last, i - 1 - last); + ib_str(&out, piece); + scr_str_release(piece); + last = i + 1; + } else if (next == '%') { + ib_bytes(&out, text + last, i - last); + last = i + 1; + } + } + if (last != 0) { + a++; + join = " "; + if (last < first->len) ib_bytes(&out, text + last, first->len - last); + } + } + for (; a < argc; a++) { + ib_cstr(&out, join); + if (args[a]->kind == SCR_DYN_STR) { + ib_str(&out, args[a]->v.str); + } else { + ScrStr *piece = ops->inspect(args[a], o, o->depth); + if (!piece) { + free(out.data); + return NULL; + } + ib_str(&out, piece); + scr_str_release(piece); + } + join = " "; + } + return ib_take(&out); +} + + +static ScrStr *insp_fmt_inspect_full(ScrDyn *value, const ScrInspFormatOptions *o, double depth) { + return insp_fmt_inspect(value, o, depth, o->compact, o->colors); +} + +static const InspFmtOps insp_fmt_full_ops = {insp_fmt_s, scr_util_format_j, insp_fmt_number, + insp_fmt_inspect_full}; + +ScrStr *scr_insp_format(ScrDyn *const *args, size_t argc, const ScrInspFormatOptions *o) { + return insp_format_loop(args, argc, o, &insp_fmt_full_ops); +} + +/* The formatter over arguments that are all strings, numbers, booleans, + * null or undefined (the compiler proves it from their types): the same + * rules for those kinds, without the checked-dynamic walk the other + * values need. */ +static ScrStr *insp_prim_to_string(const ScrDyn *v) { + switch (v->kind) { + case SCR_DYN_STR: return scr_str_retain(v->v.str); + case SCR_DYN_NUM: { + char buf[40]; + size_t n = scr_f64_to_str(v->v.num, buf); + return scr_str_new(buf, n); + } + case SCR_DYN_BOOL: return v->v.b ? scr_str_new("true", 4) : scr_str_new("false", 5); + case SCR_DYN_NULL: return scr_str_new("null", 4); + default: return scr_str_new("undefined", 9); + } +} + +static ScrStr *insp_prim_inspect(ScrDyn *v, const ScrInspFormatOptions *o, double depth) { + (void)o; + (void)depth; + if (v->kind == SCR_DYN_STR) return scr_insp_str(v->v.str); + if (v->kind == SCR_DYN_NUM) return scr_insp_f64(v->v.num); + return insp_prim_to_string(v); +} + +static ScrStr *insp_prim_s(ScrDyn *v, const ScrInspFormatOptions *o) { + (void)o; + if (v->kind == SCR_DYN_NUM) return scr_insp_f64(v->v.num); + return insp_prim_to_string(v); +} + +static ScrStr *insp_prim_j(const ScrDyn *v) { + if (v->kind == SCR_DYN_STR) { + ScrJsonBuf b; + scr_jb_init(&b); + scr_jb_put_json_str(&b, v->v.str); + return scr_jb_finish(&b); + } + if (v->kind == SCR_DYN_NUM && !isfinite(v->v.num)) return scr_str_new("null", 4); + return insp_prim_to_string(v); +} + +static ScrStr *insp_prim_number(ScrDyn *v, int spec) { + double number; + if (spec == 'd') { + number = v->kind == SCR_DYN_NUM ? v->v.num + : v->kind == SCR_DYN_STR ? scr_string_to_number(v->v.str) + : v->kind == SCR_DYN_BOOL ? (v->v.b ? 1 : 0) + : v->kind == SCR_DYN_NULL ? 0 + : NAN; + } else { + ScrStr *text = insp_prim_to_string(v); + number = spec == 'i' ? scr_parse_int(text, 0) : scr_parse_float(text); + scr_str_release(text); + } + return scr_insp_f64(number); +} + +static const InspFmtOps insp_fmt_primitive_ops = {insp_prim_s, insp_prim_j, insp_prim_number, + insp_prim_inspect}; + +ScrStr *scr_insp_format_primitives(ScrDyn *pack) { + ScrInspFormatOptions o; + scr_insp_format_defaults(&o, "util.format"); + if (pack->kind != SCR_DYN_ARR) return scr_str_new("", 0); + return insp_format_loop(pack->v.arr.items, pack->v.arr.len, &o, &insp_fmt_primitive_ops); +} + +/* The arguments of a dyn array pack (+1 text, or NULL pending). */ +static ScrStr *insp_fmt_pack(ScrDyn *pack, const ScrInspFormatOptions *o) { + if (pack->kind != SCR_DYN_ARR) return scr_insp_format(NULL, 0, o); + return scr_insp_format(pack->v.arr.items, pack->v.arr.len, o); +} + +ScrStr *scr_insp_format_pack(ScrDyn *pack) { + ScrInspFormatOptions o; + scr_insp_format_defaults(&o, "util.format"); + return insp_fmt_pack(pack, &o); +} + +/* Whether a format string can substitute: a '%' before its last character. + * A string without one formats as its arguments joined with spaces. */ +bool scr_insp_format_specifies(ScrStr *first) { + return first->len > 1 && memchr(first->data, '%', first->len - 1) != NULL; +} + +bool scr_insp_format_specifies_dyn(ScrDyn *first) { + return first->kind == SCR_DYN_STR && scr_insp_format_specifies(first->v.str); +} + +/* One own enumerable option of an inspect options object: +1 value, a + * borrowed undefined when absent, or NULL with the exception pending. */ +static ScrDyn *insp_opt(const ScrDyn *options, const char *key, bool *present) { + size_t len = strlen(key); + *present = options->kind == SCR_DYN_OBJ && scr_dyn_obj_get(options, key, len) != NULL && + scr_dyn_obj_enumerable(options, key, len); + if (!*present) return scr_dyn_retain(scr_dyn_undefined()); + return scr_dyn_obj_read(options, key, len); +} + +/* ctx.depth from an option or inspect's third argument: a number, null + * or undefined (no limit for null, undefined and NaN). */ +static bool insp_depth_option(const ScrDyn *value, ScrInspFormatOptions *o) { + if (value->kind == SCR_DYN_NUM) { + o->depth = isnan(value->v.num) ? INFINITY : value->v.num; + o->depth_kind = 0; + o->depth_number = value->v.num; + } else if (value->kind == SCR_DYN_NULL || value->kind == SCR_DYN_UNDEF) { + o->depth = INFINITY; + o->depth_kind = value->kind == SCR_DYN_NULL ? 1 : 2; + } else { + return false; + } + return true; +} + +/* inspect's option keys (inspectDefaultOptions and stylize) over an + * options object, as `inspect(value, options)` copies them. Options at + * their defaults, depth, colors, compact (a number or false), + * breakLength (a number) and sorted (a boolean) lower; other values of + * the known options throw the SC2020 refusal. */ +bool scr_insp_format_read_options(const ScrDyn *options, ScrInspFormatOptions *o) { + ScrDyn *view = NULL; + if (options->kind == SCR_DYN_TYPED_REF) { + view = scr_dyn_typed_ref_materialize(options); + if (scr_exc_pending()) { + scr_dyn_release(view); + return false; + } + options = view; + } + bool ok = false, present; + ScrDyn *value = NULL; +#define INSP_OPT(key) \ + do { \ + scr_dyn_release(value); \ + value = insp_opt(options, key, &present); \ + if (!value) goto done; \ + } while (0) + INSP_OPT("depth"); + if (present) { + if (!insp_depth_option(value, o)) { + insp_fmt_fence(o, "with a depth option that is not a number or null has no lowering"); + goto done; + } + } + INSP_OPT("colors"); + if (present) o->colors = scr_dyn_truthy(value); + INSP_OPT("customInspect"); + if (present) o->custom_inspect = scr_dyn_truthy(value); + INSP_OPT("compact"); + if (present) { + if (value->kind == SCR_DYN_NUM) o->compact = value->v.num < 0 ? 0 : value->v.num; + else if (value->kind == SCR_DYN_BOOL && !value->v.b) o->compact = -1; + else { + insp_fmt_fence(o, "with this compact option has no lowering (compact must be false or a number)"); + goto done; + } + } + INSP_OPT("breakLength"); + if (present) { + if (value->kind != SCR_DYN_NUM) { + insp_fmt_fence(o, "with a breakLength option that is not a number has no lowering"); + goto done; + } + o->break_length = value->v.num; + } + INSP_OPT("sorted"); + if (present) { + if (value->kind != SCR_DYN_BOOL) { + insp_fmt_fence(o, "with a sorted option that is not a boolean has no lowering"); + goto done; + } + o->sorted = value->v.b; + } + /* The options whose non-default values change renderings this runtime + * does not produce. */ + static const struct { + const char *key; + int kind; /* 0: falsy, 1: true, 2: the number 100, 3: the number 10000 */ + } fixed[] = { + {"showHidden", 0}, {"showProxy", 0}, {"maxArrayLength", 2}, {"maxStringLength", 3}, + {"getters", 0}, {"numericSeparator", 0}, {"stylize", 0}, + }; + for (size_t i = 0; i < sizeof fixed / sizeof *fixed; i++) { + INSP_OPT(fixed[i].key); + if (!present) continue; + bool at_default = fixed[i].kind == 0 ? !scr_dyn_truthy(value) + : fixed[i].kind == 1 ? value->kind == SCR_DYN_BOOL && value->v.b + : value->kind == SCR_DYN_NUM && value->v.num == (fixed[i].kind == 2 ? 100 : 10000); + /* The stylize function a custom inspect function received (options + * passed back into util.inspect) is the default for its colors. */ + if (!at_default && strcmp(fixed[i].key, "stylize") == 0 && insp_own_stylize(value, o->colors)) + at_default = true; + /* stylize is replaced by the color stylizer when colors is set. */ + if (!at_default && !(o->colors && strcmp(fixed[i].key, "stylize") == 0)) { + InspBuf b = {0}; + ib_cstr(&b, "with the "); + ib_cstr(&b, fixed[i].key); + ib_cstr(&b, " option has no lowering"); + ScrStr *what = ib_take(&b); + insp_fmt_fence(o, what->data); + scr_str_release(what); + goto done; + } + } +#undef INSP_OPT + /* Keys inspect does not read: the custom functions receive them. */ + if (!view && options->kind == SCR_DYN_OBJ) { + static const char *const known[] = { + "showHidden", "depth", "colors", "customInspect", "showProxy", + "maxArrayLength", "maxStringLength", "breakLength", "compact", "sorted", + "getters", "numericSeparator", "stylize", + }; + for (size_t i = 0; i < options->v.obj.len && !o->user_options; i++) { + const ScrDynEntry *ent = &options->v.obj.entries[i]; + if (!ent->enumerable) continue; + bool read = false; + for (size_t k = 0; k < sizeof known / sizeof *known && !read; k++) + read = ent->key_len == strlen(known[k]) && memcmp(ent->key, known[k], ent->key_len) == 0; + if (!read) o->user_options = options; + } + } + ok = true; +done: + scr_dyn_release(value); + scr_dyn_release(view); + return ok; +} + +/* util.inspect(value[, options]) and inspect(value, showHidden, depth, + * colors) over checked-dynamic arguments; `argc` is the call's argument + * count. A third argument other than undefined sets the depth and a + * fourth the colors; a boolean second argument is showHidden, and an + * object supplies the options, which apply after the legacy arguments. + * +1, or NULL with the exception pending. */ +ScrStr *scr_insp_inspect(ScrDyn *value, ScrDyn *options, ScrDyn *depth, ScrDyn *colors, + double argc) { + ScrInspFormatOptions o; + scr_insp_format_defaults(&o, "util.inspect"); + if (argc >= 3 && depth->kind != SCR_DYN_UNDEF) { + if (!insp_depth_option(depth, &o)) { + insp_fmt_fence(&o, "with a depth argument that is not a number or null has no lowering"); + return NULL; + } + } + if (argc >= 4 && colors->kind != SCR_DYN_UNDEF) o.colors = scr_dyn_truthy(colors); + if (options->kind == SCR_DYN_BOOL) { + if (options->v.b) { + insp_fmt_fence(&o, "with showHidden has no lowering"); + return NULL; + } + } else if (options->kind == SCR_DYN_OBJ || options->kind == SCR_DYN_TYPED_REF) { + if (!scr_insp_format_read_options(options, &o)) return NULL; + } + return insp_fmt_inspect(value, &o, o.depth, o.compact, o.colors); +} + +static ScrDyn *insp_inspect_call(ScrClosure *closure, ScrDyn *const *args, size_t argc) { + (void)closure; + ScrDyn *undefined = scr_dyn_undefined(); + scr_insp_nested_begin(); + ScrStr *text = scr_insp_inspect(argc > 0 ? args[0] : undefined, argc > 1 ? args[1] : undefined, + argc > 2 ? args[2] : undefined, argc > 3 ? args[3] : undefined, + (double)argc); + scr_insp_nested_restore(); + if (!text) return NULL; + ScrDyn *out = scr_dyn_new_str(text); + scr_str_release(text); + return out; +} + +/* util.formatWithOptions(inspectOptions, ...pack): validateObject (arrays + * pass, null and functions do not), then the options as inspect reads + * them. */ +ScrStr *scr_insp_format_with_options(ScrDyn *options, ScrDyn *pack) { + if (options->kind != SCR_DYN_OBJ && options->kind != SCR_DYN_ARR && + options->kind != SCR_DYN_TYPED_REF) { + scr_dyn_arg_type_fail("inspectOptions", "of type object", options); + return NULL; + } + ScrInspFormatOptions o; + scr_insp_format_defaults(&o, "util.formatWithOptions"); + if (!scr_insp_format_read_options(options, &o)) return NULL; + return insp_fmt_pack(pack, &o); +} + /* ── console.table cells ─────────────────────────────────────────────── */ size_t scr_insp_display_width(const char *data, size_t len) { return insp_width(data, len); } diff --git a/packages/runtime/src/scr_inspect.h b/packages/runtime/src/scr_inspect.h index 5ac3aa9dd..a95ac1fde 100644 --- a/packages/runtime/src/scr_inspect.h +++ b/packages/runtime/src/scr_inspect.h @@ -47,8 +47,50 @@ ScrStr *scr_insp_dyn(ScrDyn *d, double recurse, double depth); ScrStr *scr_insp_typed_array(ScrBytes *b, double recurse, double depth); /* format's %s / rest-args twin: dyn strings pass verbatim. */ ScrStr *scr_insp_dyn_s(ScrDyn *d, double depth); +/* util.format over checked-dynamic arguments (formatWithOptionsInternal): + * the options an inspect of each argument takes, and the API name the + * SC2020 refusals name. These entries call program code, so they answer + * NULL with the exception pending when it throws. */ +typedef struct ScrInspFormatOptions { + const char *surface; + double depth; /* Infinity for null, undefined and NaN */ + /* The depth as the options gave it, which custom inspect functions + * receive when the depth is unbounded: 0 a number (depth_number, + * Infinity or NaN), 1 null, 2 undefined. */ + int depth_kind; + double depth_number; + bool colors; + bool custom_inspect; + double break_length; + double compact; /* -1 for false */ + bool sorted; + /* The options object, when it holds keys inspect does not read: custom + * inspect functions receive them (borrowed; NULL otherwise). */ + const ScrDyn *user_options; +} ScrInspFormatOptions; +void scr_insp_format_defaults(ScrInspFormatOptions *o, const char *surface); +/* Reads inspect's options from an options object into o; false with the + * SC2020 refusal pending for values that have no lowering. */ +bool scr_insp_format_read_options(const ScrDyn *options, ScrInspFormatOptions *o); +ScrStr *scr_insp_format(ScrDyn *const *args, size_t argc, const ScrInspFormatOptions *o); +/* util.format(...pack) and util.formatWithOptions(options, ...pack) over a + * dyn array of the arguments (+1). */ +ScrStr *scr_insp_format_pack(ScrDyn *pack); +/* util.format(...pack) when every argument is a string, number, boolean, + * null or undefined, without the checked-dynamic walk (+1). */ +ScrStr *scr_insp_format_primitives(ScrDyn *pack); +ScrStr *scr_insp_format_with_options(ScrDyn *options, ScrDyn *pack); +/* util.inspect(value, options, depth, colors) over checked-dynamic + * arguments with Node's legacy-argument rules; argc is the call's + * argument count (+1, or NULL with the exception pending). */ +ScrStr *scr_insp_inspect(ScrDyn *value, ScrDyn *options, ScrDyn *depth, ScrDyn *colors, + double argc); +/* Whether a format string has a '%' before its last character, the only + * strings whose substitutions can change the output. */ +bool scr_insp_format_specifies(ScrStr *first); +bool scr_insp_format_specifies_dyn(ScrDyn *first); void scr_insp_begin(double recurse); -/* util.inspect's breakLength, compact (0 for false) and sorted for the +/* util.inspect's breakLength, compact (-1 for false) and sorted for the * renders that follow; the call site restores the defaults (80, 3, false) * afterwards. */ void scr_insp_layout(double break_length, double compact, bool sorted); @@ -75,6 +117,33 @@ ScrStr *scr_insp_end(ScrStr *base, ScrStr *b0, ScrStr *b1, double recurse, * caller's level (+1; borrows both). */ void scr_insp_nested_begin(void); ScrStr *scr_insp_nested_end(ScrStr *prefix, ScrStr *inner); +/* nested_end's restore without its text: the caller's context returns. */ +void scr_insp_nested_restore(void); +/* ctx.customInspect for the renders that follow; answers the previous + * value for the caller to restore. */ +bool scr_insp_custom_inspect(bool enabled); +/* How the unbounded depth of the renders that follow was given, for custom + * inspect functions: 0 Infinity, 1 null, 2 undefined. Answers the + * previous kind for the caller to restore. */ +double scr_insp_depth_kind(double kind); +/* A custom inspect function's string result re-indented at the current + * level (+1; borrows text). */ +ScrStr *scr_insp_custom_text(ScrStr *text); +/* The compiled [util.inspect.custom] calls: ctx.customInspect, the depth + * argument at recursion `recurse` of `depth` and the options object (+1 + * dyn values; options NULL with an exception pending), util.inspect as a + * function value (+1), whether a dyn result is the instance itself, and a + * dyn result's rendering at the instance's level (+1). */ +bool scr_insp_custom_enabled(void); +/* Registers a class's compiled custom inspect function under the type key + * of its instances' typed refs: fn(self, depth, options, inspect) answers + * the method's result (borrows both). */ +void scr_insp_register_custom(ScrStr *type_key, ScrDyn *fn); +ScrDyn *scr_insp_custom_depth(double recurse, double depth); +ScrDyn *scr_insp_custom_options(double depth); +ScrDyn *scr_insp_inspect_function(void); +bool scr_insp_custom_is_self(ScrDyn *result, ScrDyn *self); +ScrStr *scr_insp_custom_result(ScrDyn *result, double recurse, double depth); /* The buf.read* / buf.write* numeric families on u8 bytes, Node-exact: * boundsError's ladder (non-integer offset → 'It must be an integer', a diff --git a/packages/runtime/src/scr_json.c b/packages/runtime/src/scr_json.c index dfa81ace6..3a4e6190d 100644 --- a/packages/runtime/src/scr_json.c +++ b/packages/runtime/src/scr_json.c @@ -172,6 +172,24 @@ bool scr_dyn_is_async_dispose_symbol(const ScrDyn *key) { key->v.symbol.value == scr_async_dispose_symbol->v.symbol.value; } +/* Symbol.for('nodejs.util.inspect.custom'), installed by the symbol unit + * when the program creates it: the key util.inspect reads custom inspect + * functions under. */ +static SCR_TL ScrDyn *scr_inspect_custom_symbol; + +static void scr_inspect_custom_symbol_cleanup(void) { + scr_dyn_release(scr_inspect_custom_symbol); + scr_inspect_custom_symbol = NULL; +} + +void scr_dyn_install_inspect_custom_symbol(ScrDyn *key) { + if (scr_inspect_custom_symbol) return; + scr_inspect_custom_symbol = scr_dyn_retain(key); + scr_atexit(scr_inspect_custom_symbol_cleanup); +} + +ScrDyn *scr_dyn_inspect_custom_symbol(void) { return scr_inspect_custom_symbol; } + static void scr_to_primitive_symbol_cleanup(void) { scr_dyn_release(scr_to_primitive_symbol); scr_to_primitive_symbol = NULL; diff --git a/packages/runtime/src/scr_symbol.c b/packages/runtime/src/scr_symbol.c index 304f2544f..e49edb370 100644 --- a/packages/runtime/src/scr_symbol.c +++ b/packages/runtime/src/scr_symbol.c @@ -125,6 +125,11 @@ ScrSym *scr_sym_for(ScrStr *key) { s->reg_key = scr_str_retain(key); s->reg_next = g_sym_registry; g_sym_registry = scr_sym_retain(s); + if (key->len == 26 && !memcmp(key->data, "nodejs.util.inspect.custom", 26)) { + ScrDyn *custom = scr_dyn_new_symbol(s); + scr_dyn_install_inspect_custom_symbol(custom); + scr_dyn_release(custom); + } return s; } diff --git a/packages/runtime/src/scr_util_callable.c b/packages/runtime/src/scr_util_callable.c index c905ea8ae..e3efd0be7 100644 --- a/packages/runtime/src/scr_util_callable.c +++ b/packages/runtime/src/scr_util_callable.c @@ -9,8 +9,8 @@ * and reported on the next tick (scr_warning_emit_deferred), which also * applies process.noDeprecation and process.throwDeprecation. debuglog * matches sections against NODE_DEBUG as the process started and writes - * Node's "SECTION PID: message" lines to stderr. All arguments are - * borrowed. */ + * Node's "SECTION PID: message" lines, formatted by scr_insp_format, to + * stderr. All arguments are borrowed. */ #include "scr_runtime.h" #include @@ -513,163 +513,6 @@ ScrDyn *scr_util_callbackify(ScrDyn *original) { return callbackified; } -/* ── util.format over dyn arguments ──────────────────────────────────── - * formatWithOptionsInternal for debuglog's lines: the %-substitutions - * (%s %d %i %f %j %o %O %c %%) with Node's argument cursor, then the - * remaining arguments space-joined, strings verbatim and everything else - * inspected. With terminal colors, primitives take inspect's default - * styles; colored object renderings have no lowering. */ -static bool uc_primitive(const ScrDyn *value) { - return !uc_object(value); -} - -static ScrStr *uc_inspect(ScrDyn *value, double depth, bool colors) { - if (!colors) return scr_insp_dyn(value, 0, depth); - if (!uc_primitive(value)) { - uc_fence("util.debuglog output of objects on a color terminal has no lowering"); - return NULL; - } - const char *open = "\x1b[33m", *close = "\x1b[39m"; - if (value->kind == SCR_DYN_UNDEF) open = "\x1b[90m"; - else if (value->kind == SCR_DYN_NULL) open = "\x1b[1m", close = "\x1b[22m"; - else if (value->kind == SCR_DYN_STR || value->kind == SCR_DYN_SYMBOL) open = "\x1b[32m"; - ScrStr *text = scr_insp_dyn(value, 0, depth); - ScrJsonBuf b; - scr_jb_init(&b); - scr_jb_puts(&b, open); - scr_jb_put_str(&b, text); - scr_jb_puts(&b, close); - scr_str_release(text); - return scr_jb_finish(&b); -} - -/* hasBuiltInToString's negative answer for checked objects: an own or - * inherited toString the program defined. Symbol-keyed members (where - * Symbol.toPrimitive would also count) are reported as unknown. */ -static int uc_custom_to_string(const ScrDyn *value) { - for (const ScrDyn *current = value; current && current->kind == SCR_DYN_OBJ; current = current->prototype) { - if (current->symbol_properties) return -1; - const ScrDyn *member = scr_dyn_obj_get(current, "toString", 8); - if (member) return member->kind == SCR_DYN_FUNC; - } - return 0; -} - -static ScrStr *uc_format_s(ScrDyn *value) { - if (value->kind == SCR_DYN_STR) return scr_str_retain(value->v.str); - if (value->kind == SCR_DYN_NUM) return scr_insp_f64(value->v.num); - if (value->kind == SCR_DYN_FUNC) { - uc_fence("util.debuglog %s of a function prints its source text, which a compiled binary does not keep"); - return NULL; - } - if (value->kind == SCR_DYN_OBJ && scr_dyn_obj_get(value, "%error", 6) == NULL) { - int custom = uc_custom_to_string(value); - if (custom < 0) { - uc_fence("util.debuglog %s of objects with symbol-keyed members has no lowering"); - return NULL; - } - if (custom) return scr_dyn_string_coerce_js(value); - } - return scr_insp_dyn_s(value, 0); -} - -static ScrStr *uc_format_number(ScrDyn *value, int spec) { - if (value->kind == SCR_DYN_BIGINT && spec != 'f') return scr_bigint_inspect(value->v.bigint); - if (value->kind == SCR_DYN_SYMBOL) return scr_str_new("NaN", 3); - double number; - if (spec == 'd') { - if (value->kind == SCR_DYN_BIGINT) return scr_bigint_inspect(value->v.bigint); - if (!scr_dyn_number_coerce_js(value, &number)) return NULL; - } else { - ScrStr *text = scr_dyn_string_coerce_js(value); - if (!text) return NULL; - number = spec == 'i' ? scr_parse_int(text, 0) : scr_parse_float(text); - scr_str_release(text); - } - return scr_insp_f64(number); -} - -static ScrStr *uc_format(ScrDyn *const *args, size_t argc, bool colors) { - if (argc == 0) return scr_str_new("", 0); - ScrJsonBuf out; - scr_jb_init(&out); - size_t a = 0; - const char *join = ""; - if (args[0]->kind == SCR_DYN_STR) { - const ScrStr *first = args[0]->v.str; - if (argc == 1) { - scr_str_release(scr_jb_finish(&out)); - return scr_str_retain((ScrStr *)first); - } - const char *text = first->data; - size_t last = 0; - for (size_t i = 0; first->len > 0 && i < first->len - 1; i++) { - if (text[i] != '%') continue; - int next = (unsigned char)text[++i]; - if (a + 1 != argc) { - ScrStr *piece = NULL; - switch (next) { - case 's': piece = uc_format_s(args[++a]); break; - case 'j': piece = scr_util_format_j(args[++a]); break; - case 'd': - case 'i': - case 'f': piece = uc_format_number(args[++a], next); break; - case 'O': piece = uc_inspect(args[++a], 2, colors); break; - case 'o': - if (!uc_primitive(args[a + 1])) { - uc_fence("util.debuglog %o of objects renders hidden properties, which have no lowering; use %O"); - break; - } - piece = uc_inspect(args[++a], 4, colors); - break; - case 'c': - a += 1; - piece = scr_str_new("", 0); - break; - case '%': - scr_jb_write(&out, text + last, i - last); - last = i + 1; - continue; - default: - continue; - } - if (!piece) { - scr_str_release(scr_jb_finish(&out)); - return NULL; - } - if (last != i - 1) scr_jb_write(&out, text + last, i - 1 - last); - scr_jb_put_str(&out, piece); - scr_str_release(piece); - last = i + 1; - } else if (next == '%') { - scr_jb_write(&out, text + last, i - last); - last = i + 1; - } - } - if (last != 0) { - a++; - join = " "; - if (last < first->len) scr_jb_write(&out, text + last, first->len - last); - } - } - for (; a < argc; a++) { - scr_jb_puts(&out, join); - if (args[a]->kind == SCR_DYN_STR) { - scr_jb_put_str(&out, args[a]->v.str); - } else { - ScrStr *piece = uc_inspect(args[a], 2, colors); - if (!piece) { - scr_str_release(scr_jb_finish(&out)); - return NULL; - } - scr_jb_put_str(&out, piece); - scr_str_release(piece); - } - join = " "; - } - return scr_jb_finish(&out); -} - /* ── util.debuglog ───────────────────────────────────────────────────── * The logger defers its NODE_DEBUG test to its first call or `enabled` * read. The first call also selects the section's shared implementation @@ -735,7 +578,10 @@ static bool uc_debug_test(const ScrStr *section) { static ScrDyn *uc_debug_write(ScrClosure *closure, ScrDyn *const *args, size_t argc) { bool colors = scr_getenv("FORCE_COLOR") ? scr_util_terminal_colors() : scr_process_is_tty(2) && scr_util_terminal_colors(); - ScrStr *message = uc_format(args, argc, colors); + ScrInspFormatOptions options; + scr_insp_format_defaults(&options, "util.debuglog"); + options.colors = colors; + ScrStr *message = scr_insp_format(args, argc, &options); if (!message) return NULL; ScrDyn *set = uc_cap(closure, 0); char pid[64]; diff --git a/tests/corpus/7520-util-format-runtime.ts b/tests/corpus/7520-util-format-runtime.ts new file mode 100644 index 000000000..d4a7f9ea6 --- /dev/null +++ b/tests/corpus/7520-util-format-runtime.ts @@ -0,0 +1,110 @@ +// util.format, util.formatWithOptions and console output with format +// strings known only at run time, spread arguments and options objects: +// every substitution, the argument cursor, %% and %c, missing and extra +// arguments, the space-joined form of strings without a '%', and Node's +// inspectOptions validation. +import * as util from "node:util"; + +class Point { + x: number; + y: number; + constructor(x: number, y: number) { + this.x = x; + this.y = y; + } +} + +interface Row { + id: number; + tags: string[]; + nested: { deep: { deeper: { deepest: number } } }; +} + +const formats: string[] = [ + "%s and %s", + "%d|%i|%f", + "%j", + "%O / %o", + "100%% of %s", + "%c%s%c", + "no specifiers", + "trailing %", + "%x %s", + "%s", +]; +const row: Row = { id: 7, tags: ["a", "b"], nested: { deep: { deeper: { deepest: 1 } } } }; + +for (const format of formats) { + console.log(util.format(format, "first", 42, row)); + console.log(util.format(format, 3.5, "12.75")); + console.log(util.format(format)); +} + +function pick(i: number): string { + return ["%s=%d", "%j", "%O", "plain"][i % 4]!; +} +for (let i = 0; i < 4; i++) { + console.log(util.format(pick(i), new Point(1, 2), -0, [1, 2, 3])); + console.log(pick(i), new Map([["k", new Point(3, 4)]]), new Set([1n, 2n])); + console.error(pick(i), "to stderr", null, undefined); +} + +const label: string = process.argv.length > 99 ? "" : "count: %d of %s"; +console.log(label, 5, "five", "extra"); +console.log(util.format(label, 10n, Symbol("sym"))); +console.log(util.format(label, new Point(0, 0), [new Point(5, 6)])); +console.log(util.format(pick(0), { toString: "not a function" }, true)); + +// Arguments that are all strings, numbers, booleans, null or undefined. +const prim: string = process.argv.length > 99 ? "" : "%s|%d|%i|%f|%j|%O|%o|%c|%%"; +console.log(prim, -0, "12.5", -0, true, 'q"uote', "s", 3, "css", "rest", null, undefined); +console.log(prim, NaN, null, undefined, "0x10", Infinity, NaN, false); +console.log(util.format(prim, "a\nb", " 7 ", "-3.9e2", "", -Infinity, 'it\'s', 1e21)); +console.log(util.format(label, 2.5)); + +// A union first argument formats only when it holds a string. +const maybe: string | number = process.argv.length > 99 ? 1 : "%s/%s"; +const notString: string | number = process.argv.length > 99 ? "%s" : 9; +console.log(util.format(maybe, "u", "v")); +console.log(util.format(notString, "u", "v")); +console.log(maybe, "w", "x"); + +// An unknown first argument. +const loose: unknown = process.argv.length > 99 ? 0 : "%s-%i"; +console.log(util.format(loose, "a", "7.9")); +console.log(loose, "b", 3.2); + +// Spread arguments. +const values: unknown[] = ["%s + %s = %d", 1, 2, 3, "tail"]; +console.log(util.format(...values)); +console.log(util.format("%s|%s", ...values)); +console.log(...values); +console.log("prefix", ...new Set(["x", "y"]), "suffix"); +console.error(...[1, "two", [3]]); +console.log(util.format(...([] as string[]))); + +// formatWithOptions with an options object. +const deep = { a: { b: { c: { d: { e: 1 } } } } }; +const depth = process.argv.length > 99 ? 1 : 0; +console.log(util.formatWithOptions({ depth }, "%O", deep)); +console.log(util.formatWithOptions({ depth: null }, deep)); +console.log(util.formatWithOptions({ depth: undefined }, "x", deep)); +console.log(util.formatWithOptions({ depth: 0 }, "%s|%O", deep, deep)); +console.log(util.formatWithOptions({ compact: false }, "%s", "v", { a: 1, b: [1, 2] })); +console.log(util.formatWithOptions({ breakLength: 20 }, { alpha: 1, beta: 2, gamma: 3 })); +console.log(util.formatWithOptions({ sorted: true }, "%O", { b: 1, a: 2, c: { z: 1, y: 2 } })); +console.log(util.formatWithOptions({ colors: true }, "%s %d %O", "s", 2, 3), "|"); +console.log(util.formatWithOptions({ colors: true }, 1, "plain", null, undefined, true, 4n)); +const options: util.InspectOptions = { depth: 1, compact: 1 }; +console.log(util.formatWithOptions(options, "%O", deep)); +console.log(util.formatWithOptions(options, ...values)); +console.log(util.formatWithOptions([] as unknown as util.InspectOptions, "%s", 1)); +const invalid: unknown[] = [null, 1, "x", undefined, () => {}]; +for (const bad of invalid) { + try { + util.formatWithOptions(bad as util.InspectOptions, "a"); + } catch (error) { + const e = error as NodeJS.ErrnoException; + console.log(e.name, e.code, e.message); + } +} diff --git a/tests/corpus/7521-util-format-runtime.cjs b/tests/corpus/7521-util-format-runtime.cjs new file mode 100644 index 000000000..559d09243 --- /dev/null +++ b/tests/corpus/7521-util-format-runtime.cjs @@ -0,0 +1,64 @@ +// util.format, util.formatWithOptions and console output over untyped +// values: format strings held in variables and parameters, spread +// arguments, %s over objects with and without their own toString, %d and +// %i coercions, %j over toJSON, and options objects built at run time. +"use strict"; +const util = require("util"); + +function log(message, ...rest) { + console.log(message, ...rest); +} + +function formatAll(format, values) { + return util.format(format, ...values); +} + +const messages = ["%s: %d items", "%s", "%i%%", "%j", "plain text", 42]; +for (const message of messages) { + log(message, "cart", "3.7", { total: 9 }); + console.log(formatAll(message, ["a", 1n])); +} + +const withToString = { + name: "custom", + toString() { + return `<${this.name}>`; + }, +}; +const inherited = Object.create(withToString); +inherited.name = "inherited"; +const fmt = "%s / %s / %s"; +console.log(util.format(fmt, withToString, inherited, [1, [2, [3]]])); +console.log(fmt, new Date(0), new Error("boom").message, null, Symbol("s")); + +const numeric = "%d %i %f"; +console.log(util.format(numeric, "0x1f", "12.9px", "1e3")); +console.log(util.format(numeric, { valueOf: () => 5 }, true, [])); +console.log(util.format(numeric, 7n, 8n, 9)); + +const json = "%j|%j|%j"; +const circular = { name: "loop" }; +circular.self = circular; +console.log(util.format(json, { toJSON: () => ({ replaced: 1 }) }, circular, undefined)); + +const options = {}; +options.depth = 0; +options.breakLength = 40; +console.log(util.formatWithOptions(options, "%O", { a: { b: 1 } }, ["x", { y: { z: 2 } }])); +options.depth = null; +options.unknownOption = true; +console.log(util.formatWithOptions(options, { a: { b: { c: { d: 1 } } } })); +console.log(util.formatWithOptions({ compact: 1 }, ...[{ k: [1, 2] }, "end"])); +for (const bad of [null, 0, "options", () => {}]) { + try { + util.formatWithOptions(bad, "x"); + } catch (error) { + console.log(error.name, error.code, error.message); + } +} + +const args = ["%s=%s", "key"]; +args.push("value", "rest"); +console.log(...args); +console.warn(...args); +console.log(util.format(...args.slice(1))); diff --git a/tests/corpus/7522-promisify-untyped.cjs b/tests/corpus/7522-promisify-untyped.cjs new file mode 100644 index 000000000..2e1476eac --- /dev/null +++ b/tests/corpus/7522-promisify-untyped.cjs @@ -0,0 +1,64 @@ +// util.promisify over JavaScript functions whose types are not known: +// arrows, function expressions, methods and functions chosen at run time. +// The callback's error rejects and its first value fulfills, a +// synchronous throw rejects, the call's receiver reaches the function, +// the wrapper keeps the original's name and length, and arguments that +// are not functions throw Node's ERR_INVALID_ARG_TYPE. +"use strict"; +const util = require("util"); + +function pick(kind) { + if (kind === "double") return (value, callback) => setImmediate(() => callback(null, value * 2)); + if (kind === "fail") return function failing(callback) { callback(new RangeError("failed")); }; + if (kind === "throw") return () => { throw new TypeError("thrown"); }; + return null; +} + +const store = { + prefix: "item", + read(key, callback) { + setTimeout(() => callback(null, `${this.prefix}:${key}`, "ignored"), 1); + }, +}; + +async function main() { + const double = util.promisify(pick("double")); + console.log("double:", await double(21), double.name, double.length); + try { + await util.promisify(pick("fail"))(); + } catch (error) { + console.log("rejected:", error.name, error.message); + } + try { + await util.promisify(pick("throw"))(); + } catch (error) { + console.log("sync:", error.name, error.message); + } + const read = util.promisify(store.read); + console.log("unbound:", await read.call(store, "a")); + store.readAsync = read; + console.log("method:", await store.readAsync("b")); + const bound = util.promisify(store.read.bind({ prefix: "bound" })); + console.log("bound:", await bound("c")); + const named = util.promisify(function lookup(a, b, callback) { + callback(null, [a, b]); + }); + console.log("named:", named.name, named.length, await named(1, 2)); + const values = []; + for (const n of [1, 2, 3]) values.push(await double(n)); + console.log("each:", values); + for (const value of [pick("none"), 42, "fn", {}, undefined]) { + try { + util.promisify(value); + } catch (error) { + console.log(error.name, error.code, error.message); + } + } + try { + util.promisify(); + } catch (error) { + console.log(error.code, error.message); + } +} + +main(); diff --git a/tests/corpus/7523-promisify-untyped.ts b/tests/corpus/7523-promisify-untyped.ts new file mode 100644 index 000000000..65d1d01c1 --- /dev/null +++ b/tests/corpus/7523-promisify-untyped.ts @@ -0,0 +1,38 @@ +// util.promisify over untyped functions in TypeScript: values typed `any` +// and functions whose parameters are `any` promisify at run time, and +// their promises settle with the callback's value (not the type of a +// builtin target's promisified form). +import { promisify } from "node:util"; + +const loose: any = (value: number, callback: (error: Error | null, result?: number) => void) => + callback(null, value + 1); +const looseAsync = promisify(loose); + +function legacy(a: any, b: any, callback: any): void { + setImmediate(() => (b === 0 ? callback(new RangeError("divide by zero")) : callback(null, a / b))); +} +const divide = promisify(legacy); + +const handlers: Record = { + echo: (text: string, callback: (error: null, value: string) => void) => callback(null, text), + later: (callback: (error: null, value: string[]) => void) => + setTimeout(() => callback(null, ["a", "b"]), 1), +}; + +async function main(): Promise { + console.log("loose:", await looseAsync(1)); + looseAsync(2).then((value: unknown) => console.log("then:", value)); + console.log("divide:", await divide(10, 4)); + try { + await divide(1, 0); + } catch (error) { + console.log("rejected:", (error as Error).message); + } + for (const name of ["echo", "later"]) { + const fn = promisify(handlers[name]); + console.log(`${name}:`, name === "echo" ? await fn("hi") : await fn()); + } + console.log("name:", divide.name, "length:", divide.length); +} + +main(); diff --git a/tests/corpus/7524-inspect-custom.ts b/tests/corpus/7524-inspect-custom.ts new file mode 100644 index 000000000..642ce75a9 --- /dev/null +++ b/tests/corpus/7524-inspect-custom.ts @@ -0,0 +1,91 @@ +// [util.inspect.custom] methods on classes: the method's string replaces +// the instance in util.inspect, console output and util.format, with +// Node's depth argument (negative past the depth limit, null for depth: +// null), options object and util.inspect; a multi-line result indents at +// its position, a returned value renders in the instance's place, and +// console.dir, customInspect: false, String() and JSON skip the method. +import * as util from "node:util"; +import { inspect } from "node:util"; + +class Money { + cents: number; + constructor(cents: number) { + this.cents = cents; + } + [inspect.custom](): string { + return `Money(${(this.cents / 100).toFixed(2)})`; + } +} + +class Probe { + label: string; + constructor(label: string) { + this.label = label; + } + [util.inspect.custom](depth: number | null, options: util.InspectOptionsStylized): string { + const keys = Object.keys(options).join(","); + return `Probe<${this.label} depth=${depth} optionsDepth=${options.depth} compact=${options.compact}> ${keys}`; + } +} + +class Lines { + [inspect.custom](): string { + return "first line\nsecond line"; + } +} + +class Point { + x: number; + y: number; + constructor(x: number, y: number) { + this.x = x; + this.y = y; + } + [inspect.custom](): { x: number; y: number; sum: number } { + return { x: this.x, y: this.y, sum: this.x + this.y }; + } +} + +class Wrapper { + inner: { values: number[]; nested: { deep: { deeper: number } } }; + constructor() { + this.inner = { values: [1, 2, 3], nested: { deep: { deeper: 4 } } }; + } + [inspect.custom](depth: number, options: util.InspectOptionsStylized, show: typeof inspect): string { + if (depth < 0) return options.stylize("[Wrapper]", "special"); + return `Wrapper ${show(this.inner, { ...options, depth: options.depth == null ? null : depth - 1 })}`; + } +} + +class Base { + name = "base"; + [inspect.custom](): string { + return `Custom(${this.name})`; + } +} +class Derived extends Base { + extra = 1; +} + +console.log(util.inspect(new Money(1234))); +console.log(new Money(5), [new Money(1), new Money(2)]); +console.log(util.inspect({ wallet: { cash: new Money(99) } })); +console.log(util.inspect(new Probe("top"))); +console.log(util.inspect({ a: { b: new Probe("nested") } })); +console.log(util.inspect([new Probe("past depth")], { depth: 0 })); +console.log(util.inspect(new Probe("unbounded"), { depth: null })); +console.log(util.inspect(new Probe("infinite"), { depth: Infinity })); +console.log(util.inspect([new Probe("undefined")], { depth: undefined })); +console.log(util.inspect(new Probe("compact"), { compact: false, breakLength: 40 })); +console.log(util.inspect({ key: new Lines(), list: [new Lines()] })); +console.log(util.inspect(new Map([["k", new Lines()]]))); +console.log(util.inspect(new Point(1, 2)), util.inspect({ p: new Point(3, 4) })); +console.log(util.inspect(new Wrapper()), util.inspect({ w: new Wrapper() }, { depth: 0 })); +console.log(util.inspect(new Derived()), util.inspect([new Base(), new Derived()])); +console.log(util.format("%s | %O", new Money(10), new Money(20))); +console.log("%s and %d", new Money(1), 2); +console.log(util.inspect(new Money(7), { customInspect: false })); +console.dir(new Money(8)); +console.log(String(new Money(9)), JSON.stringify(new Money(10))); +const values: unknown[] = [new Money(11), { money: new Money(12) }]; +console.log(values, util.inspect(values[1]), [new Lines(), values[0]]); diff --git a/tests/corpus/7525-inspect-custom.cjs b/tests/corpus/7525-inspect-custom.cjs new file mode 100644 index 000000000..9c7302fac --- /dev/null +++ b/tests/corpus/7525-inspect-custom.cjs @@ -0,0 +1,75 @@ +// [util.inspect.custom] in JavaScript: methods on classes and object +// literals (also keyed by Symbol.for('nodejs.util.inspect.custom')), the +// CommonJS call form (0, util_1.inspect)(value), results that are the +// object itself or another value, a non-function member under the key, +// and the options and util.inspect a method receives. +"use strict"; +const util_1 = require("util"); + +class Temperature { + constructor(degrees) { + this.degrees = degrees; + } + [util_1.inspect.custom](depth, options, inspect) { + return `Temperature<${this.degrees}> ${typeof options.stylize} ${inspect === util_1.inspect}`; + } +} + +const literal = { + id: 1, + [util_1.inspect.custom](depth) { + return `literal at depth ${depth}`; + }, +}; +const registered = { + [Symbol.for("nodejs.util.inspect.custom")]() { + return { replaced: [1, 2], nested: { x: 1 } }; + }, +}; +const self = { + v: 1, + [util_1.inspect.custom]() { + return this; + }, +}; +const notFunction = { x: 1, [util_1.inspect.custom]: "not a function" }; +const styled = { + [util_1.inspect.custom](depth, options) { + return options.stylize("styled", "special") + options.stylize(5, "number"); + }, +}; +const nesting = { + data: { a: { b: { c: 1 } } }, + [util_1.inspect.custom](depth, options, inspect) { + return `nesting ${inspect(this.data, { ...options, depth: 0 })}`; + }, +}; + +console.log((0, util_1.inspect)(new Temperature(21))); +console.log((0, util_1.inspect)({ t: new Temperature(22), list: [literal] })); +console.log(util_1.inspect([[literal]], { depth: 0 })); +console.log(util_1.inspect(registered), util_1.inspect({ r: registered })); +console.log(util_1.inspect(self), util_1.inspect(notFunction)); +console.log(util_1.inspect(styled), JSON.stringify(util_1.inspect(styled, { colors: true }))); +console.log(util_1.inspect(nesting), util_1.inspect({ n: nesting })); +console.log(literal, [registered]); +console.log((0, util_1.format)("%s / %O", literal, literal)); +console.log(util_1.inspect(literal, { customInspect: false })); +console.dir(literal); +console.log(util_1.inspect.custom === Symbol.for("nodejs.util.inspect.custom"), util_1.inspect.custom); + +const throwing = { + [util_1.inspect.custom]() { + throw new RangeError("from custom"); + }, +}; +try { + console.log([throwing]); +} catch (error) { + console.log("caught", error.name, error.message); +} +try { + util_1.inspect({ t: throwing }); +} catch (error) { + console.log("caught", error.message); +} diff --git a/tests/corpus/7526-inspect-options-runtime.ts b/tests/corpus/7526-inspect-options-runtime.ts new file mode 100644 index 000000000..b9f0c61d7 --- /dev/null +++ b/tests/corpus/7526-inspect-options-runtime.ts @@ -0,0 +1,62 @@ +// util.inspect with colors, option values and options objects known only +// at run time, and the legacy inspect(value, showHidden, depth, colors) +// form: Node's ANSI styles per value kind, color-aware line breaking and +// grouping, runtime depth and layout options, and colored output from +// util.formatWithOptions. +import * as util from "node:util"; + +class Point { + x: number; + y: number; + constructor(x: number, y: number) { + this.x = x; + this.y = y; + } +} + +const sparse: number[] = [1, 2, 3]; +sparse.length = 5; +const values: unknown[] = [ + 1, + -0, + "text", + "it's", + null, + undefined, + true, + 10n, + Symbol("sym"), + [1, "two", [3, [4, [5]]]], + { a: 1, "b-c": "d", nested: { deep: { deeper: { deepest: 1 } } } }, + new Map([["k", { v: 1 }]]), + new Set(["s", 2]), + new Date(0), + new Uint8Array([1, 2, 3]), + Buffer.from("hi"), + function named() {}, + new Point(1, 2), + sparse, + Object.create(null), + Array.from({ length: 30 }, (_, i) => i * 37), + "a long string with a newline\nand more text than fits on one line of eighty characters", +]; +for (const value of values) { + console.log(JSON.stringify(util.inspect(value, { colors: true }))); +} +const depth = process.argv.length > 99 ? 5 : 1; +const breakLength = process.argv.length > 99 ? 80 : 30; +console.log(util.inspect(values[10], { depth })); +console.log(util.inspect(values[10], { depth: depth - 1, breakLength })); +console.log(util.inspect(values[20], { breakLength, compact: depth })); +console.log(util.inspect({ b: 2, a: 1 }, { sorted: depth > 0 })); +console.log(util.inspect(values[10], { depth: undefined })); +const options: util.InspectOptions = { depth: 0, colors: true }; +console.log(JSON.stringify(util.inspect(values[10], options))); +options.colors = false; +console.log(util.inspect(values[10], options)); +console.log(util.inspect(values[10], false, 0)); +console.log(util.inspect(values[10], false, null, false)); +console.log(JSON.stringify(util.inspect(values[11], false, 1, true))); +console.log(util.inspect(values[10], undefined, depth)); +console.log(JSON.stringify(util.formatWithOptions({ colors: true }, "%s %O", "x", values[12]))); +console.log(JSON.stringify(util.formatWithOptions({ colors: true }, values[10], values[9]))); diff --git a/tests/diagnostics/inspect-custom-fences.ts b/tests/diagnostics/inspect-custom-fences.ts new file mode 100644 index 000000000..f4c7c593d --- /dev/null +++ b/tests/diagnostics/inspect-custom-fences.ts @@ -0,0 +1,44 @@ +// [util.inspect.custom] methods keep their boundaries: a class whose +// subclass declares the method without declaring it itself, an Error +// subclass with the method, a method with more than three parameters, a +// method returning a class instance, and %o over instances with one. +import { format, inspect } from "node:util"; + +class Plain { + a = 1; +} +class Fancy extends Plain { + [inspect.custom](): string { + return "fancy"; + } +} +const items: Plain[] = [new Fancy()]; +console.log(inspect(items)); + +class Failure extends Error { + [inspect.custom](): string { + return "failure"; + } +} +console.log(new Failure("x").message); + +class Wide { + [inspect.custom](a: number, b: unknown, c: unknown, d: unknown): string { + return `${a}${String(b)}${String(c)}${String(d)}`; + } +} +console.log(inspect(new Wide())); + +class Echo { + [inspect.custom](): Echo { + return new Echo(); + } +} +console.log(inspect(new Echo())); + +class Shown { + [inspect.custom](): string { + return "shown"; + } +} +console.log(format("%o", new Shown())); diff --git a/tests/harness/__snapshots__/inspect-custom-fences.ts.txt b/tests/harness/__snapshots__/inspect-custom-fences.ts.txt new file mode 100644 index 000000000..ba7cefefc --- /dev/null +++ b/tests/harness/__snapshots__/inspect-custom-fences.ts.txt @@ -0,0 +1,49 @@ +inspect-custom-fences.ts:16:21 - error SC2020: 'util.inspect of 'Plain[]' values' is part of the standard library types but has no scriptc lowering yet + + 15 | const items: Plain[] = [new Fancy()]; + 16 | console.log(inspect(items)); + | ^~~~~ + 17 | + + hint: classes whose subclasses declare [util.inspect.custom] have no inspect lowering yet (declare the method on the class itself) + +inspect-custom-fences.ts:19:3 - error SC1090: [util.inspect.custom] methods on Error subclasses are not supported yet + + 18 | class Failure extends Error { + 19 | [inspect.custom](): string { + | ^~~~~~~~~~~~~~~~ + 20 | return "failure"; + +inspect-custom-fences.ts:23:13 - error SC1090: constructing through a class value whose class has no lowering (the class declaration itself was rejected — see its own diagnostic) is not supported yet + + 22 | } + 23 | console.log(new Failure("x").message); + | ^~~~~~~~~~~~~~~~ + 24 | + +inspect-custom-fences.ts:30:21 - error SC2020: 'util.inspect of 'Wide' values' is part of the standard library types but has no scriptc lowering yet + + 29 | } + 30 | console.log(inspect(new Wide())); + | ^~~~~~~~~~ + 31 | + + hint: [util.inspect.custom] methods with more than three parameters have no inspect lowering + +inspect-custom-fences.ts:37:21 - error SC2020: 'util.inspect of 'Echo' values' is part of the standard library types but has no scriptc lowering yet + + 36 | } + 37 | console.log(inspect(new Echo())); + | ^~~~~~~~~~ + 38 | + + hint: [util.inspect.custom] methods that return a class instance have no inspect lowering + +inspect-custom-fences.ts:44:26 - error SC2020: 'util.format %o over values with [util.inspect.custom] methods' is part of the standard library types but has no scriptc lowering yet + + 43 | } + 44 | console.log(format("%o", new Shown())); + | ^~~~~~~~~~~ + 45 | + + hint: %o passes showHidden to the methods — use %O \ No newline at end of file diff --git a/tests/harness/corpus-costs/7520-util-format-runtime.ts.json b/tests/harness/corpus-costs/7520-util-format-runtime.ts.json new file mode 100644 index 000000000..9cb750202 --- /dev/null +++ b/tests/harness/corpus-costs/7520-util-format-runtime.ts.json @@ -0,0 +1 @@ +{ "plain": 6414, "san": 9821 } diff --git a/tests/harness/corpus-costs/7521-util-format-runtime.cjs.json b/tests/harness/corpus-costs/7521-util-format-runtime.cjs.json new file mode 100644 index 000000000..05cc5696d --- /dev/null +++ b/tests/harness/corpus-costs/7521-util-format-runtime.cjs.json @@ -0,0 +1 @@ +{ "plain": 4635, "san": 6305 } diff --git a/tests/harness/corpus-costs/7522-promisify-untyped.cjs.json b/tests/harness/corpus-costs/7522-promisify-untyped.cjs.json new file mode 100644 index 000000000..c5358642b --- /dev/null +++ b/tests/harness/corpus-costs/7522-promisify-untyped.cjs.json @@ -0,0 +1 @@ +{ "plain": 3984, "san": 6122 } diff --git a/tests/harness/corpus-costs/7523-promisify-untyped.ts.json b/tests/harness/corpus-costs/7523-promisify-untyped.ts.json new file mode 100644 index 000000000..be46b70d4 --- /dev/null +++ b/tests/harness/corpus-costs/7523-promisify-untyped.ts.json @@ -0,0 +1 @@ +{ "plain": 3595, "san": 36166 } diff --git a/tests/harness/corpus-costs/7524-inspect-custom.ts.json b/tests/harness/corpus-costs/7524-inspect-custom.ts.json new file mode 100644 index 000000000..b9c089238 --- /dev/null +++ b/tests/harness/corpus-costs/7524-inspect-custom.ts.json @@ -0,0 +1 @@ +{ "plain": 5331, "san": 10004 } diff --git a/tests/harness/corpus-costs/7525-inspect-custom.cjs.json b/tests/harness/corpus-costs/7525-inspect-custom.cjs.json new file mode 100644 index 000000000..010f8b482 --- /dev/null +++ b/tests/harness/corpus-costs/7525-inspect-custom.cjs.json @@ -0,0 +1 @@ +{ "plain": 4178, "san": 6619 } diff --git a/tests/harness/corpus-costs/7526-inspect-options-runtime.ts.json b/tests/harness/corpus-costs/7526-inspect-options-runtime.ts.json new file mode 100644 index 000000000..0599901b9 --- /dev/null +++ b/tests/harness/corpus-costs/7526-inspect-options-runtime.ts.json @@ -0,0 +1 @@ +{ "plain": 4855, "san": 7570 } diff --git a/tests/harness/errors.test.ts b/tests/harness/errors.test.ts index c8b0b51b8..cfa8680ee 100644 --- a/tests/harness/errors.test.ts +++ b/tests/harness/errors.test.ts @@ -1985,3 +1985,50 @@ console.log("unreachable"); ); }); }); + +// The runtime formatter (format strings known only at run time, spread +// arguments, options objects) throws a coded SC2020 error for renderings +// it has no lowering for. +describe("runtime format refusals", () => { + test("%s of a class instance with toString, %o of objects, colored errors and options", async () => { + const r = await compileAndRun( + "format-runtime-refusals", + `import * as util from "node:util"; +class Named { + name = "n"; + toString(): string { + return this.name; + } +} +const s: string = process.argv.length > 99 ? "" : "%s"; +const o: string = process.argv.length > 99 ? "" : "%o"; +const attempts: Array<() => string> = [ + () => util.format(s, new Named()), + () => util.format(o, { a: 1 }), + () => util.formatWithOptions({ colors: true }, s, "x", new Error("e")), + () => util.formatWithOptions({ maxArrayLength: 1 }, [1, 2]), + () => util.formatWithOptions({ compact: true }, [1, 2]), +]; +for (const attempt of attempts) { + try { + console.log(attempt()); + } catch (error) { + const e = error as Error & { code?: string }; + console.log(e.name, e.code, e.message); + } +} +console.log(util.format(s, { a: 1 }), util.formatWithOptions({ maxArrayLength: 100 }, [1, 2])); +`, + ); + expect(r.exitCode).toBe(0); + expect(r.stdout).toBe( + "Error SC2020 util.format %s of a class instance whose class defines toString has no lowering\n" + + "Error SC2020 util.format %o of objects renders hidden properties, which have no lowering; use %O\n" + + "Error SC2020 colored inspect output of Error values has no lowering\n" + + "Error SC2020 util.formatWithOptions with the maxArrayLength option has no lowering\n" + + "Error SC2020 util.formatWithOptions with this compact option has no lowering (compact must be false or a number)\n" + + "{ a: 1 } [ 1, 2 ]\n", + ); + expect(r.stderr).toBe(""); + }); +}); From e48b2c343508a831bbef57e569b97705586fa2e9 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 22 Oct 2026 21:24:51 -0500 Subject: [PATCH 02/44] Keep held format arguments static and compiling The runtime format branch packed its arguments with a checked-dynamic array literal, a dynamic operation: a console.log with a string first argument put its function in the dynamic tier, and the property-bag analysis read the format calls as possible dynamic stores. The held arguments now go to the formatter as a native list of checked values (insp.formatList and insp.formatPrimitives), which is a tier boundary, and the format and inspect entries join the helpers that never store into a class instance. A first argument whose type may hold a string only through a union, or whose checker type does not map, packed every argument as a checked value and failed to compile when one did not convert (7380's `console.log(top ? [...top] : "none", byName)`). It now takes the held-argument form: the lowered value is tested for a substitution and otherwise joins as before. In TypeScript an argument without a checked-dynamic form is refused at compile time only when the joined form cannot print it either; otherwise the branch that substitutes throws SC2020, as in JavaScript. --- .../compiler/src/backend/llvm/lib-shared.ts | 1 + .../src/frontend/lowering/lower-console.ts | 3 +- .../src/frontend/lowering/lower-inspect.ts | 55 +++++++++++++------ packages/compiler/src/ir/builtin-effects.ts | 1 + .../compiler/src/ir/builtin-signatures.ts | 3 +- packages/compiler/src/ir/int32-slots.ts | 12 ++++ packages/runtime/src/scr_inspect.c | 23 +++++++- packages/runtime/src/scr_inspect.h | 8 ++- 8 files changed, 81 insertions(+), 25 deletions(-) diff --git a/packages/compiler/src/backend/llvm/lib-shared.ts b/packages/compiler/src/backend/llvm/lib-shared.ts index b02cf44f3..365393326 100644 --- a/packages/compiler/src/backend/llvm/lib-shared.ts +++ b/packages/compiler/src/backend/llvm/lib-shared.ts @@ -1036,6 +1036,7 @@ export const LIB_FN_SYMS: Record = { "insp.dynS": "scr_insp_dyn_s", "insp.format": "scr_insp_format_pack", "insp.formatPrimitives": "scr_insp_format_primitives", + "insp.formatList": "scr_insp_format_list", "insp.formatWithOptions": "scr_insp_format_with_options", "insp.inspect": "scr_insp_inspect", "insp.customEnabled": "scr_insp_custom_enabled", diff --git a/packages/compiler/src/frontend/lowering/lower-console.ts b/packages/compiler/src/frontend/lowering/lower-console.ts index db4d39e70..21540cf47 100644 --- a/packages/compiler/src/frontend/lowering/lower-console.ts +++ b/packages/compiler/src/frontend/lowering/lower-console.ts @@ -21,6 +21,7 @@ import { lowerInspectValue, lowerRuntimeFormat, runtimeFormatString, + formatStringConverts, } from "./lower-inspect.js"; import { streamSidesOf } from "./lower-stream.js"; @@ -372,7 +373,7 @@ export function lowerConsoleOutput( return consoleWrite(stdout, [lowerRuntimeFormat(lowerer, nodes, loc)], loc); const runtime = nodes.length > 1 && !isStringLiteralLike(first) ? runtimeFormatString(lowerer, first!) : null; - if (runtime === "other") + if (runtime === "other" && !formatStringConverts(lowerer, first!)) return consoleWrite(stdout, [lowerRuntimeFormat(lowerer, nodes, loc)], loc); if (runtime !== null) { const held = heldFormatArgs(lowerer, nodes, loc); diff --git a/packages/compiler/src/frontend/lowering/lower-inspect.ts b/packages/compiler/src/frontend/lowering/lower-inspect.ts index dd950a2a2..9afe12ed4 100644 --- a/packages/compiler/src/frontend/lowering/lower-inspect.ts +++ b/packages/compiler/src/frontend/lowering/lower-inspect.ts @@ -35,6 +35,7 @@ import { canConvertToDyn, canDynCheckTo, funcOf, + arrayOf, CLASS_FIELD_ATTRIBUTES_PREFIX, classDynViewSupported, DYN_CLASS_PROPERTIES, @@ -3205,7 +3206,8 @@ export function lowerFormatCall( if (!firstIsLiteral) { const runtime = argNodes.length > 1 ? runtimeFormatString(lowerer, first) : null; - if (runtime === "other") return lowerRuntimeFormat(lowerer, argNodes, loc); + if (runtime === "other" && !formatStringConverts(lowerer, first)) + return lowerRuntimeFormat(lowerer, argNodes, loc); if (runtime !== null) { // A string that substitutes formats at run time; any other string // joins with the remaining arguments, as below. @@ -3576,6 +3578,14 @@ function runtimeFenceValue( }; } +/** Whether the held-argument form can test a format string whose type + * only may hold a string: a union that converts to a checked-dynamic value, + * or a value whose checker type does not map (its lowered type decides). */ +export function formatStringConverts(lowerer: Lowerer, first: ts.Expression): boolean { + const t = lowerer.mapTypeOf(lowerer.typeOf(first)); + return t === null || t.kind === "dyn" || lowerer.dynConvertible(t); +} + /** How a non-literal format string is known: "string" and "dyn" values * test for a substitution at run time, "other" types (unions with a string * arm) format at run time, and null types never hold a string. */ @@ -3673,14 +3683,10 @@ export function heldFormatArgs( "util.format of a void call result (call it as its own statement, then pass undefined)", ); } - if (value.type.kind !== "dyn" && !lowerer.dynConvertible(value.type)) { - // JavaScript keeps the space-joined branch, which printed these - // values before, and refuses only a string that substitutes. - if (!isJsSourceFile(ts.getSourceFileOfNode(node))) { - lowerer.noLowering(unconvertibleSurface(value.type), node, UNCONVERTIBLE_HINT); - } + // The space-joined branch prints these values as before; only a string + // that substitutes refuses them, where it runs. + if (value.type.kind !== "dyn" && !lowerer.dynConvertible(value.type)) unconvertible ??= { node, type: value.type }; - } // Unit values have no local: their effects run here and a literal // stands in. const k = value.type.kind; @@ -3694,16 +3700,20 @@ export function heldFormatArgs( return { node, ref: () => varRef(held.id, value.type, loc) }; }); const first = args[0]!.ref(); + // A lowered value that is neither a string nor convertible holds no + // string, so it never substitutes. const specifies: IrExpr = first.type.kind === "string" ? { kind: "libCall", fn: "insp.formatSpecifies", args: [first], type: BOOL, loc } - : { - kind: "libCall", - fn: "insp.formatSpecifiesDyn", - args: [lowerer.coerceToExpected(first, DYN)], - type: BOOL, - loc, - }; + : first.type.kind === "dyn" || lowerer.dynConvertible(first.type) + ? { + kind: "libCall", + fn: "insp.formatSpecifiesDyn", + args: [lowerer.coerceToExpected(first, DYN)], + type: BOOL, + loc, + } + : { kind: "boolLit", value: false, type: BOOL, loc }; const format = (): IrExpr => { if (unconvertible !== null) { const fenced = unconvertible; @@ -3725,14 +3735,18 @@ export function heldFormatArgs( t.kind === "undefinedT" || t.kind === "nullT" || (t.kind === "union" && (lowerer.unions.get(t.unionId)?.arms.every(primitive) ?? false)); + // A native list of the checked values: converting into it is a tier + // boundary, so the statement stays static. return { kind: "libCall", - fn: args.every((arg) => primitive(arg.ref().type)) ? "insp.formatPrimitives" : "insp.format", + fn: args.every((arg) => primitive(arg.ref().type)) + ? "insp.formatPrimitives" + : "insp.formatList", args: [ { - kind: "dynArrLit", + kind: "arrayLit", elems: args.map((arg) => lowerer.coerceToExpected(arg.ref(), DYN)), - type: DYN, + type: arrayOf(DYN), loc, }, ], @@ -3748,6 +3762,11 @@ export function heldFormatArgs( if (t.kind === "bytes") return true; return inspectSupport(lowerer, t, new Set(), { recursive: false }) === null; }); + // Without a joined branch every call formats: TypeScript refuses the + // value at compile time. + const refused = unconvertible as { node: ts.Expression; type: IrType } | null; + if (!joinable && refused !== null && !isJsSourceFile(ts.getSourceFileOfNode(refused.node))) + lowerer.noLowering(unconvertibleSurface(refused.type), refused.node, UNCONVERTIBLE_HINT); return { stmts, args, specifies, format, joinable }; } diff --git a/packages/compiler/src/ir/builtin-effects.ts b/packages/compiler/src/ir/builtin-effects.ts index d676b05dd..357992a5a 100644 --- a/packages/compiler/src/ir/builtin-effects.ts +++ b/packages/compiler/src/ir/builtin-effects.ts @@ -183,6 +183,7 @@ export const MAY_THROW_LIB_FNS: ReadonlySet = new Set([ // The runtime formatter calls toString, valueOf and toJSON, and refuses // renderings it has no lowering for. "insp.format", + "insp.formatList", "insp.formatWithOptions", "insp.inspect", // The options object reads the caller's options (getters included), and diff --git a/packages/compiler/src/ir/builtin-signatures.ts b/packages/compiler/src/ir/builtin-signatures.ts index 6e07e1467..c127af93c 100644 --- a/packages/compiler/src/ir/builtin-signatures.ts +++ b/packages/compiler/src/ir/builtin-signatures.ts @@ -4376,7 +4376,8 @@ export const LIB_FN_SIGS = defineLibFnSignatures({ "insp.format": { argTypes: [DYN], result: STRING }, /** insp.format over arguments that are all strings, numbers, booleans, * null or undefined, without the checked-dynamic walk. */ - "insp.formatPrimitives": { argTypes: [DYN], result: STRING }, + "insp.formatPrimitives": { argTypes: [arrayOf(DYN)], result: STRING }, + "insp.formatList": { argTypes: [arrayOf(DYN)], result: STRING }, "insp.formatWithOptions": { argTypes: [DYN, DYN], result: STRING }, /** util.inspect(value, options, depth, colors) with the call's argument * count: the runtime reads the options and renders the checked-dynamic diff --git a/packages/compiler/src/ir/int32-slots.ts b/packages/compiler/src/ir/int32-slots.ts index ac0747276..6a30eb92b 100644 --- a/packages/compiler/src/ir/int32-slots.ts +++ b/packages/compiler/src/ir/int32-slots.ts @@ -407,6 +407,18 @@ const DYN_READ_ONLY = new Set([ "fs.utimesSync", "insp.dyn", "insp.dynS", + // util.format and util.inspect over checked values: the inspect walk and + // the format conversions read them; insp.registerCustom keeps a class's + // custom inspect thunk in the runtime's registry. + "insp.customIsSelf", + "insp.customResult", + "insp.format", + "insp.formatList", + "insp.formatPrimitives", + "insp.formatSpecifiesDyn", + "insp.formatWithOptions", + "insp.inspect", + "insp.registerCustom", // The dynamic-tier operators run their dyn twins (scr_js_value.c); the // conversions move a value between dyn and jsval, and js.toF64 is // dynCheck's number check. diff --git a/packages/runtime/src/scr_inspect.c b/packages/runtime/src/scr_inspect.c index 2c7951103..da49804c1 100644 --- a/packages/runtime/src/scr_inspect.c +++ b/packages/runtime/src/scr_inspect.c @@ -2575,11 +2575,21 @@ static ScrStr *insp_prim_number(ScrDyn *v, int spec) { static const InspFmtOps insp_fmt_primitive_ops = {insp_prim_s, insp_prim_j, insp_prim_number, insp_prim_inspect}; -ScrStr *scr_insp_format_primitives(ScrDyn *pack) { +/* A native list of checked values (the compiled format calls) as the + * runtime's argument pack. +1. */ +static ScrDyn *insp_list_pack(ScrArr *list) { + ScrDyn *pack = scr_dyn_new_arr(); + for (uint32_t i = 0; i < list->len; i++) scr_dyn_arr_push(pack, scr_arr_get_ref(list, i)); + return pack; +} + +ScrStr *scr_insp_format_primitives(ScrArr *list) { ScrInspFormatOptions o; scr_insp_format_defaults(&o, "util.format"); - if (pack->kind != SCR_DYN_ARR) return scr_str_new("", 0); - return insp_format_loop(pack->v.arr.items, pack->v.arr.len, &o, &insp_fmt_primitive_ops); + ScrDyn *pack = insp_list_pack(list); + ScrStr *out = insp_format_loop(pack->v.arr.items, pack->v.arr.len, &o, &insp_fmt_primitive_ops); + scr_dyn_release(pack); + return out; } /* The arguments of a dyn array pack (+1 text, or NULL pending). */ @@ -2594,6 +2604,13 @@ ScrStr *scr_insp_format_pack(ScrDyn *pack) { return insp_fmt_pack(pack, &o); } +ScrStr *scr_insp_format_list(ScrArr *list) { + ScrDyn *pack = insp_list_pack(list); + ScrStr *out = scr_insp_format_pack(pack); + scr_dyn_release(pack); + return out; +} + /* Whether a format string can substitute: a '%' before its last character. * A string without one formats as its arguments joined with spaces. */ bool scr_insp_format_specifies(ScrStr *first) { diff --git a/packages/runtime/src/scr_inspect.h b/packages/runtime/src/scr_inspect.h index a95ac1fde..339fe781b 100644 --- a/packages/runtime/src/scr_inspect.h +++ b/packages/runtime/src/scr_inspect.h @@ -76,9 +76,13 @@ ScrStr *scr_insp_format(ScrDyn *const *args, size_t argc, const ScrInspFormatOpt /* util.format(...pack) and util.formatWithOptions(options, ...pack) over a * dyn array of the arguments (+1). */ ScrStr *scr_insp_format_pack(ScrDyn *pack); -/* util.format(...pack) when every argument is a string, number, boolean, +/* util.format over a native list of checked values, the compiled calls' + * held arguments (+1). */ +struct ScrArr; +ScrStr *scr_insp_format_list(struct ScrArr *list); +/* scr_insp_format_list when every argument is a string, number, boolean, * null or undefined, without the checked-dynamic walk (+1). */ -ScrStr *scr_insp_format_primitives(ScrDyn *pack); +ScrStr *scr_insp_format_primitives(struct ScrArr *list); ScrStr *scr_insp_format_with_options(ScrDyn *options, ScrDyn *pack); /* util.inspect(value, options, depth, colors) over checked-dynamic * arguments with Node's legacy-argument rules; argc is the call's From 15cc7310aca7924abc26f6d2ef91c0543ab945ef Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Fri, 23 Oct 2026 15:15:34 -0700 Subject: [PATCH 03/44] Keep record values that subtype reduction folds into an inferred type TypeScript infers a function's return type as the subtype-reduced union of the values it returns: returning `{ status }` in one branch and `{ status, value }` (or a call returning either) in another infers just `{ status }`. Records have fixed layouts, so that type cannot hold the second value: the return failed its union wrap at run time ('... is not representable in the target union'), and values that reached a binding of the reduced type failed a checked narrowing. - A function with an inferred return type and no contextual signature returns the union of the record types it actually returns, including through calls of other such functions. Only value types the checker's reduction dropped from the inferred union are added, so functions whose inferred type already names every returned type keep their layout. Declared return types keep their meaning. - Reading such a union through the reduced type (a binding, a member read, JSON.stringify) uses a dedicated checked extraction that converts the wider record arms with the ordinary width coercion; other narrowings are unchanged. Co-Authored-By: Claude Opus 5.5 --- .../src/frontend/lowering/builtins/json.ts | 10 +- .../src/frontend/lowering/coercions/state.ts | 9 +- .../src/frontend/lowering/coercions/unions.ts | 71 ++++++++- .../lowering/expressions/property-access.ts | 20 ++- .../src/frontend/lowering/function-returns.ts | 143 ++++++++++++++++++ .../compiler/src/frontend/lowering/lowerer.ts | 10 +- .../src/frontend/lowering/union-narrow.ts | 17 +++ ...nferred-return-overlapping-records.ts.json | 6 + ...990-inferred-return-overlapping-records.ts | 59 ++++++++ ...nferred-return-overlapping-records.ts.json | 1 + 10 files changed, 340 insertions(+), 6 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7990-inferred-return-overlapping-records.ts.json create mode 100644 tests/corpus/7990-inferred-return-overlapping-records.ts create mode 100644 tests/harness/corpus-costs/7990-inferred-return-overlapping-records.ts.json diff --git a/packages/compiler/src/frontend/lowering/builtins/json.ts b/packages/compiler/src/frontend/lowering/builtins/json.ts index 64f829865..5aad5077f 100644 --- a/packages/compiler/src/frontend/lowering/builtins/json.ts +++ b/packages/compiler/src/frontend/lowering/builtins/json.ts @@ -1,3 +1,4 @@ +import { widensReducedRecords } from "../function-returns.js"; import { keyedHelperName } from "../stable-names.js"; import { isDateJsonSource } from "./dates.js"; import { dynUndefinedExpr, varRef } from "../../../ir/build.js"; @@ -96,7 +97,14 @@ export function lowerJsonMethodCall( !isDateJsonSource(lowerer, argNode) ) { const narrowed = lowerer.mapTypeOf(lowerer.typeOf(argNode)); - if (narrowed && !isUnitType(narrowed) && narrowed.kind !== "void") { + // A union of record types the checker's subtype reduction spelled as + // one of them serializes each value by its own layout. + if ( + narrowed && + !isUnitType(narrowed) && + narrowed.kind !== "void" && + !widensReducedRecords(lowerer, narrowed, value.type) + ) { const helper = narrowed.kind === "union" ? lowerer.narrowedRetagHelper(argNode, value.type.unionId, narrowed.unionId, loc) diff --git a/packages/compiler/src/frontend/lowering/coercions/state.ts b/packages/compiler/src/frontend/lowering/coercions/state.ts index ad1d7c6ea..3c2ecf7bd 100644 --- a/packages/compiler/src/frontend/lowering/coercions/state.ts +++ b/packages/compiler/src/frontend/lowering/coercions/state.ts @@ -12,7 +12,14 @@ export class CoercionState { // Provenance for checked extraction; consumers must not infer it from names. readonly checkedNarrows = new Set(); // The extracted arm of each checked (non-defaulting) extraction helper. - readonly checkedNarrowArms = new Map(); + readonly checkedNarrowArms = new Map< + string, + { unionId: string; tag: number; target: IrType; widens?: true } + >(); readonly retags = new Map(); readonly narrows = new Map(); + readonly widthNarrows = new Map(); + /** Unions widenReducedReturn produced: only these read through the + * width-converting extraction (widthNarrowHelper). */ + readonly widenedReturns = new Set(); } diff --git a/packages/compiler/src/frontend/lowering/coercions/unions.ts b/packages/compiler/src/frontend/lowering/coercions/unions.ts index 434e687a5..776892487 100644 --- a/packages/compiler/src/frontend/lowering/coercions/unions.ts +++ b/packages/compiler/src/frontend/lowering/coercions/unions.ts @@ -421,6 +421,46 @@ export function narrowedArmHelper( return name; } +/** Interned `%union.narrow.width.(u)`: the checked extraction of a + * record arm from a union that also holds records whose fields strictly + * include the arm's (a function's widened inferred return, see + * widenReducedReturn). Those arms convert like any record width coercion; + * every other arm throws like narrowedArmHelper's. */ +export function widthNarrowHelper( + lowerer: Lowerer, + fromId: string, + target: IrType, + loc: SrcLoc, +): string | null { + const from = lowerer.unions.get(fromId); + if (!from || target.kind !== "record") return null; + const tag = lowerer.armTag(fromId, target); + if (tag < 0) return null; + const key = `${fromId}:${tag}`; + const existing = lowerer.coercions.widthNarrows.get(key); + if (existing) return existing; + const name = keyedHelperName(lowerer, "%union.narrow.width", key); + lowerer.coercions.widthNarrows.set(key, name); + const fn = buildUnionNarrow( + name, + from, + target, + loc, + (type) => lowerer.fmt(type), + null, + (armTag, payload) => widthSubtypeConversion(lowerer, from.arms[armTag]!, target, payload, loc), + ); + if (!fn) throw new InternalCompilerError("lowerer bug: invalid checked union extraction"); + lowerer.coercions.checkedNarrowArms.set(name, { + unionId: fromId, + tag, + target: from.arms[tag]!, + widens: true, + }); + lowerer.liftedFns.push(fn); + return name; +} + /** A checked extraction at its use. A plain read tests the stored tag in * place and projects the payload directly, so member reads can borrow it; * every other arm throws the helper's TypeError with the same message. @@ -442,7 +482,8 @@ export function checkedNarrowCall( value.type.kind !== "union" || value.type.unionId !== arm.unionId || !typeEquals(type, arm.target) || - !isSafeToRepeat(value) + !isSafeToRepeat(value) || + arm.widens === true ) return call; const unionId = arm.unionId; @@ -599,3 +640,31 @@ export function deferredReadHelper( lowerer.liftedFns.push(fn); return name; } + +/** Whether `arm` is a record whose fields strictly include those of the + * record `target`: a value of the target type under TypeScript's structural + * typing, which the checker's subtype reduction may have spelled as the + * target (`{ status } | { status, value }` infers `{ status }`). */ +export function isWidthSubtypeRecord(lowerer: Lowerer, arm: IrType, target: IrType): boolean { + if (arm.kind !== "record" || target.kind !== "record") return false; + const own = lowerer.shapes.get(arm.shapeId); + const want = lowerer.shapes.get(target.shapeId); + if (!own || !want || own.tuple || want.tuple || own.indexValue || want.indexValue) return false; + if (want.fields.length >= own.fields.length) return false; + const names = new Set(own.fields.map((f) => f.name)); + return want.fields.every((f) => names.has(f.name)); +} + +/** A checked narrow that meets a width-subtype record arm converts it like + * any record width coercion instead of reporting a lying narrowing. */ +function widthSubtypeConversion( + lowerer: Lowerer, + arm: IrType, + target: IrType, + payload: IrExpr, + loc: SrcLoc, +): IrExpr | null { + if (!isWidthSubtypeRecord(lowerer, arm, target)) return null; + const lift = lowerer.widthLiftPlan(arm, target); + return lift === null ? null : lowerer.applyWidthLift(lift, payload, target, loc); +} diff --git a/packages/compiler/src/frontend/lowering/expressions/property-access.ts b/packages/compiler/src/frontend/lowering/expressions/property-access.ts index 9894cfb71..bac0d0daa 100644 --- a/packages/compiler/src/frontend/lowering/expressions/property-access.ts +++ b/packages/compiler/src/frontend/lowering/expressions/property-access.ts @@ -1,3 +1,5 @@ +import { widensReducedRecords } from "../function-returns.js"; +import { widthNarrowHelper } from "../coercions/unions.js"; /* Property reads (`a.b`): lowerExprInner's property-access branch, intrinsic and union properties, field reads and targets, and match-group reads. */ import { keyedHelperName } from "../stable-names.js"; import { staticallySelected } from "../static-typeof.js"; @@ -1293,7 +1295,23 @@ export function lowerPropertyAccessExpr( // honest one, not a generic recitation about the outer dot. A // receiver that lowers cleanly means the MEMBER is the gap — name // the property and the receiver's type instead of "property access". - const recvLowered = lowerer.lowerExpr(expr.expression); + let recvLowered = lowerer.lowerExpr(expr.expression); + // A receiver whose value union holds record types the checker's subtype + // reduction folded into its type (`{ status } | { status, value }` typed + // `{ status }`, a call of a function with an inferred return type): read + // through the checker's type, converting the wider arms. + if (recvLowered.type.kind === "union") { + const declared = lowerer.mapTypeOf(lowerer.typeOf(expr.expression)); + if ( + declared !== null && + declared.kind === "record" && + widensReducedRecords(lowerer, declared, recvLowered.type) + ) { + const helper = widthNarrowHelper(lowerer, recvLowered.type.unionId, declared, loc); + if (helper !== null) + recvLowered = lowerer.checkedNarrowCall(helper, recvLowered, declared, loc); + } + } // A dyn receiver — JSON.parse's `any`, or `unknown` the checker // narrowed to `object` — reads through the dyn keyed read: member // or undefined (JS's own-property answer), throwing JS's TypeError diff --git a/packages/compiler/src/frontend/lowering/function-returns.ts b/packages/compiler/src/frontend/lowering/function-returns.ts index dde109ea7..b1132a8b9 100644 --- a/packages/compiler/src/frontend/lowering/function-returns.ts +++ b/packages/compiler/src/frontend/lowering/function-returns.ts @@ -11,12 +11,14 @@ import { VOID, isUnitType, typeEquals, + typeKey, } from "../../ir/ir.js"; import { isJsSourceFile } from "../program.js"; import { PoisonError, dynFallbackType } from "./lowerer.js"; import { functionCanReturnUndefined } from "../function-completion.js"; import { jsBindingAppendsOnly, jsBindingHasOpenWrites } from "./lower-stmts.js"; import type { FnCtx } from "./function-context.js"; +import { constituentTypes } from "../ts7/checker.js"; /** Keep the conservative completion rule shared by return construction * and inference. Nested control flow may still make completion unreachable. */ @@ -175,6 +177,8 @@ export function declaredReturnType( // (`.action(() => { throw ... })`). `never` VALUES stay unmapped. if (retTsType.flags & ts.TypeFlags.Never) return VOID; let mappedReturn = lowerer.mapTypeOf(retTsType); + if (mappedReturn !== null) + mappedReturn = widenReducedReturn(lowerer, decl, retTsType, mappedReturn); // A generator declared to return Iterable returns a generator over T. if (mappedReturn && decl.asteriskToken) mappedReturn = iterableGeneratorArm(lowerer, mappedReturn); @@ -522,3 +526,142 @@ export function blockBodyOf(decl: ts.FunctionLikeDeclaration): ts.Block | null { if (ts.isBlock(body)) return body; return ts.forEachChild(decl, (c) => (ts.isBlock(c) ? c : undefined)) ?? null; } + +const widening = new Set(); + +/** An inferred return type is the checker's SUBTYPE-REDUCED union of the + * returned values' types: `return aborted` (`{ status }`) beside `return + * { status, value }` infers just `{ status }`, because the second is a + * subtype of the first. Records have fixed layouts, so that type cannot + * hold the second value: the return would drop `value` or fail the union + * wrap. A function whose return type is inferred therefore returns the + * union of the record types it actually returns. Declared return types keep + * their meaning. */ +function widenReducedReturn( + lowerer: Lowerer, + decl: ts.SignatureDeclaration, + retTsType: ts.Type, + mapped: IrType, +): IrType { + if (decl.type !== undefined || hasExplicitJsDocReturn(decl)) return mapped; + const body = (decl as { body?: ts.Node }).body; + if (body === undefined || decl.asteriskToken !== undefined) return mapped; + if (decl.modifiers?.nodes.some((m) => m.kind === ts.SyntaxKind.AsyncKeyword)) return mapped; + // A function expression with a contextual signature returns that + // signature's type; only free-standing inference is widened. + if ( + (ts.isFunctionExpression(decl) || ts.isArrowFunction(decl)) && + lowerer.checker.getContextualType(decl) !== undefined + ) + return mapped; + const arms = + mapped.kind === "union" ? (lowerer.unions.get(mapped.unionId)?.arms ?? []) : [mapped]; + const recordArms = arms.filter((a) => a.kind === "record"); + if (recordArms.length === 0 || widening.has(decl)) return mapped; + const checkerArms = new Set( + retTsType.flags & ts.TypeFlags.Union ? constituentTypes(retTsType) : [retTsType], + ); + const fieldsOf = (t: IrType): readonly string[] | null => { + if (t.kind !== "record") return null; + const shape = lowerer.shapes.get(t.shapeId); + return shape === undefined || shape.tuple || shape.indexValue !== undefined + ? null + : shape.fields.map((f) => f.name); + }; + const narrower = (part: IrType): boolean => { + const own = fieldsOf(part); + if (own === null || arms.some((a) => typeKey(a) === typeKey(part))) return false; + return recordArms.some((arm) => { + const fields = fieldsOf(arm); + return fields !== null && fields.length < own.length && fields.every((f) => own.includes(f)); + }); + }; + const extra = new Map(); + const returned = (expr: ts.Expression): void => { + // A call of another inferred function returns ITS widened type. + const callee = ts.isCallExpression(expr) + ? lowerer.checker.getResolvedSignature(expr)?.declaration + : undefined; + if ( + callee !== undefined && + ts.isFunctionLike(callee) && + (callee as { body?: ts.Node }).body !== undefined && + callee.type === undefined + ) { + let t: IrType | null = null; + widening.add(decl); + try { + t = declaredReturnType(lowerer, callee, expr); + } catch { + t = null; + } finally { + widening.delete(decl); + } + if (t === null) return; + const parts = t.kind === "union" ? (lowerer.unions.get(t.unionId)?.arms ?? []) : [t]; + for (const part of parts) if (narrower(part)) extra.set(typeKey(part), part); + return; + } + // Only a value type the checker's reduction dropped from the inferred + // union is a candidate; a constituent of the inferred type is already + // represented by it. + const valueType = lowerer.typeOf(expr); + const valueArms = + valueType.flags & ts.TypeFlags.Union ? constituentTypes(valueType) : [valueType]; + for (const part of valueArms) { + if (checkerArms.has(part) || !(part.flags & ts.TypeFlags.Object)) continue; + if (!lowerer.checker.isTypeAssignableTo(part, retTsType)) continue; + const t = lowerer.mapTypeOf(part); + if (t !== null && narrower(t)) extra.set(typeKey(t), t); + } + }; + const scan = (node: ts.Node): void => { + if (node !== decl && ts.isFunctionLike(node)) return; + if (ts.isReturnStatement(node) && node.expression !== undefined) returned(node.expression); + ts.forEachChild(node, scan); + }; + if (ts.isArrowFunction(decl) && !ts.isBlock(body)) returned(body as ts.Expression); + else scan(body); + if (extra.size === 0) return mapped; + const joined = new Map(); + for (const a of [...arms, ...extra.values()]) joined.set(typeKey(a), a); + const sorted = [...joined.values()].sort((a, b) => (typeKey(a) < typeKey(b) ? -1 : 1)); + const unionId = lowerer.unions.intern(sorted); + lowerer.coercions.widenedReturns.add(unionId); + return { kind: "union", unionId }; +} + +/** Whether `wide` is a widened inferred return that holds `narrow`'s arms plus records that are width + * subtypes of `narrow`'s record arms: the shape a subtype-reduced inferred + * type takes next to the values it was inferred from. */ +export function widensReducedRecords(lowerer: Lowerer, narrow: IrType, wide: IrType): boolean { + if (wide.kind !== "union" || typeKey(narrow) === typeKey(wide)) return false; + if (!lowerer.coercions.widenedReturns.has(wide.unionId)) return false; + const narrowArms = + narrow.kind === "union" ? (lowerer.unions.get(narrow.unionId)?.arms ?? []) : [narrow]; + const wideArms = lowerer.unions.get(wide.unionId)?.arms ?? []; + const keys = new Set(wideArms.map(typeKey)); + if (!narrowArms.every((a) => keys.has(typeKey(a)))) return false; + const fieldsOf = (t: IrType): readonly string[] | null => { + if (t.kind !== "record") return null; + const shape = lowerer.shapes.get(t.shapeId); + return shape === undefined || shape.tuple || shape.indexValue !== undefined + ? null + : shape.fields.map((f) => f.name); + }; + const narrowKeys = new Set(narrowArms.map(typeKey)); + const extras = wideArms.filter((a) => !narrowKeys.has(typeKey(a))); + return ( + extras.length > 0 && + extras.every((extra) => { + const own = fieldsOf(extra); + return ( + own !== null && + narrowArms.some((arm) => { + const f = fieldsOf(arm); + return f !== null && f.length < own.length && f.every((x) => own.includes(x)); + }) + ); + }) + ); +} diff --git a/packages/compiler/src/frontend/lowering/lowerer.ts b/packages/compiler/src/frontend/lowering/lowerer.ts index 7aad166e9..4bf938027 100644 --- a/packages/compiler/src/frontend/lowering/lowerer.ts +++ b/packages/compiler/src/frontend/lowering/lowerer.ts @@ -115,6 +115,7 @@ import { checkedNarrowCall, checkedNarrowOperand, deferredReadHelper, + widthNarrowHelper, } from "./coercions/unions.js"; import { lowerUtilTypeValue, @@ -443,7 +444,7 @@ import { bindingNeverReassigned } from "./binding-analysis.js"; import type { InstantiationPath } from "./instantiation-path.js"; import { bodyReadsArguments } from "../arguments-usage.js"; import { completeArgs, wrappedUndefined, undefinedArgFor } from "./call-arguments.js"; -import { bodyReturnType, declaredReturnType } from "./function-returns.js"; +import { bodyReturnType, declaredReturnType, widensReducedRecords } from "./function-returns.js"; import { lowerCall, lowerPromiseMethodCall, lowerFilterNarrowCall } from "./lower-calls.js"; import { lowerFfiCall, validateFfiImports } from "./native-ffi.js"; import { @@ -3464,6 +3465,7 @@ export class Lowerer { this.internTableHelpers, this.coercions.retags, this.coercions.narrows, + this.coercions.widthNarrows, ]) for (const name of table.values()) interned.add(name); for (const fn of [...this.liftedFns, ...this.implicitFns]) @@ -10013,7 +10015,11 @@ export class Lowerer { expected.kind !== "void" && this.armTag(expr.type.unionId, expected) >= 0 ) { - const helper = this.narrowedArmHelper(expr.type.unionId, expected, expr.loc); + // A function's widened inferred return (widenReducedReturn) also + // holds records with more fields; those convert instead of throwing. + const helper = widensReducedRecords(this, expected, expr.type) + ? widthNarrowHelper(this, expr.type.unionId, expected, expr.loc) + : this.narrowedArmHelper(expr.type.unionId, expected, expr.loc); if (helper) { return this.checkedNarrowCall(helper, expr, expected, expr.loc); } diff --git a/packages/compiler/src/frontend/lowering/union-narrow.ts b/packages/compiler/src/frontend/lowering/union-narrow.ts index 3bb09cb26..7ae036fad 100644 --- a/packages/compiler/src/frontend/lowering/union-narrow.ts +++ b/packages/compiler/src/frontend/lowering/union-narrow.ts @@ -28,6 +28,9 @@ export function buildUnionNarrow( loc: SrcLoc, formatType: (type: IrType) => string, undefinedDefault: IrExpr | null = null, + /** Converts another arm's payload into the target (a record arm that is + * a width subtype of a record target), or null to throw. */ + convert: ((tag: number, payload: IrExpr) => IrExpr | null) | null = null, ): IrFunction | null { if (isUnitType(target) || target.kind === "void") return null; const targetTag = from.arms.findIndex((arm) => typeEquals(arm, target)); @@ -43,6 +46,7 @@ export function buildUnionNarrow( const inputType: IrType = { kind: "union", unionId: from.id }; const input: IrExpr = { kind: "varRef", localId: "u.0", type: inputType, loc }; const body: IrStmt[] = []; + let converted: IrExpr | null = null; for (let tag = 0; tag < from.arms.length; tag++) { const arm = from.arms[tag]!; let branch: IrStmt[]; @@ -56,6 +60,19 @@ export function buildUnionNarrow( ]; } else if (arm.kind === "undefinedT" && undefinedDefault !== null) { branch = [{ kind: "return", value: undefinedDefault, loc }]; + } else if ( + convert !== null && + !isUnitType(arm) && + (converted = convert(tag, { + kind: "unionNarrow", + unionId: from.id, + tag, + value: input, + type: arm, + loc, + })) !== null + ) { + branch = [{ kind: "return", value: converted, loc }]; } else { branch = [ { diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7990-inferred-return-overlapping-records.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7990-inferred-return-overlapping-records.ts.json new file mode 100644 index 000000000..6a55f1f2e --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/7990-inferred-return-overlapping-records.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/7990-inferred-return-overlapping-records.ts" + ], + "diags": [] +} diff --git a/tests/corpus/7990-inferred-return-overlapping-records.ts b/tests/corpus/7990-inferred-return-overlapping-records.ts new file mode 100644 index 000000000..c2d1c9ea0 --- /dev/null +++ b/tests/corpus/7990-inferred-return-overlapping-records.ts @@ -0,0 +1,59 @@ +// Functions whose inferred return type is a union of overlapping records. +// TypeScript's subtype reduction infers `{ status }` for a function that +// returns `{ status }` in one branch and `{ status, value }` in another, so +// the inferred type is narrower than the values. Returned values keep their +// fields, chained returns keep them, and bindings typed by the reduced type +// read their declared fields. +type Aborted = { status: string }; +const ABORTED: Aborted = { status: "aborted" }; + +function inner(ok: boolean, value: unknown) { + if (!ok) return ABORTED; + return { status: "valid", value }; +} + +// Returns inner's union through a call: inferred as `{ status }` alone. +function outer(ok: boolean, value: unknown) { + if (value === "stop") return ABORTED; + return inner(ok, value); +} + +function merge(entries: [string, number][]) { + const result: Record = {}; + for (const [key, n] of entries) { + if (key === "__proto__") return ABORTED; + result[key] = n; + } + return { status: "valid", value: result }; +} + +function parse(entries: [string, number][]) { + if (entries.length === 0) return ABORTED; + return merge(entries); +} + +console.log(JSON.stringify(outer(true, [1, 2]))); +console.log(JSON.stringify(outer(false, 1))); +console.log(JSON.stringify(outer(true, "stop"))); +console.log(JSON.stringify(parse([["a", 1], ["b", 2]]))); +console.log(JSON.stringify(parse([]))); + +// Into untyped positions: the value keeps every field. +const seen: unknown[] = []; +seen.push(outer(true, { nested: true })); +seen.push(parse([["c", 3]])); +console.log(JSON.stringify(seen)); +const loose: unknown = outer(true, "kept"); +console.log(JSON.stringify(loose)); + +// Bindings typed by the reduced type: declared fields read as in Node. +const r = outer(true, 42); +console.log(r.status); +let status = ""; +for (const ok of [true, false]) status += outer(ok, 0).status + " "; +console.log(status.trim()); +function statusOf(ok: boolean): string { + const local = outer(ok, null); + return local.status; +} +console.log(statusOf(true), statusOf(false)); diff --git a/tests/harness/corpus-costs/7990-inferred-return-overlapping-records.ts.json b/tests/harness/corpus-costs/7990-inferred-return-overlapping-records.ts.json new file mode 100644 index 000000000..7da9b7545 --- /dev/null +++ b/tests/harness/corpus-costs/7990-inferred-return-overlapping-records.ts.json @@ -0,0 +1 @@ +{ "plain": 3700 } From c9ce961458c83e81d798e181cbb313a1b79e6df2 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Fri, 23 Oct 2026 20:54:08 -0500 Subject: [PATCH 04/44] Restore the inspection context when a custom inspector throws --- .../src/frontend/lowering/lower-inspect.ts | 33 ++++++++++++------ .../compiler/src/ir/builtin-signatures.ts | 7 ++-- .../corpus/8125-inspect-custom-throws.ts.json | 6 ++++ .../8126-inspect-custom-throws.cjs.json | 6 ++++ packages/runtime/src/scr_inspect.c | 34 +++++++++++++++---- packages/runtime/src/scr_inspect.h | 8 ++--- tests/corpus/8125-inspect-custom-throws.ts | 32 +++++++++++++++++ tests/corpus/8126-inspect-custom-throws.cjs | 30 ++++++++++++++++ .../8125-inspect-custom-throws.ts.json | 1 + .../8126-inspect-custom-throws.cjs.json | 1 + 10 files changed, 134 insertions(+), 24 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8125-inspect-custom-throws.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8126-inspect-custom-throws.cjs.json create mode 100644 tests/corpus/8125-inspect-custom-throws.ts create mode 100644 tests/corpus/8126-inspect-custom-throws.cjs create mode 100644 tests/harness/corpus-costs/8125-inspect-custom-throws.ts.json create mode 100644 tests/harness/corpus-costs/8126-inspect-custom-throws.cjs.json diff --git a/packages/compiler/src/frontend/lowering/lower-inspect.ts b/packages/compiler/src/frontend/lowering/lower-inspect.ts index 86255ca56..334c1dcac 100644 --- a/packages/compiler/src/frontend/lowering/lower-inspect.ts +++ b/packages/compiler/src/frontend/lowering/lower-inspect.ts @@ -1277,21 +1277,34 @@ function customInspectHelper( const resultT = found.sig.ret; const call = accessorCall(lowerer, t.className, INSPECT_CUSTOM_MEMBER, v(), args, resultT, loc); // The method runs in a fresh inspect context (insp.customCallBegin); the - // walk's own context returns before the result renders. - const callStmt = (fn: "insp.customCallBegin" | "insp.customCallEnd"): IrStmt => ({ + // walk's own context returns before the result renders. The token lets + // an enclosing end restore this walk's context when the method throws. + locals.push({ id: "tok.0", name: "tok", type: F64, mutable: false }); + const end: IrStmt = { kind: "exprStmt", - expr: { kind: "libCall", fn, args: [], type: VOID, loc }, + expr: { + kind: "libCall", + fn: "insp.customCallEnd", + args: [varRef("tok.0", F64, loc)], + type: VOID, + loc, + }, loc, - }); - const render: IrStmt[] = [...argDecls, callStmt("insp.customCallBegin")]; + }; + const render: IrStmt[] = [ + ...argDecls, + { + kind: "varDecl", + localId: "tok.0", + init: { kind: "libCall", fn: "insp.customCallBegin", args: [], type: F64, loc }, + loc, + }, + ]; if (resultT.kind === "void") { - render.push({ kind: "exprStmt", expr: call, loc }, callStmt("insp.customCallEnd")); + render.push({ kind: "exprStmt", expr: call, loc }, end); } else { locals.push({ id: "res.0", name: "res", type: resultT, mutable: false }); - render.push( - { kind: "varDecl", localId: "res.0", init: call, loc }, - callStmt("insp.customCallEnd"), - ); + render.push({ kind: "varDecl", localId: "res.0", init: call, loc }, end); } const res = (): IrExpr => varRef("res.0", resultT, loc); if (resultT.kind === "string") { diff --git a/packages/compiler/src/ir/builtin-signatures.ts b/packages/compiler/src/ir/builtin-signatures.ts index d8f150605..4730f283d 100644 --- a/packages/compiler/src/ir/builtin-signatures.ts +++ b/packages/compiler/src/ir/builtin-signatures.ts @@ -4492,9 +4492,10 @@ export const LIB_FN_SIGS = defineLibFnSignatures({ "insp.customIsSelf": { argTypes: [DYN, DYN], result: BOOL }, "insp.customResult": { argTypes: [DYN, F64, F64], result: STRING }, /** Around the method's call: the inspect and format calls it makes run - * in a fresh inspect context with the default options. */ - "insp.customCallBegin": { argTypes: [], result: VOID }, - "insp.customCallEnd": { argTypes: [], result: VOID }, + * in a fresh inspect context with the default options. begin answers the + * token end takes; end also discards what a call that threw left. */ + "insp.customCallBegin": { argTypes: [], result: F64 }, + "insp.customCallEnd": { argTypes: [F64], result: VOID }, /** Whether a runtime format string has a '%' before its last character. */ "insp.formatSpecifies": { argTypes: [STRING], result: BOOL }, "insp.formatSpecifiesDyn": { argTypes: [DYN], result: BOOL }, diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8125-inspect-custom-throws.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8125-inspect-custom-throws.ts.json new file mode 100644 index 000000000..1dc598fd4 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8125-inspect-custom-throws.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/8125-inspect-custom-throws.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8126-inspect-custom-throws.cjs.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8126-inspect-custom-throws.cjs.json new file mode 100644 index 000000000..374de8db3 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8126-inspect-custom-throws.cjs.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/8126-inspect-custom-throws.cjs" + ], + "diags": [] +} diff --git a/packages/runtime/src/scr_inspect.c b/packages/runtime/src/scr_inspect.c index 9905518ab..9cfe3a3ef 100644 --- a/packages/runtime/src/scr_inspect.c +++ b/packages/runtime/src/scr_inspect.c @@ -1706,6 +1706,8 @@ typedef struct { double break_length; double compact; const ScrDyn *user_options; + size_t nsaved; /* the nested-context stack's depth before the call */ + size_t nframes; /* the frame stack's depth before the call */ } InspCallSaved; static SCR_TL InspCallSaved *g_call_saved; @@ -1721,8 +1723,13 @@ static void insp_call_saved_cleanup(void) { /* A custom inspect function's call: Node gives each util.inspect and * util.format call its own ctx, so the calls the function makes start * with no indentation, no seen objects, no colors and the default layout - * and options. end restores the walk's context. */ -void scr_insp_custom_call_begin(void) { + * and options. begin answers a token; end(token) restores the walk's + * context as it was at that begin. A call that threw skips its end, and + * an enclosing custom function's end (or the walk's, when the function + * caught the error) discards what the throw left: the contexts of calls + * that never ended and the frames of renders that never finished. */ +double scr_insp_custom_call_begin(void) { + size_t nsaved = g_nsaved; scr_insp_nested_begin(); if (g_ncall_saved == g_call_saved_cap) { if (g_call_saved_cap == 0) scr_atexit(insp_call_saved_cleanup); @@ -1730,7 +1737,10 @@ void scr_insp_custom_call_begin(void) { g_call_saved = realloc(g_call_saved, g_call_saved_cap * sizeof(InspCallSaved)); if (!g_call_saved) insp_oom(); } + double token = (double)g_ncall_saved; InspCallSaved *sv = &g_call_saved[g_ncall_saved++]; + sv->nsaved = nsaved; + sv->nframes = g_nframes; sv->colors = g_colors; sv->custom_inspect = g_custom_inspect; sv->sorted = g_sorted; @@ -1747,10 +1757,20 @@ void scr_insp_custom_call_begin(void) { g_depth_number = 0; g_user_options = NULL; scr_insp_layout(80, 3, false); + return token; } -void scr_insp_custom_call_end(void) { - InspCallSaved *sv = &g_call_saved[--g_ncall_saved]; +void scr_insp_custom_call_end(double token) { + size_t depth = (size_t)token; + if (depth >= g_ncall_saved) return; /* an enclosing end already restored it */ + InspCallSaved *sv = &g_call_saved[depth]; + g_ncall_saved = depth; + while (g_nframes > sv->nframes) { + InspFrame *f = &g_frames[--g_nframes]; + for (size_t i = 0; i < f->len; i++) scr_str_release(f->items[i]); + free(f->items); + } + g_pending_seen = NULL; g_colors = sv->colors; g_custom_inspect = sv->custom_inspect; g_no_break = sv->no_break; @@ -1758,7 +1778,7 @@ void scr_insp_custom_call_end(void) { g_depth_number = sv->depth_number; g_user_options = sv->user_options; scr_insp_layout(sv->break_length, sv->compact, sv->sorted); - scr_insp_nested_restore(); + while (g_nsaved > sv->nsaved) scr_insp_nested_restore(); } /* The style inspect.styles maps a styleType to, as [open, close] codes: @@ -2024,7 +2044,7 @@ static ScrStr *insp_custom(ScrDyn *d, double recurse, double depth, bool *thrown } args[2] = insp_inspect_value(); ScrDyn *ret; - scr_insp_custom_call_begin(); + double token = scr_insp_custom_call_begin(); if (compiled) { ScrDyn *call[4] = {d, args[0], args[1], args[2]}; ret = scr_dyn_call(fn, call, 4, "[nodejs.util.inspect.custom]"); @@ -2033,7 +2053,7 @@ static ScrStr *insp_custom(ScrDyn *d, double recurse, double depth, bool *thrown ret = scr_dyn_call(fn, args, 3, "[nodejs.util.inspect.custom]"); scr_dyn_this_pop(); } - scr_insp_custom_call_end(); + scr_insp_custom_call_end(token); for (size_t i = 0; i < 3; i++) scr_dyn_release(args[i]); scr_dyn_release(fn); if (!ret) { diff --git a/packages/runtime/src/scr_inspect.h b/packages/runtime/src/scr_inspect.h index d56f17bba..9619f0dee 100644 --- a/packages/runtime/src/scr_inspect.h +++ b/packages/runtime/src/scr_inspect.h @@ -127,10 +127,10 @@ void scr_insp_nested_restore(void); * value for the caller to restore. */ bool scr_insp_custom_inspect(bool enabled); /* Around a custom inspect function's call: begin gives the calls it makes - * a fresh inspect context with the default options; end restores the - * caller's. */ -void scr_insp_custom_call_begin(void); -void scr_insp_custom_call_end(void); + * a fresh inspect context with the default options and answers a token; + * end(token) restores the caller's, also after calls that threw. */ +double scr_insp_custom_call_begin(void); +void scr_insp_custom_call_end(double token); /* How the unbounded depth of the renders that follow was given, for custom * inspect functions: 0 Infinity, 1 null, 2 undefined. Answers the * previous kind for the caller to restore. */ diff --git a/tests/corpus/8125-inspect-custom-throws.ts b/tests/corpus/8125-inspect-custom-throws.ts new file mode 100644 index 000000000..33077a32b --- /dev/null +++ b/tests/corpus/8125-inspect-custom-throws.ts @@ -0,0 +1,32 @@ +// A [util.inspect.custom] method that throws inside an outer inspect: an +// enclosing custom method that catches the error, and the walk around it, +// continue with their own indentation, circular references and colors. +import * as util from "node:util"; +class Throws { + [util.inspect.custom](): string { + throw new Error("inner"); + } +} +class Catches { + [util.inspect.custom](): string { + let note = ""; + try { + note = util.inspect({ a: { b: [new Throws()] } }); + } catch (e) { + note = "caught " + (e as Error).message; + } + return note + " then " + util.inspect({ x: { y: [1, 2] } }); + } +} +const shared = { k: 1 }; +type Outer = { first: { deep: Catches }; list: unknown[]; s: object; t: object; me?: Outer }; +const outer: Outer = { first: { deep: new Catches() }, list: [new Catches(), { z: 3 }], s: shared, t: shared }; +outer.me = outer; +console.log(util.inspect(outer, { breakLength: 50 })); +console.log(JSON.stringify(util.inspect([new Catches(), "s"], { colors: true }))); +try { + console.log(util.inspect({ a: { b: new Throws() } })); +} catch (e) { + console.log("outer caught", (e as Error).message); +} +console.log({ after: { deep: { x: 1 } }, list: [1, 2, 3] }); diff --git a/tests/corpus/8126-inspect-custom-throws.cjs b/tests/corpus/8126-inspect-custom-throws.cjs new file mode 100644 index 000000000..8d4e97608 --- /dev/null +++ b/tests/corpus/8126-inspect-custom-throws.cjs @@ -0,0 +1,30 @@ +// A [util.inspect.custom] method that throws inside an outer inspect: an +// enclosing custom method that catches the error, and the walk around it, +// continue with their own indentation and colors. +const util = require("node:util"); +class Throws { + [util.inspect.custom]() { + throw new Error("inner"); + } +} +class Catches { + [util.inspect.custom]() { + let note = ""; + try { + note = util.inspect({ a: { b: [new Throws()] } }); + } catch (e) { + note = "caught " + e.message; + } + return note + " then " + util.inspect({ x: { y: [1, 2] } }); + } +} +const shared = { k: 1 }; +const outer = { first: { deep: new Catches() }, list: [new Catches(), { z: 3 }], s: shared, t: shared }; +console.log(util.inspect(outer, { breakLength: 50 })); +console.log(JSON.stringify(util.inspect([new Catches(), "s"], { colors: true }))); +try { + console.log(util.inspect({ a: { b: new Throws() } })); +} catch (e) { + console.log("outer caught", e.message); +} +console.log({ after: { deep: { x: 1 } }, list: [1, 2, 3] }); diff --git a/tests/harness/corpus-costs/8125-inspect-custom-throws.ts.json b/tests/harness/corpus-costs/8125-inspect-custom-throws.ts.json new file mode 100644 index 000000000..7d8f08f56 --- /dev/null +++ b/tests/harness/corpus-costs/8125-inspect-custom-throws.ts.json @@ -0,0 +1 @@ +{ "plain": 3090, "san": 5295 } diff --git a/tests/harness/corpus-costs/8126-inspect-custom-throws.cjs.json b/tests/harness/corpus-costs/8126-inspect-custom-throws.cjs.json new file mode 100644 index 000000000..49ea4c6d4 --- /dev/null +++ b/tests/harness/corpus-costs/8126-inspect-custom-throws.cjs.json @@ -0,0 +1 @@ +{ "plain": 2826, "san": 4308 } From 908d87d3e0bdcb3ba91e3d6ad21722bd9a819392 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Sat, 24 Oct 2026 13:32:14 -0500 Subject: [PATCH 05/44] Key widened returns by their type-parameter bindings and skip self-calls Two review findings on the widened-return cache: - A nested function lowered under different type-parameter bindings reused the record layout of the first instance it was widened in. In `function outer(x: T) { const inner = () => ...; return inner(); }`, the closures of outer and outer returned the records of outer, and returning one threw a TypeError. A kept result now records the bindings it was mapped under (typeParamBindings, its checker-type and implicit-any twins, and whether an erased body is lowering) and answers only under the same ones. - A function that calls itself on one return cut its own widening, so its result was never kept and a chain of such functions forwarding to each other on two branches was scanned 2^depth times. A call of the function itself returns its own type, whose arms add nothing, so the scan now skips it. --- .../src/frontend/lowering/coercions/state.ts | 18 +++++- .../src/frontend/lowering/function-returns.ts | 25 ++++++++- .../compiler/test/widened-returns.test.ts | 55 ++++++++++++++++++- 3 files changed, 92 insertions(+), 6 deletions(-) diff --git a/packages/compiler/src/frontend/lowering/coercions/state.ts b/packages/compiler/src/frontend/lowering/coercions/state.ts index 27667197c..cbc577634 100644 --- a/packages/compiler/src/frontend/lowering/coercions/state.ts +++ b/packages/compiler/src/frontend/lowering/coercions/state.ts @@ -1,5 +1,6 @@ import type { IrType } from "../../../ir/ir.js"; import type * as ts from "../../ts7/adapter.js"; +import type { Lowerer } from "../lowerer.js"; /** Per-compilation conversion state. Recursive plans are scoped to the * active planning stack; only fully validated copy routes are memoized. */ @@ -24,6 +25,19 @@ export class CoercionState { * width-converting extraction (widthNarrowHelper). */ readonly widenedReturns = new Set(); /** widenReducedReturn's results by declaration and inferred return type, - * for the results computed without cutting a recursive widening. */ - readonly widenedReturnTypes = new Map>(); + * for the results computed without cutting a widening cycle through + * another function. Each keeps the type-parameter bindings it was mapped + * under and answers only under the same bindings. */ + readonly widenedReturnTypes = new Map>(); +} + +/** A widened inferred return type and the lowering state that mapped it + * (Lowerer.typeParamBindings and its checker-type and implicit-any twins, + * and whether an erased generic body is lowering). */ +export interface WidenedReturn { + type: IrType; + bindings: Lowerer["typeParamBindings"]; + tsBindings: Lowerer["typeParamTsBindings"]; + implicit: Lowerer["implicitParamTypes"]; + erasing: boolean; } diff --git a/packages/compiler/src/frontend/lowering/function-returns.ts b/packages/compiler/src/frontend/lowering/function-returns.ts index 0b87710e0..4d586fcc3 100644 --- a/packages/compiler/src/frontend/lowering/function-returns.ts +++ b/packages/compiler/src/frontend/lowering/function-returns.ts @@ -584,15 +584,30 @@ function widenReducedReturn( return mapped; } // Functions that forward to the same callees along several return - // branches read each callee's widened type once. + // branches read each callee's widened type once. A nested function + // lowered under other type-parameter bindings maps its returns again. const known = lowerer.coercions.widenedReturnTypes.get(decl)?.get(retTsType); - if (known !== undefined) return known; + const erasing = lowerer.erasingDepth > 0; + if ( + known !== undefined && + known.bindings === lowerer.typeParamBindings && + known.tsBindings === lowerer.typeParamTsBindings && + known.implicit === lowerer.implicitParamTypes && + known.erasing === erasing + ) + return known.type; const cutsBefore = wideningCuts; const widened = widenReturnArms(lowerer, decl, retTsType, mapped, arms, recordArms); if (wideningCuts === cutsBefore) { let byType = lowerer.coercions.widenedReturnTypes.get(decl); if (byType === undefined) lowerer.coercions.widenedReturnTypes.set(decl, (byType = new Map())); - byType.set(retTsType, widened); + byType.set(retTsType, { + type: widened, + bindings: lowerer.typeParamBindings, + tsBindings: lowerer.typeParamTsBindings, + implicit: lowerer.implicitParamTypes, + erasing, + }); } return widened; } @@ -638,6 +653,10 @@ function widenReturnArms( (callee as { body?: ts.Node }).body !== undefined && callee.type === undefined ) { + // A call of the function itself returns its own type, whose arms + // add nothing. Reading it would cut the widening and keep this + // result from being reused. + if (callee === decl) return; let t: IrType | null = null; widening.add(decl); try { diff --git a/packages/compiler/test/widened-returns.test.ts b/packages/compiler/test/widened-returns.test.ts index 63d24c1da..fe78c6b75 100644 --- a/packages/compiler/test/widened-returns.test.ts +++ b/packages/compiler/test/widened-returns.test.ts @@ -22,7 +22,7 @@ async function lowerModule(source: string): Promise { /** Functions that each forward to the previous one along two return * branches, above one whose inferred type the checker reduced. */ -function forwardingChain(depth: number): string { +function forwardingChain(depth: number, recursive = false): string { const lines = [ "export {};", "type Short = { status: string };", @@ -35,6 +35,7 @@ function forwardingChain(depth: number): string { for (let i = 1; i <= depth; i++) lines.push( `function f${i}(a: Short, b: Long, n: number) {`, + ...(recursive ? [` if (n > 20) return f${i}(a, b, n - 1);`] : []), ` if (n > 1) return f${i - 1}(a, b, n - 1);`, ` if (n > 0) return f${i - 1}(a, b, n + 1);`, " return a;", @@ -53,3 +54,55 @@ test("return widening reads each forwarded callee's type once", { timeout: 60_00 expect(top).toBeDefined(); expect(top!.returnType.kind).toBe("union"); }); + +test( + "return widening reads each forwarded callee's type once when the callees call themselves", + { timeout: 60_000 }, + async () => { + const mod = await lowerModule(forwardingChain(40, true)); + const top = mod.functions.find((f) => f.name === "f40"); + expect(top).toBeDefined(); + expect(top!.returnType.kind).toBe("union"); + }, +); + +test("a nested function's widened return follows each instance's type arguments", async () => { + const mod = await lowerModule( + [ + "export {};", + "function outer(x: T, n: number) {", + " const inner = () => {", + " if (n > 0) return { status: x, value: n };", + " return { status: x };", + " };", + " return inner();", + "}", + "console.log(JSON.stringify(outer(1, 1)));", + 'console.log(JSON.stringify(outer("s", 1)));', + "console.log(JSON.stringify(outer(true, 0)));", + ].join("\n"), + ); + const shapes = new Map((mod.records ?? []).map((r) => [r.id, r])); + const unions = new Map((mod.unions ?? []).map((u) => [u.id, u])); + // Each closure returns records whose `status` has the type its captured + // `x` has. + const closures = mod.functions.filter( + (f) => f.name.startsWith("%fn.") && !f.name.includes("adapt"), + ); + expect(closures.length).toBe(3); + for (const closure of closures) { + const captured = closure.captures?.find((c) => c.type.kind !== "f64")?.type.kind ?? "f64"; + const ret = closure.returnType; + expect(ret.kind).toBe("union"); + const arms = ret.kind === "union" ? (unions.get(ret.unionId)?.arms ?? []) : []; + expect(arms.length).toBe(2); + for (const arm of arms) { + expect(arm.kind).toBe("record"); + const status = + arm.kind === "record" + ? shapes.get(arm.shapeId)?.fields.find((f) => f.name === "status") + : undefined; + expect(status?.type.kind).toBe(captured); + } + } +}); From c41af6c36dc2a741ee762dbc57c485f39cb6225f Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Sat, 24 Oct 2026 19:04:38 -0500 Subject: [PATCH 06/44] Move the MIME handle tags to 81 and 82 Main now uses tag 78 (STRING_DECODER), 79 (MODULE_NAMESPACE, after #2161) and 80; 81 and 82 are the next free values. --- packages/runtime/src/scr_dyn.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/runtime/src/scr_dyn.h b/packages/runtime/src/scr_dyn.h index 4fcba7bfe..f7014f259 100644 --- a/packages/runtime/src/scr_dyn.h +++ b/packages/runtime/src/scr_dyn.h @@ -186,8 +186,8 @@ typedef enum { X(MATCH_SOURCE, 55) /* a match result's pending metadata (scr_regex.c) */ \ X(MESSAGE_EVENT, 50) /* ScrEvent with data — BroadcastChannel's 'message' */ \ X(MESSAGE_PORT, 36) \ - X(MIME_PARAMS, 78) /* util.MIMEParams (scr_mime.c) */ \ - X(MIME_TYPE, 79) /* util.MIMEType (scr_mime.c) */ \ + X(MIME_PARAMS, 81) /* util.MIMEParams (scr_mime.c) */ \ + X(MIME_TYPE, 82) /* util.MIMEType (scr_mime.c) */ \ X(NET_BLOCK_LIST, 48) /* net.BlockList (scr_net.c) */ \ X(NET_SERVER, 3) /* ScrNetServer — net.Server (http.Server rides the same handle) */ \ X(NET_SOCKET, 2) /* ScrNetSocket — net.Socket */ \ From 320a97bf3b8dd57c35376dfedcb5cb67aef44051 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Sat, 24 Oct 2026 20:19:50 -0500 Subject: [PATCH 07/44] Throw Node's TypeError for util.diff of two distinct empty sequences util.diff([], []) with two arrays read one element past its trace row while backtracking. Node's myersDiff answers undefined for two empty sequences, and util.diff's ArrayPrototypeReverse throws "Cannot convert undefined or null to object"; scr_util_diff now throws the same TypeError before it allocates the trace. --- packages/runtime/src/scr_util_callable.c | 11 +++++++++++ tests/corpus/7705-util-diff.cjs | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/packages/runtime/src/scr_util_callable.c b/packages/runtime/src/scr_util_callable.c index df6a68272..574b9d423 100644 --- a/packages/runtime/src/scr_util_callable.c +++ b/packages/runtime/src/scr_util_callable.c @@ -881,6 +881,17 @@ ScrDyn *scr_util_diff(ScrDyn *actual, ScrDyn *expected) { scr_throw_error_msg_code(SCR_ERR_RANGE, message, (size_t)n, "ERR_OUT_OF_RANGE"); return NULL; } + /* Two distinct empty sequences: Node's myersDiff reads past its + * one-element state, finds no end and answers undefined, which the + * ArrayPrototypeReverse of util.diff rejects. The trace below would be read + * out of bounds. */ + if (max == 0) { + uc_diff_seq_free(&a); + uc_diff_seq_free(&e); + static const char message[] = "Cannot convert undefined or null to object"; + scr_throw_error_msg(SCR_ERR_TYPE, message, sizeof message - 1); + return NULL; + } size_t width = 2 * max + 1; int32_t *v = calloc(width, sizeof *v); int32_t **trace = malloc((max + 1) * sizeof *trace); diff --git a/tests/corpus/7705-util-diff.cjs b/tests/corpus/7705-util-diff.cjs index 9e52fd165..489586a3b 100644 --- a/tests/corpus/7705-util-diff.cjs +++ b/tests/corpus/7705-util-diff.cjs @@ -7,7 +7,7 @@ const show = (a, e) => { try { console.log(JSON.stringify(util.diff(a, e))); } catch (err) { console.log(err.name, err.code, err.message); } }; show("abc", "abd"); show("kitten", "sitting"); show("", "x"); show("same", "same"); show("", ""); -show(["a", "b", "c"], ["a", "c", "d"]); show(["x"], []); show([], ["y", "z"]); +show(["a", "b", "c"], ["a", "c", "d"]); show(["x"], []); show([], ["y", "z"]); show([], []); const arr = ["p"]; show(arr, arr); show(1, 1); show(1, 2); show("a", 2); show(["a", 1], ["a"]); show(["a"], [, "b"]); show(null, "x"); show("x", undefined); show("héllo😀", "hello😀"); show({}, "x"); show("the quick brown fox", "the quack brown box"); From e6a8d735ca0355d24c3e57fb3a5320119aeab75f Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Sat, 24 Oct 2026 20:20:39 -0500 Subject: [PATCH 08/44] Bound strict throttle state and evaluate MIME arguments before the call A strict throttler kept every invocation timestamp until its whole window emptied, so sustained traffic grew the array without bound; ticks that left the window now give their slots back, and at most limit ticks are live. new MIMEType, new MIMEParams and MIMEType.parse in JavaScript evaluated surplus arguments after the call. All arguments now evaluate first, in order, as in JavaScript, so a surplus argument's effects happen before the constructor coerces or throws. --- .../src/frontend/lowering/builtins/mime.ts | 80 +++++++++---------- packages/runtime/src/scr_util_timing.c | 7 ++ tests/corpus/7707-util-mime-type.cjs | 8 ++ 3 files changed, 55 insertions(+), 40 deletions(-) diff --git a/packages/compiler/src/frontend/lowering/builtins/mime.ts b/packages/compiler/src/frontend/lowering/builtins/mime.ts index da3d16712..27f95226c 100644 --- a/packages/compiler/src/frontend/lowering/builtins/mime.ts +++ b/packages/compiler/src/frontend/lowering/builtins/mime.ts @@ -2,7 +2,7 @@ import * as ts from "../../ts7/adapter.js"; import { type Lowerer } from "../lowerer.js"; import { locOf } from "../../program.js"; import { dynUndefinedExpr, numLit } from "../../../ir/build.js"; -import { BOOL, DYN, type IrExpr } from "../../../ir/ir.js"; +import { BOOL, DYN, type IrExpr, type IrLibFn } from "../../../ir/ir.js"; import { builtinImportOf } from "./module-bindings.js"; /* node:util's MIMEType and MIMEParams are native checked-dynamic handles @@ -34,38 +34,51 @@ function mimeValueClass(lowerer: Lowerer, node: ts.Expression): MimeClass | null return lowerer.isStdlibSymbol(sym) ? name : null; } -/** Every argument in order, as checked-dynamic values; the first `count` - * fill the call, missing ones as undefined, and the others still - * evaluate. */ -function mimeArguments( +/** The call of `fn` over the first `count` arguments, as checked-dynamic + * values (missing ones as undefined). Every argument evaluates in order + * before the call, as in JavaScript: with surplus arguments, the used ones + * are held in hidden locals while the others evaluate. */ +function mimeCall( lowerer: Lowerer, + fn: IrLibFn, nodes: readonly ts.Expression[], count: number, loc: ReturnType, -): { used: IrExpr[]; extra: IrExpr[] } { - const values = nodes.map((arg) => lowerer.lowerExprExpecting(arg, DYN)); - const used = values.slice(0, count); - while (used.length < count) used.push(dynUndefinedExpr(loc)); - return { used, extra: values.slice(count) }; -} - -/** The call, after which any surplus arguments evaluate. */ -function withSurplus( - lowerer: Lowerer, - call: IrExpr, - extra: IrExpr[], - loc: ReturnType, ): IrExpr { - if (extra.length === 0) return call; - const local = lowerer.declareHiddenLocal("%mimeResult", call.type); + const values = nodes.map((arg) => lowerer.lowerExprExpecting(arg, DYN)); + if (values.length <= count) { + const args = [...values]; + while (args.length < count) args.push(dynUndefinedExpr(loc)); + return { kind: "libCall", fn, args, type: DYN, loc }; + } + const locals = values.slice(0, count).map((value) => ({ + value, + local: lowerer.declareHiddenLocal("%mimeArg", DYN), + })); return { kind: "seqExpr", stmts: [ - { kind: "varDecl", localId: local.id, init: call, loc }, - ...extra.map((arg) => ({ kind: "exprStmt" as const, expr: arg, loc: arg.loc })), + ...locals.map(({ value, local }) => ({ + kind: "varDecl" as const, + localId: local.id, + init: value, + loc, + })), + ...values.slice(count).map((arg) => ({ kind: "exprStmt" as const, expr: arg, loc: arg.loc })), ], - result: { kind: "varRef", localId: local.id, type: call.type, loc }, - type: call.type, + result: { + kind: "libCall", + fn, + args: locals.map(({ local }): IrExpr => ({ + kind: "varRef", + localId: local.id, + type: DYN, + loc, + })), + type: DYN, + loc, + }, + type: DYN, loc, }; } @@ -77,15 +90,8 @@ export function lowerMimeNew(lowerer: Lowerer, expr: ts.NewExpression): IrExpr | const nodes = expr.arguments?.nodes ?? []; if (nodes.some(ts.isSpreadElement)) return null; const loc = locOf(expr); - const { used, extra } = mimeArguments(lowerer, nodes, cls === "MIMEType" ? 2 : 0, loc); - const call: IrExpr = { - kind: "libCall", - fn: cls === "MIMEType" ? "mime.typeNew" : "mime.paramsNew", - args: used, - type: DYN, - loc, - }; - return withSurplus(lowerer, call, extra, loc); + const fn: IrLibFn = cls === "MIMEType" ? "mime.typeNew" : "mime.paramsNew"; + return mimeCall(lowerer, fn, nodes, cls === "MIMEType" ? 2 : 0, loc); } const INSTANCE_METHODS: Record> = { @@ -113,13 +119,7 @@ export function lowerMimeCall(lowerer: Lowerer, call: ts.CallExpression): IrExpr const name = access.name.text; const loc = locOf(call); if (name === "parse" && mimeClassOf(lowerer, access.expression) === "MIMEType") { - const { used, extra } = mimeArguments(lowerer, call.arguments.nodes, 1, loc); - return withSurplus( - lowerer, - { kind: "libCall", fn: "mime.typeParse", args: used, type: DYN, loc }, - extra, - loc, - ); + return mimeCall(lowerer, "mime.typeParse", call.arguments.nodes, 1, loc); } const cls = mimeValueClass(lowerer, access.expression); if (cls === null || !INSTANCE_METHODS[cls].has(name)) return null; diff --git a/packages/runtime/src/scr_util_timing.c b/packages/runtime/src/scr_util_timing.c index c5393da27..a8a700d47 100644 --- a/packages/runtime/src/scr_util_timing.c +++ b/packages/runtime/src/scr_util_timing.c @@ -392,6 +392,13 @@ static void th_reserve(UtState *s, double now) { s->window_count++; return; } + /* Ticks that left the window free their slots: at most limit ticks are + * live, so the storage stays within twice that. */ + if (s->tick_len == s->tick_cap && s->tick_head > 0) { + s->tick_len -= s->tick_head; + memmove(s->ticks, s->ticks + s->tick_head, s->tick_len * sizeof *s->ticks); + s->tick_head = 0; + } if (s->tick_len == s->tick_cap) { size_t cap = s->tick_cap ? s->tick_cap * 2 : 8; double *grown = realloc(s->ticks, cap * sizeof *grown); diff --git a/tests/corpus/7707-util-mime-type.cjs b/tests/corpus/7707-util-mime-type.cjs index 06d159306..e860d1d30 100644 --- a/tests/corpus/7707-util-mime-type.cjs +++ b/tests/corpus/7707-util-mime-type.cjs @@ -76,4 +76,12 @@ console.log(seen, `${m}`); const bare = new MIMEType("a/b"); console.log(Object.keys(bare), bare.foo, bare.params.size); console.log(format("%s|%s", bare, bare.params), "%s", new MIMEType("c/d;e=f")); + // Every argument evaluates before the constructor or parse runs. + const order = []; + const note = (label) => { order.push(label); return label; }; + const coerced = { toString() { order.push("coerce"); return "a/b"; } }; + show("surplus, then coercion", () => { new MIMEType(coerced, undefined, note("third")); return order.splice(0).join(","); }); + show("surplus, then parse error", () => { try { new MIMEType("bad", undefined, note("x")); } catch (e) { order.push(e.code); } return order.splice(0).join(","); }); + show("parse surplus", () => { MIMEType.parse(coerced, note("second")); return order.splice(0).join(","); }); + show("params surplus", () => { new MIMEParams(note("p")); return order.splice(0).join(","); }); } From f716dec7567fc2deffc7a05dcdef5bfe69802ad3 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Sat, 24 Oct 2026 23:26:01 -0500 Subject: [PATCH 09/44] Convert Buffers to and from latin1 and hex text a word at a time - toString("latin1"): counts the high bytes eight at a time, copies words of ASCII bytes, and writes other bytes from a table of their UTF-8 pairs without branches. ASCII-only results are marked ASCII, and other results record their UTF-16 length (one unit per byte), so `.length` does not walk the string. - Buffer.from(text, "latin1" | "ascii" | "binary"): copies text known to be ASCII, copies words of ASCII bytes, and decodes eight bytes of one- and two-byte sequences in one word, storing each output byte without branches. Longer sequences take the previous path. - Buffer.from(text, "hex"): reads digit values from a table. - toString("hex") and toString("base64"/"base64url") mark their results ASCII, so a later decode, length or index read skips the scan for non-ASCII bytes. --- .../corpus/8005-buffer-encoding-paths.ts.json | 6 + packages/runtime/src/scr_bytes.c | 214 +++++++++++++++--- packages/runtime/src/scr_string.c | 13 ++ packages/runtime/src/scr_string.h | 3 + tests/corpus/8005-buffer-encoding-paths.ts | 105 +++++++++ .../8005-buffer-encoding-paths.ts.json | 1 + 6 files changed, 315 insertions(+), 27 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8005-buffer-encoding-paths.ts.json create mode 100644 tests/corpus/8005-buffer-encoding-paths.ts create mode 100644 tests/harness/corpus-costs/8005-buffer-encoding-paths.ts.json diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8005-buffer-encoding-paths.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8005-buffer-encoding-paths.ts.json new file mode 100644 index 000000000..7d6dac570 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8005-buffer-encoding-paths.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/8005-buffer-encoding-paths.ts" + ], + "diags": [] +} diff --git a/packages/runtime/src/scr_bytes.c b/packages/runtime/src/scr_bytes.c index e6df837c8..c1638596c 100644 --- a/packages/runtime/src/scr_bytes.c +++ b/packages/runtime/src/scr_bytes.c @@ -1577,6 +1577,7 @@ static ScrStr *scr_bytes_encode_b64(const uint8_t *in, size_t n, bool url) { if (!url) out[o++] = '='; } out[outlen] = '\0'; + scr_str_note_ascii(s); return s; } @@ -2497,6 +2498,106 @@ static ScrStr *scr_bytes_decode_utf16le(const uint8_t *in, size_t n) { return s; } +/* The UTF-8 bytes of U+0000..U+00FF: the byte itself below 0x80 (the + * second entry is overwritten by the next write), else a two-byte + * sequence. */ +static const uint8_t scr_latin1_utf8[256][2] = { + {0x00, 0x00}, {0x01, 0x00}, {0x02, 0x00}, {0x03, 0x00}, {0x04, 0x00}, {0x05, 0x00}, {0x06, 0x00}, {0x07, 0x00}, + {0x08, 0x00}, {0x09, 0x00}, {0x0a, 0x00}, {0x0b, 0x00}, {0x0c, 0x00}, {0x0d, 0x00}, {0x0e, 0x00}, {0x0f, 0x00}, + {0x10, 0x00}, {0x11, 0x00}, {0x12, 0x00}, {0x13, 0x00}, {0x14, 0x00}, {0x15, 0x00}, {0x16, 0x00}, {0x17, 0x00}, + {0x18, 0x00}, {0x19, 0x00}, {0x1a, 0x00}, {0x1b, 0x00}, {0x1c, 0x00}, {0x1d, 0x00}, {0x1e, 0x00}, {0x1f, 0x00}, + {0x20, 0x00}, {0x21, 0x00}, {0x22, 0x00}, {0x23, 0x00}, {0x24, 0x00}, {0x25, 0x00}, {0x26, 0x00}, {0x27, 0x00}, + {0x28, 0x00}, {0x29, 0x00}, {0x2a, 0x00}, {0x2b, 0x00}, {0x2c, 0x00}, {0x2d, 0x00}, {0x2e, 0x00}, {0x2f, 0x00}, + {0x30, 0x00}, {0x31, 0x00}, {0x32, 0x00}, {0x33, 0x00}, {0x34, 0x00}, {0x35, 0x00}, {0x36, 0x00}, {0x37, 0x00}, + {0x38, 0x00}, {0x39, 0x00}, {0x3a, 0x00}, {0x3b, 0x00}, {0x3c, 0x00}, {0x3d, 0x00}, {0x3e, 0x00}, {0x3f, 0x00}, + {0x40, 0x00}, {0x41, 0x00}, {0x42, 0x00}, {0x43, 0x00}, {0x44, 0x00}, {0x45, 0x00}, {0x46, 0x00}, {0x47, 0x00}, + {0x48, 0x00}, {0x49, 0x00}, {0x4a, 0x00}, {0x4b, 0x00}, {0x4c, 0x00}, {0x4d, 0x00}, {0x4e, 0x00}, {0x4f, 0x00}, + {0x50, 0x00}, {0x51, 0x00}, {0x52, 0x00}, {0x53, 0x00}, {0x54, 0x00}, {0x55, 0x00}, {0x56, 0x00}, {0x57, 0x00}, + {0x58, 0x00}, {0x59, 0x00}, {0x5a, 0x00}, {0x5b, 0x00}, {0x5c, 0x00}, {0x5d, 0x00}, {0x5e, 0x00}, {0x5f, 0x00}, + {0x60, 0x00}, {0x61, 0x00}, {0x62, 0x00}, {0x63, 0x00}, {0x64, 0x00}, {0x65, 0x00}, {0x66, 0x00}, {0x67, 0x00}, + {0x68, 0x00}, {0x69, 0x00}, {0x6a, 0x00}, {0x6b, 0x00}, {0x6c, 0x00}, {0x6d, 0x00}, {0x6e, 0x00}, {0x6f, 0x00}, + {0x70, 0x00}, {0x71, 0x00}, {0x72, 0x00}, {0x73, 0x00}, {0x74, 0x00}, {0x75, 0x00}, {0x76, 0x00}, {0x77, 0x00}, + {0x78, 0x00}, {0x79, 0x00}, {0x7a, 0x00}, {0x7b, 0x00}, {0x7c, 0x00}, {0x7d, 0x00}, {0x7e, 0x00}, {0x7f, 0x00}, + {0xc2, 0x80}, {0xc2, 0x81}, {0xc2, 0x82}, {0xc2, 0x83}, {0xc2, 0x84}, {0xc2, 0x85}, {0xc2, 0x86}, {0xc2, 0x87}, + {0xc2, 0x88}, {0xc2, 0x89}, {0xc2, 0x8a}, {0xc2, 0x8b}, {0xc2, 0x8c}, {0xc2, 0x8d}, {0xc2, 0x8e}, {0xc2, 0x8f}, + {0xc2, 0x90}, {0xc2, 0x91}, {0xc2, 0x92}, {0xc2, 0x93}, {0xc2, 0x94}, {0xc2, 0x95}, {0xc2, 0x96}, {0xc2, 0x97}, + {0xc2, 0x98}, {0xc2, 0x99}, {0xc2, 0x9a}, {0xc2, 0x9b}, {0xc2, 0x9c}, {0xc2, 0x9d}, {0xc2, 0x9e}, {0xc2, 0x9f}, + {0xc2, 0xa0}, {0xc2, 0xa1}, {0xc2, 0xa2}, {0xc2, 0xa3}, {0xc2, 0xa4}, {0xc2, 0xa5}, {0xc2, 0xa6}, {0xc2, 0xa7}, + {0xc2, 0xa8}, {0xc2, 0xa9}, {0xc2, 0xaa}, {0xc2, 0xab}, {0xc2, 0xac}, {0xc2, 0xad}, {0xc2, 0xae}, {0xc2, 0xaf}, + {0xc2, 0xb0}, {0xc2, 0xb1}, {0xc2, 0xb2}, {0xc2, 0xb3}, {0xc2, 0xb4}, {0xc2, 0xb5}, {0xc2, 0xb6}, {0xc2, 0xb7}, + {0xc2, 0xb8}, {0xc2, 0xb9}, {0xc2, 0xba}, {0xc2, 0xbb}, {0xc2, 0xbc}, {0xc2, 0xbd}, {0xc2, 0xbe}, {0xc2, 0xbf}, + {0xc3, 0x80}, {0xc3, 0x81}, {0xc3, 0x82}, {0xc3, 0x83}, {0xc3, 0x84}, {0xc3, 0x85}, {0xc3, 0x86}, {0xc3, 0x87}, + {0xc3, 0x88}, {0xc3, 0x89}, {0xc3, 0x8a}, {0xc3, 0x8b}, {0xc3, 0x8c}, {0xc3, 0x8d}, {0xc3, 0x8e}, {0xc3, 0x8f}, + {0xc3, 0x90}, {0xc3, 0x91}, {0xc3, 0x92}, {0xc3, 0x93}, {0xc3, 0x94}, {0xc3, 0x95}, {0xc3, 0x96}, {0xc3, 0x97}, + {0xc3, 0x98}, {0xc3, 0x99}, {0xc3, 0x9a}, {0xc3, 0x9b}, {0xc3, 0x9c}, {0xc3, 0x9d}, {0xc3, 0x9e}, {0xc3, 0x9f}, + {0xc3, 0xa0}, {0xc3, 0xa1}, {0xc3, 0xa2}, {0xc3, 0xa3}, {0xc3, 0xa4}, {0xc3, 0xa5}, {0xc3, 0xa6}, {0xc3, 0xa7}, + {0xc3, 0xa8}, {0xc3, 0xa9}, {0xc3, 0xaa}, {0xc3, 0xab}, {0xc3, 0xac}, {0xc3, 0xad}, {0xc3, 0xae}, {0xc3, 0xaf}, + {0xc3, 0xb0}, {0xc3, 0xb1}, {0xc3, 0xb2}, {0xc3, 0xb3}, {0xc3, 0xb4}, {0xc3, 0xb5}, {0xc3, 0xb6}, {0xc3, 0xb7}, + {0xc3, 0xb8}, {0xc3, 0xb9}, {0xc3, 0xba}, {0xc3, 0xbb}, {0xc3, 0xbc}, {0xc3, 0xbd}, {0xc3, 0xbe}, {0xc3, 0xbf}, +}; + +/* Each byte is U+00XX ("binary" normalizes here upstream): a byte below + * 0x80 is one UTF-8 byte and any other byte is two. Eight bytes without a + * high bit copy as one word; other bytes take their two table bytes and + * advance by their length, without branches (the terminator slot holds + * the last spare byte). */ +/* The number of bytes from 0x80: eight per step, each step adding its + * high bits into byte lanes that are summed every 255 steps. */ +static size_t scr_bytes_count_high(const uint8_t *in, size_t n) { + const uint64_t ones = UINT64_C(0x0101010101010101); + const uint64_t pairs = UINT64_C(0x00ff00ff00ff00ff); + size_t high = 0, i = 0; + while (n - i >= 8) { + size_t steps = (n - i) / 8 < 255 ? (n - i) / 8 : 255; + uint64_t lanes = 0; + for (size_t end = i + steps * 8; i < end; i += 8) { + uint64_t w; + memcpy(&w, in + i, 8); + lanes += (w >> 7) & ones; + } + lanes = (lanes & pairs) + ((lanes >> 8) & pairs); + high += (size_t)((lanes * UINT64_C(0x0001000100010001)) >> 48); + } + for (; i < n; i++) high += in[i] >> 7; + return high; +} + +static ScrStr *scr_bytes_latin1_text(const uint8_t *in, size_t n) { + size_t high = scr_bytes_count_high(in, n); + if (n > SIZE_MAX - high) scr_bytes_oom(); + ScrStr *s = scr_str_alloc_raw(n + high, n + high); + uint8_t *out = (uint8_t *)s->data; + if (high == 0) { + memcpy(out, in, n); + out[n] = '\0'; + scr_str_note_ascii(s); + return s; + } + const uint64_t top = UINT64_C(0x8080808080808080); + size_t i = 0, o = 0; + while (n - i >= 8) { + uint64_t w; + memcpy(&w, in + i, 8); + if (!(w & top)) { + memcpy(out + o, &w, 8); + i += 8; + o += 8; + continue; + } + for (size_t end = i + 8; i < end; i++) { + memcpy(out + o, scr_latin1_utf8[in[i]], 2); + o += 1 + (in[i] >> 7); + } + } + for (; i < n; i++) { + memcpy(out + o, scr_latin1_utf8[in[i]], 2); + o += 1 + (in[i] >> 7); + } + out[o] = '\0'; + scr_str_note_utf16_len(s, n); + return s; +} + ScrStr *scr_bytes_to_str(const ScrBytes *b, const ScrStr *enc) { SCR_BYTES_SNAPSHOT(b); const uint8_t *in = b->data; @@ -2512,21 +2613,12 @@ ScrStr *scr_bytes_to_str(const ScrBytes *b, const ScrStr *enc) { out[i * 2 + 1] = scr_hex_digits[in[i] & 0xf]; } out[n * 2] = '\0'; + scr_str_note_ascii(s); return s; } if (scr_enc_is(enc, "base64")) return scr_bytes_encode_b64(in, n, false); if (scr_enc_is(enc, "base64url")) return scr_bytes_encode_b64(in, n, true); - if (scr_enc_is(enc, "latin1")) { - /* Each byte is U+00XX ("binary" normalizes here upstream). */ - size_t high = 0; - for (size_t i = 0; i < n; i++) high += in[i] >> 7; - if (n > SIZE_MAX - high) scr_bytes_oom(); - ScrStr *s = scr_str_alloc_raw(n + high, n + high); - size_t o = 0; - for (size_t i = 0; i < n; i++) o = scr_bytes_put_cp(s->data, o, in[i]); - s->data[o] = '\0'; - return s; - } + if (scr_enc_is(enc, "latin1")) return scr_bytes_latin1_text(in, n); if (scr_enc_is(enc, "ascii")) { /* Node's ascii decode masks the high bit: byte & 0x7f. */ ScrStr *s = scr_str_alloc_raw(n, n); @@ -2621,12 +2713,25 @@ ScrStr *scr_bytes_to_str_checked_range(const ScrBytes *b, const ScrStr *enc, return out; } -static int scr_hex_val(uint8_t c) { - if (c >= '0' && c <= '9') return c - '0'; - if (c >= 'a' && c <= 'f') return c - 'a' + 10; - if (c >= 'A' && c <= 'F') return c - 'A' + 10; - return -1; -} +/* Hex digit values, -1 for any other byte. */ +static const int8_t scr_hex_values[256] = { + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, -1, -1, -1, -1, -1, -1, + -1, 10, 11, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, 10, 11, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, +}; /* Both alphabets are accepted by both Buffer base64 encoding names. */ static const int8_t scr_b64_values[256] = { @@ -2720,17 +2825,71 @@ static ScrBytes *scr_bytes_encode_str(ScrBytesSink *sink, const ScrStr *s, const size_t n = s->len; if (scr_enc_is(enc, "latin1") || scr_enc_is(enc, "ascii")) { /* Node writes charCodeAt(i) & 0xFF for BOTH spellings — an astral - * code point contributes its two surrogates' low bytes. */ + * code point contributes its two surrogates' low bytes. ASCII text + * copies. Elsewhere eight ASCII bytes copy as one word. In eight bytes + * holding two or more high bytes and no sequence longer than two + * bytes, every continuation byte's output byte (the code point's low + * byte, from the continuation and the lead before it) is computed in + * one word; each byte is then stored without branches, a continuation + * replacing the byte its lead wrote. */ ScrBytes *b = scr_bytes_conversion_storage_in(sink, n); + if (scr_str_known_ascii(s)) { + memcpy(b->data, in, n); + b->len = n; + return b; + } + /* A local copy: a byte store could otherwise change b->data. */ + uint8_t *dst = b->data; + const uint64_t top = UINT64_C(0x8080808080808080); size_t o = 0, i = 0; while (i < n) { + if (n - i >= 8) { + uint64_t w; + memcpy(&w, in + i, 8); + uint64_t high = w & top; + if (!high) { + memcpy(dst + o, &w, 8); + i += 8; + o += 8; + continue; + } +#if defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + /* Byte k of the input is bits 8k..8k+7 of w. */ + if (i > 0 && (high & (high - 1)) && !(w & (w << 1) & (w << 2) & top)) { + uint64_t prev = (w << 8) | in[i - 1]; + uint64_t cont = w & ~(w << 1) & top; + uint64_t low = ((prev & UINT64_C(0x0303030303030303)) << 6) | + (w & UINT64_C(0x3f3f3f3f3f3f3f3f)); + uint64_t mask = (cont >> 7) * 0xff; + uint64_t out = (w & ~mask) | (low & mask); + for (unsigned k = 0; k < 8; k++) { + o -= (size_t)(cont >> (8 * k + 7)) & 1; + dst[o++] = (uint8_t)(out >> (8 * k)); + } + i += 8; + continue; + } +#endif + } + unsigned c = in[i]; + if (c < 0x80) { + dst[o++] = (uint8_t)c; + i++; + continue; + } + if (c >> 6 == 2) { + /* The rest of a two-byte sequence whose lead ended a word. */ + dst[o - 1] = (uint8_t)(((in[i - 1] & 3u) << 6) | (c & 0x3f)); + i++; + continue; + } uint32_t cp = scr_bytes_next_cp(in, &i); if (cp > 0xffff) { cp -= 0x10000; - b->data[o++] = (uint8_t)(0xd800 + (cp >> 10)); - b->data[o++] = (uint8_t)(0xdc00 + (cp & 0x3ff)); + dst[o++] = (uint8_t)(0xd800 + (cp >> 10)); + dst[o++] = (uint8_t)(0xdc00 + (cp & 0x3ff)); } else { - b->data[o++] = (uint8_t)cp; + dst[o++] = (uint8_t)cp; } } b->len = o; /* never grows past n: multi-byte cps shrink */ @@ -2762,7 +2921,7 @@ static ScrBytes *scr_bytes_encode_str(ScrBytesSink *sink, const ScrStr *s, const /* Node interprets encoded text as the low byte of each UTF-16 unit. * ASCII can borrow storage; non-ASCII must preserve that truncation. */ ScrBytes *narrowed = NULL; - if (scr_bytes_ascii_prefix(in, n) != n) { + if (!scr_str_known_ascii(s) && scr_bytes_ascii_prefix(in, n) != n) { ScrStr *latin1 = scr_str_new("latin1", 6); narrowed = scr_bytes_encode_str(NULL, s, latin1); scr_str_release(latin1); @@ -2773,11 +2932,12 @@ static ScrBytes *scr_bytes_encode_str(ScrBytesSink *sink, const ScrStr *s, const if (hex) { /* Stop at the first invalid pair, discarding an odd tail. */ b = scr_bytes_conversion_storage_in(sink, n / 2); - size_t o = 0; - for (size_t i = 0; i + 1 < n; i += 2) { - int hi = scr_hex_val(in[i]), lo = scr_hex_val(in[i + 1]); - if (hi < 0 || lo < 0) break; - b->data[o++] = (uint8_t)((hi << 4) | lo); + uint8_t *dst = b->data; + size_t pairs = n / 2, o = 0; + for (; o < pairs; o++) { + int hi = scr_hex_values[in[2 * o]], lo = scr_hex_values[in[2 * o + 1]]; + if ((hi | lo) < 0) break; + dst[o] = (uint8_t)((hi << 4) | lo); } b->len = o; } else { diff --git a/packages/runtime/src/scr_string.c b/packages/runtime/src/scr_string.c index 2d7f46d85..a5cc15ba8 100644 --- a/packages/runtime/src/scr_string.c +++ b/packages/runtime/src/scr_string.c @@ -1991,6 +1991,19 @@ double scr_str_utf16_len(ScrStr *s) { return (double)(int64_t)units; } +/* Records the UTF-16 length of a string whose producer counted it (latin1 + * text has one unit per byte), so scr_str_utf16_len does not walk it. Only + * strings that scr_str_utf16_len would otherwise record are kept. */ +void scr_str_note_utf16_len(ScrStr *s, size_t units) { + if (s->ascii || s->len <= 64 || s->rc == SIZE_MAX) return; + if (!scr_u16len_tab) { + scr_u16len_tab = calloc(SCR_U16LEN_N, sizeof *scr_u16len_tab); + if (!scr_u16len_tab) return; + scr_sidx_register_cleanup(); + } + *scr_u16len_slot(s) = (ScrU16Len){s, units}; +} + double scr_str_char_code_at(ScrStr *s, double i) { /* A sequential scan of a non-ASCII receiver stays inside the bounded * ASCII span cached at its last resolved position. Its unit offset is an diff --git a/packages/runtime/src/scr_string.h b/packages/runtime/src/scr_string.h index fc2f15562..e269498b1 100644 --- a/packages/runtime/src/scr_string.h +++ b/packages/runtime/src/scr_string.h @@ -83,6 +83,9 @@ static inline void scr_str_hash_forget(ScrStr *s) { /* True only when every byte is proven below 0x80 (see above). */ static inline bool scr_str_known_ascii(const ScrStr *s) { return s->ascii; } +/* Record the UTF-16 length a string's producer counted (scr_string.c). */ +void scr_str_note_utf16_len(ScrStr *s, size_t units); + /* Record a proven-ASCII fact. Immortal strings may live in read-only * memory and are shared across threads, so they are never written. */ static inline void scr_str_note_ascii(ScrStr *s) { diff --git a/tests/corpus/8005-buffer-encoding-paths.ts b/tests/corpus/8005-buffer-encoding-paths.ts new file mode 100644 index 000000000..014f22abf --- /dev/null +++ b/tests/corpus/8005-buffer-encoding-paths.ts @@ -0,0 +1,105 @@ +// Buffer latin1, ascii, hex and base64 conversions in both directions: +// ASCII, binary and mixed contents at sizes around eight-byte words and the +// pool threshold, strings with code points past U+00FF, invalid hex and +// base64 text, and the lengths and characters of the strings produced. + +let seed = 12345; +function next(): number { + seed = (seed * 1103515245 + 12345) & 0x7fffffff; + return seed >>> 8; +} + +function fill(size: number, kind: string): Buffer { + const buf = Buffer.alloc(size); + for (let i = 0; i < size; i++) { + const r = next(); + if (kind === "ascii") buf[i] = 32 + (r % 95); + else if (kind === "binary") buf[i] = r & 255; + else if (kind === "high") buf[i] = 128 + (r % 128); + else buf[i] = r % 23 === 0 ? 160 + (r % 96) : 97 + (r % 26); + } + return buf; +} + +function digest(text: string): number { + let h = 0; + for (let i = 0; i < text.length; i++) h = (h * 31 + text.charCodeAt(i)) >>> 0; + return h; +} + +const sizes = [0, 1, 7, 8, 9, 15, 16, 17, 63, 64, 65, 4095, 4096, 4097, 70001]; +for (const kind of ["ascii", "binary", "high", "sparse"]) { + const rows: string[] = []; + for (const size of sizes) { + const buf = fill(size, kind); + const latin1 = buf.toString("latin1"); + const back = Buffer.from(latin1, "latin1"); + const ascii = Buffer.from(latin1, "ascii"); + const binary = Buffer.from(buf.toString("binary"), "binary"); + const hex = buf.toString("hex"); + const b64 = buf.toString("base64"); + const url = buf.toString("base64url"); + rows.push( + [ + size, + latin1.length, + digest(latin1), + back.equals(buf), + ascii.equals(buf), + binary.equals(buf), + Buffer.from(hex, "hex").equals(buf), + Buffer.from(b64, "base64").equals(buf), + Buffer.from(url, "base64url").equals(buf), + hex.length, + b64.length, + Buffer.byteLength(latin1, "latin1"), + size > 2 ? latin1.charCodeAt(size - 2) : -1, + size > 4 ? latin1.slice(1, 4).length : -1, + ].join(","), + ); + } + console.log(kind, rows.join(" ")); +} + +// Strings with code points past U+00FF keep the low byte of each UTF-16 +// unit, and an astral code point gives its two surrogates' low bytes. +const wide = [ + "abcéÿ", + "ĀŁ߿", + "xࠀ€￿y", + "\u{1F600}a\u{10FFFF}", + "é".repeat(9) + "€" + "è".repeat(9), + "aaaaaaaébbbbbbbb中cccccccc\u{1F600}", + "ÀÁÂÃÄÅÆÇࠀÈ", +]; +for (const text of wide) { + console.log( + Buffer.from(text, "latin1").toString("hex"), + Buffer.from(text, "ascii").toString("hex"), + Buffer.byteLength(text, "latin1"), + ); +} +const mixed = "é".repeat(5000) + "中" + "z".repeat(5000) + "ÿ".repeat(3000); +const mixedBytes = Buffer.from(mixed, "latin1"); +console.log(mixedBytes.length, mixedBytes[0], mixedBytes[5000], mixedBytes[5001], mixedBytes[13000]); + +// Hex stops at the first invalid pair and drops an odd last digit. +for (const text of ["0aFf", "0aF", "0a1g22", "zz00", "", "ABCDEF0123456789", "0aİ0b", "00a"]) { + console.log(JSON.stringify(text), Buffer.from(text, "hex").toString("hex"), Buffer.byteLength(text, "hex")); +} +const longHex = "0123456789abcdef".repeat(600) + "0x" + "ff".repeat(10); +console.log(Buffer.from(longHex, "hex").length); + +// base64 skips characters outside both alphabets and stops at padding. +for (const text of ["aGVsbG8=", "aGVsbG8", "aGV sbG8=", "aGV\nsbG8", "-_-_", "+/+/", "aGVsbG8=d29ybGQ=", "a", "ab", "abc", "aébc", "====", "YQ"]) { + console.log(JSON.stringify(text), Buffer.from(text, "base64").toString("hex"), Buffer.from(text, "base64url").toString("hex")); +} + +// Strings produced by hex and base64 behave as ASCII text afterwards. +const produced = fill(5000, "binary"); +const producedHex = produced.toString("hex"); +const joined = producedHex + "é" + produced.toString("base64"); +console.log(joined.length, joined.indexOf("é"), digest(joined.slice(9990, 10020))); +const latinText = fill(9000, "high").toString("latin1"); +const appended = latinText + "€"; +console.log(latinText.length, appended.length, appended.charCodeAt(8999), appended.charCodeAt(9000), appended.slice(8998).length); diff --git a/tests/harness/corpus-costs/8005-buffer-encoding-paths.ts.json b/tests/harness/corpus-costs/8005-buffer-encoding-paths.ts.json new file mode 100644 index 000000000..3aafbe296 --- /dev/null +++ b/tests/harness/corpus-costs/8005-buffer-encoding-paths.ts.json @@ -0,0 +1 @@ +{ "plain": 2558, "san": 4160 } From 12bbb9bde6069414b66a363696bad7d2bfc986b3 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Sun, 25 Oct 2026 03:00:30 -0500 Subject: [PATCH 10/44] Keep a throttler's pending-call storage within its live calls A queue that never emptied kept the slots of calls already taken, so its storage grew with lifetime traffic even when maxPending bounded the live calls. Before growing, the queue now moves its live calls to the front. --- packages/runtime/src/scr_util_timing.c | 7 +++++++ tests/corpus/7711-util-throttle.cjs | 12 ++++++++++++ 2 files changed, 19 insertions(+) diff --git a/packages/runtime/src/scr_util_timing.c b/packages/runtime/src/scr_util_timing.c index a8a700d47..13651f5ff 100644 --- a/packages/runtime/src/scr_util_timing.c +++ b/packages/runtime/src/scr_util_timing.c @@ -303,6 +303,13 @@ static size_t ut_pending_count(const UtState *s) { } static void ut_push(UtState *s, ScrPromise *promise /* moves in */, ScrDyn **args /* moves in */, size_t argc) { + /* Calls already taken free their slots, so a queue that never empties + * keeps its storage within twice its live calls. */ + if (s->len == s->cap && s->head > 0) { + s->len -= s->head; + memmove(s->calls, s->calls + s->head, s->len * sizeof *s->calls); + s->head = 0; + } if (s->len == s->cap) { size_t cap = s->cap ? s->cap * 2 : 4; UtCall *grown = realloc(s->calls, cap * sizeof *grown); diff --git a/tests/corpus/7711-util-throttle.cjs b/tests/corpus/7711-util-throttle.cjs index d786535ef..f776e7174 100644 --- a/tests/corpus/7711-util-throttle.cjs +++ b/tests/corpus/7711-util-throttle.cjs @@ -35,6 +35,18 @@ async function main() { console.log(c.activeCount, c.pendingCount); console.log(await ca, await cb, order, c.activeCount); + // A queue that never empties while calls are taken and others arrive. + const q = util.throttle(async (x) => { await null; return x; }, 1000, 1000, { concurrency: 1 }); + const queued = []; + for (let i = 0; i < 5; i++) queued.push(q(i)); + const results = []; + for (let i = 5; i < 25; i++) { + results.push(await queued[i - 5]); + queued.push(q(i)); + } + for (let i = 20; i < 25; i++) results.push(await queued[i]); + console.log(results.join(","), q.pendingCount, q.activeCount); + // A throwing fn rejects that call only. const e = util.throttle((x) => { if (x === 2) throw new RangeError("two"); return x; }, 5, 10); await settle("e1", e(1)); From c2e6c962e84988d9e178b82eb80b9841f4cdbec9 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Sun, 25 Oct 2026 06:49:46 -0500 Subject: [PATCH 11/44] Build the sanitized three.js nodes fixture in its own CI job On the san 1/20 shard the sanitized nodes build shares the runner with the corpus's other two workers and the side queue. It took 944 s and 860 s on #2194's runs and passed 1200 s on main's 7fa88083, failing the test timeout. The Linux san corpus now skips that one case (SCRIPTC_THREE_SANITIZED_NODES_JOB), and a three_nodes_san job runs it alone with the corpus's LeakSanitizer settings. The required test check waits for the new job. Every other three.js case, the plain nodes build and the macOS suites are unchanged. --- .github/workflows/ci.yml | 33 ++++++++++++++++++++++++++ scripts/ci-test-plan.mjs | 10 ++++++++ tests/harness/ci-test-plan.test.ts | 37 ++++++++++++++++++++++++++++++ tests/harness/three.test.ts | 9 ++++++++ 4 files changed, 89 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e949339fc..2b690b100 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -346,6 +346,37 @@ jobs: - run: tar -xzf "$RUNNER_TEMP/native-linux-x64-gnu.tar.gz" - run: pnpm test tests/harness/effect.test.ts tests/harness/effect4.test.ts + # The sanitized three.js nodes build alone: beside the san corpus's other + # workers it took 860 to more than 1200 seconds of its shard + # (SCRIPTC_THREE_SANITIZED_NODES_JOB in scripts/ci-test-plan.mjs). It keeps + # the corpus's LeakSanitizer settings. + three_nodes_san: + needs: [validation_scope, lanes, native_linux] + if: ${{ needs.validation_scope.outputs.full == 'true' && needs.lanes.outputs.linux == 'true' }} + name: test (Linux, san, three.js nodes) + runs-on: ubuntu-24.04 + timeout-minutes: 40 + env: + SCRIPTC_SAN: "1" + ASAN_OPTIONS: detect_leaks=1 + LSAN_OPTIONS: suppressions=${{ github.workspace }}/tests/harness/lsan.supp:print_suppressions=0 + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v4 + with: + version: 11.1.3 + - uses: actions/setup-node@v4 + with: + node-version-file: .node-version + cache: pnpm + - run: pnpm install --frozen-lockfile + - uses: actions/download-artifact@v4 + with: + name: native-linux-x64-gnu + path: ${{ runner.temp }} + - run: tar -xzf "$RUNNER_TEMP/native-linux-x64-gnu.tar.gz" + - run: pnpm test tests/harness/three.test.ts -t "'nodes' runs statically through LLVM on 'native'" + # Build each instrumented seed once. Downstream contracts retain the same # compiler and source artifacts without repeating seed construction. bootstrap_seed: @@ -950,6 +981,7 @@ jobs: tests_macos, native_linux, effects, + three_nodes_san, bootstrap_seed, bootstrap, linux_host_clang, @@ -982,6 +1014,7 @@ jobs: lane tests_macos "${{ needs.tests_macos.result }}" "${{ needs.lanes.outputs.macos }}" lane native_linux "${{ needs.native_linux.result }}" "${{ needs.lanes.outputs.linux }}" lane effects "${{ needs.effects.result }}" "${{ needs.lanes.outputs.effects }}" + lane three_nodes_san "${{ needs.three_nodes_san.result }}" "${{ needs.lanes.outputs.linux }}" lane bootstrap_seed "${{ needs.bootstrap_seed.result }}" "${{ needs.lanes.outputs.bootstrap }}" lane bootstrap "${{ needs.bootstrap.result }}" "${{ needs.lanes.outputs.bootstrap }}" lane linux_host_clang "${{ needs.linux_host_clang.result }}" "${{ needs.lanes.outputs.linux }}" diff --git a/scripts/ci-test-plan.mjs b/scripts/ci-test-plan.mjs index 184f04d1c..88a77e32e 100644 --- a/scripts/ci-test-plan.mjs +++ b/scripts/ci-test-plan.mjs @@ -134,6 +134,15 @@ export const leakSuppressions = fileURLToPath( new URL("../tests/harness/lsan.supp", import.meta.url), ); +// The sanitized three.js nodes build runs in its own Linux job +// (three_nodes_san in .github/workflows/ci.yml). Beside the corpus's other +// workers it took 860 to more than 1200 seconds, past its test timeout. +export const sanitizedNodesJobEnv = { SCRIPTC_THREE_SANITIZED_NODES_JOB: "1" }; + +function corpusThreeEnv(flavor, platform) { + return flavor === "san" && platform === "linux" ? sanitizedNodesJobEnv : {}; +} + function corpusLeakEnv(flavor, platform) { if (flavor !== "san" || platform !== "linux") return {}; return { @@ -246,6 +255,7 @@ export function ciTestPlan({ env: { ...env, ...corpusLeakEnv(flavor, platform), + ...corpusThreeEnv(flavor, platform), SCRIPTC_TEST_WORKERS: String(workers - sideWorkers), SCRIPTC_LLVM_TEST_MODE: "dev", }, diff --git a/tests/harness/ci-test-plan.test.ts b/tests/harness/ci-test-plan.test.ts index 29268c841..e67e5582b 100644 --- a/tests/harness/ci-test-plan.test.ts +++ b/tests/harness/ci-test-plan.test.ts @@ -12,6 +12,7 @@ import { isolatedFiles, leakSuppressions, memoryHeavyFiles, + sanitizedNodesJobEnv, sideOwners, testCosts, } from "../../scripts/ci-test-plan.mjs"; @@ -90,6 +91,42 @@ test("LeakSanitizer runs over the sanitized corpus on Linux only", () => { for (const task of plan("san", "linux").side) expect(task.env.ASAN_OPTIONS).toBeUndefined(); }); +test("the sanitized three.js nodes build leaves the Linux corpus for its own job", () => { + const plan = (flavor: "plain" | "san", platform: string) => + ciTestPlan({ shard: "1/20", flavor, workers: 4, platform }); + const [name, value] = Object.entries(sanitizedNodesJobEnv)[0]!; + expect(plan("san", "linux").corpus.env[name]).toBe(value); + for (const env of [plan("plain", "linux").corpus.env, plan("san", "darwin").corpus.env]) + expect(env[name]).toBeUndefined(); + const three = readFileSync(new URL("./three.test.ts", import.meta.url), "utf8"); + expect(three).toContain(`process.env["${name}"] === "${value}"`); + const { jobs } = load( + readFileSync(new URL("../../.github/workflows/ci.yml", import.meta.url), "utf8"), + ) as { + jobs: Record< + string, + { needs: string[]; env: Record; steps: { run?: string }[] } + >; + }; + const job = jobs["three_nodes_san"]!; + expect(job.needs).toContain("native_linux"); + expect(jobs["test"]!.needs).toContain("three_nodes_san"); + expect(job.env[name]).toBeUndefined(); + expect(job.env["SCRIPTC_SAN"]).toBe("1"); + // The same LeakSanitizer settings as the corpus it left. + const corpus = plan("san", "linux").corpus.env; + expect(job.env["ASAN_OPTIONS"]).toBe(corpus.ASAN_OPTIONS); + expect(job.env["LSAN_OPTIONS"]).toBe( + corpus.LSAN_OPTIONS!.replace( + leakSuppressions, + "${{ github.workspace }}/tests/harness/lsan.supp", + ), + ); + expect(job.steps.at(-1)!.run).toBe( + `pnpm test tests/harness/three.test.ts -t "'nodes' runs statically through LLVM on 'native'"`, + ); +}); + test("the side queue balances measured files across the twenty Linux shards", () => { for (const flavor of ["plain", "san"] as const) { const costs = testCosts[flavor] as Record; diff --git a/tests/harness/three.test.ts b/tests/harness/three.test.ts index 27fb86d1f..e55242149 100644 --- a/tests/harness/three.test.ts +++ b/tests/harness/three.test.ts @@ -68,6 +68,15 @@ test.for(cases)( async ({ fixture, target }, context) => { const entry = join(import.meta.dirname, `../fixtures/three/${fixture}.mjs`); if (target === "wasm32-wasi" && !hasZig) context.skip(); + // CI's Linux sanitized lane builds this one in its own job + // (scripts/ci-test-plan.mjs). + if ( + fixture === "nodes" && + target === "native" && + process.env["SCRIPTC_SAN"] === "1" && + process.env["SCRIPTC_THREE_SANITIZED_NODES_JOB"] === "1" + ) + context.skip(); const dir = await mkdtemp("/tmp/scriptc-three-"); const previousTarget = process.env["SCRIPTC_TARGET"]; const previousCc = process.env["SCRIPTC_CC"]; From 541b05ddea63bf8ddde6b4828769ba4e46886717 Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Sun, 25 Oct 2026 09:22:06 -0700 Subject: [PATCH 12/44] Map generic alias casts inside generic bodies with the instance's arguments A cast to a generic type alias spelled over the enclosing function's own type parameters (`return x as Catalog`) stays symbolic in the checker inside the body, so a mapped type with key remapping, a template key filter or a conditional member refused with SC2001 in every instance. The checker interns alias instantiations, so the body's `Catalog` is the same type the signature declares. Binding unification now records, per call-keyed instance, each declared generic alias reference together with the alias the call's resolved signature instantiates. When the symbolic form does not map, mapType maps that concrete instantiation instead. A body forwarding its own alias to another generic resolves through the enclosing instance. Types that already mapped are unaffected. Co-Authored-By: Claude Opus 5.5 --- .../frontend/lowering/generic-functions.ts | 28 +++++ .../compiler/src/frontend/lowering/lowerer.ts | 10 ++ packages/compiler/src/frontend/type-mapper.ts | 15 +++ .../generic-alias-cast-instances.ts.json | 6 + tests/corpus/generic-alias-cast-instances.ts | 103 ++++++++++++++++++ .../generic-alias-cast-instances.ts.json | 1 + 6 files changed, 163 insertions(+) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/generic-alias-cast-instances.ts.json create mode 100644 tests/corpus/generic-alias-cast-instances.ts create mode 100644 tests/harness/corpus-costs/generic-alias-cast-instances.ts.json diff --git a/packages/compiler/src/frontend/lowering/generic-functions.ts b/packages/compiler/src/frontend/lowering/generic-functions.ts index 0f9e41723..7d7124027 100644 --- a/packages/compiler/src/frontend/lowering/generic-functions.ts +++ b/packages/compiler/src/frontend/lowering/generic-functions.ts @@ -921,6 +921,27 @@ function bindDefaultTypeParams( }); } +/** Generic alias references of a call-keyed instance's declared signature + * (`Catalog`), each paired with the same alias as the call's resolved + * signature instantiates it (`Catalog<{ g1: … }, "acme">`). The checker + * keeps the alias symbolic inside the generic body and offers no way to + * instantiate it with the bound arguments, but alias instantiations are + * interned: the body's `x as Catalog` is the very type the signature + * declares, so the pair recorded during unification is that re-instantiation. + * Keyed by the instance's ts-level bindings map, which every scope that + * lowers the instance installs (typeParamTsBindings). */ +const boundAliasInstances = new WeakMap, Map>(); + +/** The concrete instantiation of a generic alias reference inside the + * instance whose ts-level bindings are `tsBindings` (boundAliasInstances), + * or null when its signature never instantiated that reference. */ +export function boundAliasInstance( + tsBindings: Map, + type: ts.Type, +): ts.Type | null { + return boundAliasInstances.get(tsBindings)?.get(type) ?? null; +} + /** The structural half of binding inference: unify the DECLARED signature * (whose types mention the type parameters) against a TARGET signature * with the substitution applied — a call's resolved signature, or the @@ -1040,6 +1061,13 @@ function unifySignatureBindings( dAliasArgs.length && iAliasArgs.length === dAliasArgs.length ) { + if (tsBindings && declared !== inst) { + let pairs = boundAliasInstances.get(tsBindings); + if (!pairs) boundAliasInstances.set(tsBindings, (pairs = new Map())); + // A body forwarding its own alias (`inner(g)` returning `Catalog` inside an instance) resolves through the enclosing one. + if (!pairs.has(declared)) pairs.set(declared, lowerer.boundAliasTsResolver(inst) ?? inst); + } dAliasArgs.forEach((da, i) => { const ia = iAliasArgs[i]; if (ia) unify(da, ia, depth + 1); diff --git a/packages/compiler/src/frontend/lowering/lowerer.ts b/packages/compiler/src/frontend/lowering/lowerer.ts index ee7477e43..52f3567fb 100644 --- a/packages/compiler/src/frontend/lowering/lowerer.ts +++ b/packages/compiler/src/frontend/lowering/lowerer.ts @@ -456,6 +456,7 @@ import { inferTypeParamBindings, lowerGenericInstance, genericLocalClassBinding, + boundAliasInstance, } from "./generic-functions.js"; import { bindingNeverReassigned } from "./binding-analysis.js"; import type { InstantiationPath } from "./instantiation-path.js"; @@ -3792,6 +3793,7 @@ export class Lowerer { classNamer: (decl) => this.classNamer(decl), resolveTypeParam: this.typeParamResolver, resolveTypeParamTs: this.typeParamTsResolver, + resolveBoundAliasTs: this.boundAliasTsResolver, isErasing: () => this.erasingDepth > 0, genericClassInstance: (decl, ref, mapArgument) => this.genericClassInstanceType(decl, ref, mapArgument), @@ -9550,6 +9552,14 @@ export class Lowerer { return (sym && this.typeParamTsBindings.get(sym)) ?? null; }; + /** The concrete instantiation of a generic alias reference (`Catalog`) the current call-keyed instance's signature declares + * (boundAliasInstance). Inert outside generic function instantiation. */ + readonly boundAliasTsResolver = (t: ts.Type): ts.Type | null => { + if (!this.typeParamTsBindings || t.getAliasTypeArguments().length === 0) return null; + return boundAliasInstance(this.typeParamTsBindings, t); + }; + irTypeOf(node: ts.Node): IrType { const t = this.typeOf(node); // JS inference can leave mutable fields, accessor parameters and diff --git a/packages/compiler/src/frontend/type-mapper.ts b/packages/compiler/src/frontend/type-mapper.ts index 17f9d0d11..5fa361db2 100644 --- a/packages/compiler/src/frontend/type-mapper.ts +++ b/packages/compiler/src/frontend/type-mapper.ts @@ -1266,6 +1266,11 @@ export interface TypeMapperCtx { * only where literal identity matters — indexed accesses (`T[K]`) inside * generic bodies whose K is bound to a literal key. */ resolveTypeParamTs?: TypeParamTsResolver; + /** A generic alias reference inside a generic body (`Catalog`) + * re-instantiated with the instance's bound type arguments: the concrete + * checker type its signature instantiates, or null. Consulted only when + * the symbolic form does not map. */ + resolveBoundAliasTs?: TypeParamTsResolver; /** GENERIC program classes (monomorphization by flow): the mapping of a * concrete instantiation reference (`Box`) — the Lowerer * registers/reuses the instantiation (`Box%0`) and answers its object @@ -1830,6 +1835,16 @@ export function mapType(type: ts.Type, ctx: TypeMapperCtx): IrType | null { if (!topLevel || !ts.isUnreadableConstituentsPanic(e)) throw e; mapped = null; } + // A generic alias the body spells over its own type parameters (`x as + // Catalog`) stays symbolic in the checker; when that form does not + // map, the instance's concrete instantiation of the same alias does. + if (mapped === null && ctx.resolveBoundAliasTs) { + const concrete = ctx.resolveBoundAliasTs(type); + if (concrete !== null && concrete !== type) { + mapped = mapType(concrete, ctx); + if (mapped !== null) contextResolutions++; + } + } if ( mapped !== null && memoKey !== null && diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/generic-alias-cast-instances.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/generic-alias-cast-instances.ts.json new file mode 100644 index 000000000..a229ee862 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/generic-alias-cast-instances.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/generic-alias-cast-instances.ts" + ], + "diags": [] +} diff --git a/tests/corpus/generic-alias-cast-instances.ts b/tests/corpus/generic-alias-cast-instances.ts new file mode 100644 index 000000000..11a098214 --- /dev/null +++ b/tests/corpus/generic-alias-cast-instances.ts @@ -0,0 +1,103 @@ +// Casts to a generic type alias spelled over the enclosing function's own +// type parameters (`x as Catalog`). The checker keeps such an alias +// symbolic inside the generic body; each instance maps it with its own bound +// type arguments. + +type Groups = Record>; + +type EntryKey = { [A in keyof G]: keyof G[A] }[keyof G] & string; +type KeyOfKind = Extract, `${K}:${string}`>; +type EntryId = K extends `${string}:${infer I}` ? I : never; + +type Catalog = { + [Key in KeyOfKind as EntryId]: { id: EntryId; label: string; owner: P }; +}; + +function flatten(groups: Groups, kind: string): Record { + return Object.fromEntries( + Object.values(groups) + .flatMap((entries) => Object.entries(entries)) + .filter(([key]) => key.startsWith(kind + ":")) + .map(([, entry]) => [(entry as { id: string }).id, entry]), + ); +} + +function toolCatalog( + _owner: P, + groups: G, +): Catalog { + return flatten(groups, "tool") as Catalog; +} + +function partCatalog( + _owner: P, + groups: G, +): Catalog { + return flatten(groups, "part") as Catalog; +} + +const workshop = { + bench: { + "tool:saw": { id: "saw", label: "Hand saw", owner: "acme" }, + "part:bolt": { id: "bolt", label: "Hex bolt", owner: "acme" }, + }, + shelf: { + "tool:drill": { id: "drill", label: "Cordless drill", owner: "acme" }, + }, +}; + +const tools = toolCatalog("acme", workshop); +console.log(tools.saw.label, tools.drill.id, tools.drill.owner); +console.log(Object.keys(tools).join(","), JSON.stringify(tools.saw)); + +const parts = partCatalog("acme", workshop); +console.log(parts.bolt.label, Object.keys(parts).length); + +// A second instantiation with different groups gets its own layout. +const garden = { + shed: { + "tool:rake": { id: "rake", label: "Leaf rake", owner: "green" }, + "tool:hoe": { id: "hoe", label: "Garden hoe", owner: "green" }, + }, +}; +const gardenTools = toolCatalog("green", garden); +console.log(gardenTools.rake.label, gardenTools.hoe.label, Object.keys(gardenTools).join(",")); + +// No entry of the kind: the alias resolves to the empty record. +const gardenParts = partCatalog("green", garden); +console.log(Object.keys(gardenParts).length, JSON.stringify(gardenParts)); + +// A homomorphic mapped alias over the bound type, cast inside the body. +type Boxed = { [K in keyof T]: { value: T[K] } }; + +function boxAll>(values: T): Boxed { + const out: Record = {}; + for (const key of Object.keys(values)) out[key] = { value: values[key] }; + return out as Boxed; +} + +const boxed = boxAll({ width: 3, height: 4 }); +console.log(boxed.width.value * boxed.height.value); + +// The alias nested in an array type of the signature. +function boxEach>(rows: T[]): Boxed[] { + return rows.map((row) => { + const out: Record = {}; + for (const key of Object.keys(row)) out[key] = { value: row[key] }; + return out as Boxed; + }); +} + +const rows = boxEach([ + { x: 1, y: 2 }, + { x: 5, y: 8 }, +]); +console.log(rows.map((r) => r.x.value + r.y.value).join(" ")); + +// A generic body forwarding its parameters to another generic cast. +function ownedTools(groups: G): Catalog { + return toolCatalog("shared", groups); +} + +const shared = ownedTools({ rack: { "tool:clamp": { id: "clamp", label: "C-clamp", owner: "shared" } } }); +console.log(shared.clamp.label, shared.clamp.owner); diff --git a/tests/harness/corpus-costs/generic-alias-cast-instances.ts.json b/tests/harness/corpus-costs/generic-alias-cast-instances.ts.json new file mode 100644 index 000000000..8c0e66c3c --- /dev/null +++ b/tests/harness/corpus-costs/generic-alias-cast-instances.ts.json @@ -0,0 +1 @@ +{ "plain": 3000, "san": 3600 } From 30e6ca9160d33cb24933fc392e86c36358dfc681 Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Sun, 25 Oct 2026 10:07:51 -0700 Subject: [PATCH 13/44] Convert promise and function values into a union's arm of their kind A promise or function value flowing into a union whose only arm of that kind has a wider payload or result (`Promise` into `Promise | undefined`, `(s) => Promise` into an optional `(s) => Promise` member) was refused with SC2003: an arm value wrapped into a union only when the union held an identical arm. It now takes the arm-level conversion (the checked payload widening, the function adapter) and wraps the result. `p === u` against such a union compares both sides as untyped values, so a promise keeps its identity although converting it into the union's arm builds a new promise. Co-Authored-By: Claude Opus 5.5 --- .../frontend/lowering/expressions/binary.ts | 25 +++++++ .../compiler/src/frontend/lowering/lowerer.ts | 25 +++++++ ...nion-promise-function-arm-widening.ts.json | 6 ++ .../union-promise-function-arm-widening.ts | 68 +++++++++++++++++++ ...nion-promise-function-arm-widening.ts.json | 1 + 5 files changed, 125 insertions(+) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/union-promise-function-arm-widening.ts.json create mode 100644 tests/corpus/union-promise-function-arm-widening.ts create mode 100644 tests/harness/corpus-costs/union-promise-function-arm-widening.ts.json diff --git a/packages/compiler/src/frontend/lowering/expressions/binary.ts b/packages/compiler/src/frontend/lowering/expressions/binary.ts index c172fc662..01b48554e 100644 --- a/packages/compiler/src/frontend/lowering/expressions/binary.ts +++ b/packages/compiler/src/frontend/lowering/expressions/binary.ts @@ -2388,6 +2388,31 @@ export function lowerStrictEquality( loc, }; } + // A plain promise against a union whose promise arm has a different + // payload (`chains.get(id) === tail` with a `Map>`): + // wrapping would convert the promise into that arm, and the converted + // promise is a new object. Promises are objects to ===, so compare both + // sides as untyped values, where a converted promise answers with its + // origin's identity (the plain promise-vs-promise rule below). + const plainPromise = + left.type.kind === "promise" ? left : right.type.kind === "promise" ? right : null; + if ( + !bothUnion && + plainPromise !== null && + lowerer.armTag(ut.unionId, plainPromise.type) < 0 && + unionArms.some((arm) => arm.kind === "promise") && + lowerer.dynConvertible(left.type) && + lowerer.dynConvertible(right.type) + ) { + return { + kind: "dynScalarEq", + left: lowerer.coerceToExpected(left, DYN), + right: lowerer.coerceToExpected(right, DYN), + ...(negated ? { negated: true as const } : {}), + type: BOOL, + loc, + }; + } if ((sameUnion || !bothUnion) && lowerer.eqComparableUnion(ut.unionId)) { aliasRefusal(ut); return { diff --git a/packages/compiler/src/frontend/lowering/lowerer.ts b/packages/compiler/src/frontend/lowering/lowerer.ts index ee7477e43..059fab488 100644 --- a/packages/compiler/src/frontend/lowering/lowerer.ts +++ b/packages/compiler/src/frontend/lowering/lowerer.ts @@ -10397,6 +10397,31 @@ export class Lowerer { } } } + // A promise or function value against a union whose ONLY arm of + // that kind differs from it (`Promise` into `Promise | + // undefined`, `() => Promise` into an optional + // `() => Promise` member): convert into + // that arm with the arm-level conversion (the checked payload + // widening, the function adapter), then wrap. Several arms of the + // kind stay ambiguous and fall through to the union fence. + if (expr.type.kind === "promise" || expr.type.kind === "func") { + const arms = this.unions.get(expected.unionId)?.arms ?? []; + const candidates = arms.filter((arm) => arm.kind === expr.type.kind); + const arm = candidates.length === 1 ? candidates[0]! : null; + if (arm) { + const converted = this.coerceToExpected(expr, arm); + if (converted !== expr && typeEquals(converted.type, arm)) { + return { + kind: "unionWrap", + unionId: expected.unionId, + tag: arms.indexOf(arm), + value: converted, + type: expected, + loc: expr.loc, + }; + } + } + } // A width-coercible value against a union: coerce into the SINGLE // width-liftable arm, then wrap like any arm value (widthLiftPlan's // liftWrap — several candidate arms are ambiguous and decline). diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/union-promise-function-arm-widening.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/union-promise-function-arm-widening.ts.json new file mode 100644 index 000000000..91c268992 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/union-promise-function-arm-widening.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/union-promise-function-arm-widening.ts" + ], + "diags": [] +} diff --git a/tests/corpus/union-promise-function-arm-widening.ts b/tests/corpus/union-promise-function-arm-widening.ts new file mode 100644 index 000000000..751af6f41 --- /dev/null +++ b/tests/corpus/union-promise-function-arm-widening.ts @@ -0,0 +1,68 @@ +// A promise or function value flowing into a union whose only arm of that +// kind has a wider payload or result: the value converts into that arm. +// Promises keep their identity under === and !== against the union. + +const chains = new Map>(); + +async function serialize(key: string, task: () => Promise): Promise { + const previous = chains.get(key) ?? Promise.resolve(); + const queued = (async () => { + await previous.catch(() => {}); + return task(); + })(); + const tail = queued.catch(() => {}); + chains.set(key, tail); + void tail.then(() => { + if (chains.get(key) === tail) chains.delete(key); + console.log(`settled ${key}: ${chains.get(key) !== tail}`); + }); + return queued; +} + +interface Store { + pending?: Promise; + lookup?: (name: string) => Promise; + scale?: (n: number) => number | string; +} + +async function findName(name: string): Promise { + return name.length > 3 ? name.toUpperCase() : null; +} + +function double(n: number): number { + return n * 2; +} + +async function failing(): Promise { + throw new Error("task failed"); +} + +async function main(): Promise { + const a = await serialize("alpha", async () => 1); + const b = await serialize("alpha", async () => "two"); + const c = await serialize("beta", async (): Promise => true); + console.log(a, b, c, chains.size); + try { + await serialize("alpha", failing); + } catch (e) { + console.log("caught", (e as Error).message); + } + for (let i = 0; i < 4; i++) await Promise.resolve(); + console.log("remaining", chains.size); + + const store: Store = { lookup: findName, scale: double }; + const work: Promise = Promise.resolve(41); + store.pending = work; + console.log(store.pending === work, store.pending !== work); + const other: Promise = Promise.resolve(41); + console.log(store.pending === other); + console.log(await store.pending); + + const lookup = store.lookup; + if (lookup) console.log(await lookup("abcd"), await lookup("ab")); + console.log(store.scale?.(21)); + const empty: Store = {}; + console.log(empty.pending === work, empty.lookup === undefined, empty.scale?.(1)); +} + +main(); diff --git a/tests/harness/corpus-costs/union-promise-function-arm-widening.ts.json b/tests/harness/corpus-costs/union-promise-function-arm-widening.ts.json new file mode 100644 index 000000000..5c3da345d --- /dev/null +++ b/tests/harness/corpus-costs/union-promise-function-arm-widening.ts.json @@ -0,0 +1 @@ +{ "plain": 14002, "san": 16102 } From 9cdbc83aee7e85428c44bff096663e16b94fd6b4 Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Sun, 25 Oct 2026 10:07:59 -0700 Subject: [PATCH 14/44] Prefer the union arm that keeps every field of a converted record A record flowing into a union where several record arms could hold it by a width conversion was refused with SC2003 as ambiguous, even when one arm keeps every field of the record and the others hold it only by dropping fields (a model record whose option union is narrower than the catalog's, beside a classifier arm with a subset of its fields). That arm is the value's own layout, widened: it now wins when it is the only one. Plans that already resolved (a single candidate, the absent-field preference, discriminant and presence routes) are unchanged. Co-Authored-By: Claude Opus 5.5 --- .../lowering/coercions/structural-plans.ts | 19 +++++++ .../corpus/union-record-lossless-arm.ts.json | 6 ++ tests/corpus/union-record-lossless-arm.ts | 55 +++++++++++++++++++ .../union-record-lossless-arm.ts.json | 1 + 4 files changed, 81 insertions(+) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/union-record-lossless-arm.ts.json create mode 100644 tests/corpus/union-record-lossless-arm.ts create mode 100644 tests/harness/corpus-costs/union-record-lossless-arm.ts.json diff --git a/packages/compiler/src/frontend/lowering/coercions/structural-plans.ts b/packages/compiler/src/frontend/lowering/coercions/structural-plans.ts index f42663a13..bff419d48 100644 --- a/packages/compiler/src/frontend/lowering/coercions/structural-plans.ts +++ b/packages/compiler/src/frontend/lowering/coercions/structural-plans.ts @@ -128,6 +128,25 @@ export function widthLiftPlan(lowerer: Lowerer, src: IrType, dst: IrType): Width const route = presenceRoute(lowerer, src.shapeId, selected[0]!, selected[1]!); if (route) return route; } + // A record whose fields some arm keeps EVERY one of, each lifting + // (`{ id; limit; opts?: A | B }` into `{ id; limit; opts?: A | B | C } | + // { id; opts?: C }`): that arm is the value's own layout, widened; the + // others hold it only by dropping fields. Prefer the unique lossless arm. + if (selected.length > 1 && src.kind === "record") { + const source = lowerer.shapes.get(src.shapeId); + const lossless = selected.filter(({ arm }) => { + const target = arm.kind === "record" ? lowerer.shapes.get(arm.shapeId) : undefined; + return ( + source !== undefined && + target !== undefined && + !source.tuple && + !target.tuple && + target.fields.length === source.fields.length && + source.fields.every((field) => target.fields.some((kept) => kept.name === field.name)) + ); + }); + if (lossless.length === 1) selected = lossless; + } if (selected.length !== 1) return null; return { how: "liftWrap", tag: selected[0]!.tag, arm: selected[0]!.arm }; } diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/union-record-lossless-arm.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/union-record-lossless-arm.ts.json new file mode 100644 index 000000000..143cda7d0 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/union-record-lossless-arm.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/union-record-lossless-arm.ts" + ], + "diags": [] +} diff --git a/tests/corpus/union-record-lossless-arm.ts b/tests/corpus/union-record-lossless-arm.ts new file mode 100644 index 000000000..b90d5486e --- /dev/null +++ b/tests/corpus/union-record-lossless-arm.ts @@ -0,0 +1,55 @@ +// A record flowing into a union where several record arms can hold it by a +// width conversion: the arm that keeps every one of its fields is the +// value's own layout, widened, and wins over arms that drop fields. + +interface FastOpts { + fast?: boolean; + strict?: boolean; +} +interface DepthOpts { + depth?: number; +} +interface StrictOpts { + strict?: boolean; +} + +interface Wide { + id: string; + window: number; + kind?: string; + limit: number; + opts?: FastOpts | DepthOpts | StrictOpts; +} + +interface Judge { + id: string; + window: number; + kind: string; + opts?: StrictOpts; +} + +interface Narrow { + id: string; + window: number; + kind?: string; + limit: number; + opts?: FastOpts | DepthOpts; +} + +function show(entry: Wide | Judge): string { + const limit = "limit" in entry ? ` limit=${entry.limit}` : ""; + return `${entry.kind ?? "wide"} ${entry.id} window=${entry.window}${limit} ${JSON.stringify(entry.opts ?? null)}`; +} + +const narrow: Narrow = { id: "n1", window: 4, limit: 2, opts: { depth: 3 } }; +console.log(show(narrow)); + +const entries: (Wide | Judge)[] = [narrow, { id: "j1", window: 1, kind: "judge", opts: { strict: true } }]; +const plain: Narrow = { id: "n2", window: 8, kind: "custom", limit: 5 }; +entries.push(plain); +for (const entry of entries) console.log(show(entry)); + +let latest: Wide | Judge | undefined; +for (const item of [narrow, plain]) latest = item; +console.log(latest === undefined ? "none" : show(latest)); + diff --git a/tests/harness/corpus-costs/union-record-lossless-arm.ts.json b/tests/harness/corpus-costs/union-record-lossless-arm.ts.json new file mode 100644 index 000000000..011a8fbbf --- /dev/null +++ b/tests/harness/corpus-costs/union-record-lossless-arm.ts.json @@ -0,0 +1 @@ +{ "plain": 11809, "san": 13580 } From ed50daa853aedeea1d644df779db93ada8479e80 Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Sun, 25 Oct 2026 11:01:57 -0700 Subject: [PATCH 15/44] Keep the whole union of an optional parameter that reaches a recursive placeholder An optional parameter typed by a union that is still being mapped gets a recursive placeholder: a union id whose arms arrive when the outer frame finishes. The function-type mapping then armed that placeholder with undefined, but the placeholder's arm list was still empty, so the result was a fresh union holding only `undefined`. The parameter lost its record arm, the same checker type mapped to two different layouts depending on where the mapping started, and later conversions failed (SC2001, SC2002, SC2003). This happens when an interface method takes an options object whose callbacks are typed by a subinterface of that same interface: `newSession(options?: { withSession?: (ctx: Replaced) => Promise })` with `interface Replaced extends CommandContext`. When the checker spelled the parameter with an undefined constituent, the placeholder already carries that arm, so the parameter keeps the placeholder. withUnitArm now refuses a pending placeholder instead of building the unit-only union. pi's ExtensionCommandContext is such an interface. It refused ExtensionAPI.registerCommand, and with it pi's main(). Co-Authored-By: Claude Opus 5.5 --- .../compiler/src/frontend/type-mapper.test.ts | 14 +++ packages/compiler/src/frontend/type-mapper.ts | 24 +++- ...recursive-optional-callback-params.ts.json | 6 + .../recursive-optional-callback-params.ts | 107 ++++++++++++++++++ ...recursive-optional-callback-params.ts.json | 1 + 5 files changed, 150 insertions(+), 2 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/recursive-optional-callback-params.ts.json create mode 100644 tests/corpus/recursive-optional-callback-params.ts create mode 100644 tests/harness/corpus-costs/recursive-optional-callback-params.ts.json diff --git a/packages/compiler/src/frontend/type-mapper.test.ts b/packages/compiler/src/frontend/type-mapper.test.ts index 163ba7b9e..6e7e339d6 100644 --- a/packages/compiler/src/frontend/type-mapper.test.ts +++ b/packages/compiler/src/frontend/type-mapper.test.ts @@ -53,6 +53,20 @@ describe("union arm lookup", () => { }); }); +describe("recursive union placeholders", () => { + test("refuse a unit arm until their arms are known", () => { + const unions = new UnionRegistry(); + const recursive = {} as Type; + const union = unions.recursiveRef(recursive, () => "Recursive"); + const placeholder: IrType = { kind: "union", unionId: union }; + expect(unions.isPendingPlaceholder(union)).toBe(true); + expect(withUndefinedArm(placeholder, unions)).toBeNull(); + unions.finalizeRecursive(recursive, [{ kind: "record", shapeId: "r1" }, UNDEFINED_T]); + expect(unions.isPendingPlaceholder(union)).toBe(false); + expect(withUndefinedArm(placeholder, unions)).toEqual(placeholder); + }); +}); + describe("nullable collection union builders", () => { test.each([mapOf(STRING, F64), setOf(STRING)])( "optional %j fields and generator results share the same union", diff --git a/packages/compiler/src/frontend/type-mapper.ts b/packages/compiler/src/frontend/type-mapper.ts index 17f9d0d11..63eed378d 100644 --- a/packages/compiler/src/frontend/type-mapper.ts +++ b/packages/compiler/src/frontend/type-mapper.ts @@ -717,6 +717,12 @@ export class UnionRegistry { return id !== undefined && this.pendingRec.has(id); } + /** True when `unionId` is a recursive placeholder whose arms are not + * known yet (its outer frame is still mapping, or failed). */ + isPendingPlaceholder(unionId: string): boolean { + return this.pendingRec.has(unionId); + } + /** Completes a recursive placeholder with its canonical arm list and * registers the structural key (first writer wins, like shapes). */ finalizeRecursive(t: ts.Type, arms: IrType[], discriminant?: IrUnionDiscriminant): string { @@ -5302,8 +5308,19 @@ function mapCallSignature(sig: ts.Signature, ctx: TypeMapperCtx): IrType | null if (pt) pt = jsOpenObjectType(decl, pt, ctx.shapes, ctx.unions); // Belt and braces for non-strict type worlds: an optional param's // ABI slot is always the undefined-armed union (strictNullChecks - // already spells it that way; arm it here if the world didn't). - if (optional && pt && pt.kind !== "void" && pt.kind !== "dyn") { + // already spells it that way; arm it here if the world didn't). A + // back-reference to a union still being mapped (`options?: { then?: + // (ctx: Sub) => void }` where Sub extends the interface declaring the + // method) is a recursive placeholder with no arms yet; when the checker + // spelled the parameter `X | undefined`, the placeholder receives that + // undefined arm when its frame finishes. + const pendingOptional = + pt?.kind === "union" && + ctx.unions.isPendingPlaceholder(pt.unionId) && + ts + .constituentTypes(checker.getTypeOfSymbol(p)) + .some((part) => (part.flags & ts.TypeFlags.Undefined) !== 0); + if (optional && pt && pt.kind !== "void" && pt.kind !== "dyn" && !pendingOptional) { const armed = withUndefinedArm(pt, ctx.unions); if (!armed) return null; pt = armed; @@ -6782,6 +6799,9 @@ export function withUnitArm( if (t.kind === "union") { const def = unions.get(t.unionId); if (!def) return null; + // A recursive placeholder's arms arrive when its outer frame finishes; + // extending its (still empty) arm list would drop every arm but the unit. + if (unions.isPendingPlaceholder(t.unionId)) return null; if (def.arms.some((a) => a.kind === kind)) return t; const arms = [...def.arms, unit]; // Date and Promise arms beside other data arms have no union home. diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/recursive-optional-callback-params.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/recursive-optional-callback-params.ts.json new file mode 100644 index 000000000..abf783f85 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/recursive-optional-callback-params.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/recursive-optional-callback-params.ts" + ], + "diags": [] +} diff --git a/tests/corpus/recursive-optional-callback-params.ts b/tests/corpus/recursive-optional-callback-params.ts new file mode 100644 index 000000000..0c81235d6 --- /dev/null +++ b/tests/corpus/recursive-optional-callback-params.ts @@ -0,0 +1,107 @@ +// An optional parameter whose type reaches back to the interface that +// declares the method keeps its whole union. A command context's +// `newSession(options?)` takes callbacks typed by a subinterface of the +// context, so mapping the context passes through the options union while +// that union is still being mapped; the parameter still holds the options +// record, not only `undefined`. + +interface CommandContext { + cwd: string; + newSession(options?: { + parent?: string; + withSession?: (ctx: ReplacedContext) => Promise; + }): Promise<{ cancelled: boolean }>; + fork( + entryId: string, + options?: { position?: "before" | "at"; withSession?: (ctx: ReplacedContext) => Promise }, + ): Promise<{ cancelled: boolean }>; +} + +interface ReplacedContext extends CommandContext { + depth: number; + send(content: string): Promise; +} + +interface Command { + description?: string; + handler: (args: string, ctx: CommandContext) => Promise; +} + +const log: string[] = []; +const commands = new Map(); + +function register(name: string, command: Command): void { + commands.set(name, command); +} + +register("new", { + description: "start a session", + handler: async (args, ctx) => { + const first = await ctx.newSession({ + parent: args, + withSession: async (next) => { + await next.send(`hello from ${next.cwd} at depth ${next.depth}`); + const nested = await next.newSession({ + withSession: async (inner) => { + await inner.send(`nested at depth ${inner.depth}`); + }, + }); + log.push(`nested cancelled=${nested.cancelled}`); + }, + }); + log.push(`first cancelled=${first.cancelled}`); + const bare = await ctx.newSession(); + log.push(`bare cancelled=${bare.cancelled}`); + }, +}); + +register("fork", { + handler: async (args, ctx) => { + const r = await ctx.fork(args, { + position: "at", + withSession: async (next) => next.send(`forked ${args} at depth ${next.depth}`), + }); + log.push(`fork cancelled=${r.cancelled}`); + const plain = await ctx.fork(args); + log.push(`plain fork cancelled=${plain.cancelled}`); + }, +}); + +function makeContext(depth: number): ReplacedContext { + return { + cwd: `/work/${depth}`, + depth, + newSession: async (options) => { + log.push(`newSession depth=${depth} parent=${options?.parent ?? "-"}`); + if (options?.withSession) { + await options.withSession(makeContext(depth + 1)); + return { cancelled: false }; + } + return { cancelled: true }; + }, + fork: async (entryId, options) => { + log.push(`fork ${entryId} position=${options?.position ?? "-"}`); + const callback = options?.withSession; + if (callback === undefined) return { cancelled: true }; + await callback(makeContext(depth + 1)); + return { cancelled: false }; + }, + send: async (content) => { + log.push(`send: ${content}`); + }, + }; +} + +async function run(): Promise { + const root: CommandContext = makeContext(0); + for (const [name, command] of commands) { + log.push(`run ${name} (${command.description ?? "no description"})`); + await command.handler(`entry-${name}`, root); + } + for (const line of log) console.log(line); +} + +run().catch((e: unknown) => { + console.log("failed", e instanceof Error ? e.message : String(e)); + process.exitCode = 1; +}); diff --git a/tests/harness/corpus-costs/recursive-optional-callback-params.ts.json b/tests/harness/corpus-costs/recursive-optional-callback-params.ts.json new file mode 100644 index 000000000..c33a4b32c --- /dev/null +++ b/tests/harness/corpus-costs/recursive-optional-callback-params.ts.json @@ -0,0 +1 @@ +{ "plain": 6000 } From bd1bf39a765d7b8993f25862b40225388fc5e0f8 Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Sun, 25 Oct 2026 11:18:18 -0700 Subject: [PATCH 16/44] Close recursive record knots through erased generic method members A record or union that refers back to itself only closes its recursive knot when the frame did not resolve anything through context-sensitive hooks; otherwise the same checker type could answer differently per instantiation, so the knot is refused. Erasing a generic method member (`send(message: Message): string`) resolves the method's own type parameters to `unknown`, and that resolution counted as context-sensitive. Any recursive interface declaring a generic method was therefore refused: `interface Sub extends Base { send(m: Message): string }` reached from `Base { child?: Sub }`, or a list node with a generic method beside its `next` field. The member erasure now records its own type parameters, and resolving one of them no longer counts. Consulting an enclosing binding still does, so a generic method that mentions an outer type parameter keeps the fence. The erased member's mapping is not memoized under the member's type, since the same type maps to nothing outside the erasure. pi's ReplacedSessionContext (sendMessage) extends ExtensionCommandContext, whose methods take callbacks over ReplacedSessionContext. The fence refused ExtensionAPI.registerCommand and, through it, pi's main(). Co-Authored-By: Claude Opus 5.5 --- packages/compiler/src/frontend/type-mapper.ts | 22 ++++++- .../recursive-generic-method-members.ts.json | 6 ++ .../recursive-generic-method-members.ts | 58 +++++++++++++++++++ .../recursive-generic-method-members.ts.json | 1 + 4 files changed, 84 insertions(+), 3 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/recursive-generic-method-members.ts.json create mode 100644 tests/corpus/recursive-generic-method-members.ts create mode 100644 tests/harness/corpus-costs/recursive-generic-method-members.ts.json diff --git a/packages/compiler/src/frontend/type-mapper.ts b/packages/compiler/src/frontend/type-mapper.ts index 17f9d0d11..99766d0ab 100644 --- a/packages/compiler/src/frontend/type-mapper.ts +++ b/packages/compiler/src/frontend/type-mapper.ts @@ -1255,6 +1255,10 @@ export interface TypeMapperCtx { * (erasedGenericMemberType): the signature's own type parameters * resolve to `unknown` instead of refusing the mapping. */ erasingSignature?: ts.Signature; + /** Set with erasingSignature: the signature's own type parameters, and + * whether the erasure consulted an enclosing binding. An erasure that + * resolved only its own parameters answers the same in every context. */ + erasedMember?: { own: ReadonlySet; outerConsulted: boolean }; /** True while an erased generic body lowers (its type parameters are * bound to `unknown`). */ isErasing?: () => boolean; @@ -1835,6 +1839,7 @@ export function mapType(type: ts.Type, ctx: TypeMapperCtx): IrType | null { memoKey !== null && topLevel && contextResolutions === sensitivityAtEntry && + ctx.erasingSignature === undefined && (ctx.canMemoizeType?.() ?? true) ) { ctx.typeMemo?.set(memoKey, mapped); @@ -1991,7 +1996,7 @@ function mapTypeInner(type: ts.Type, ctx: TypeMapperCtx): IrType | null { !ERASURE_KEPT_CONTAINERS.has(ts.symbolNameOf(type.getSymbol()) ?? "") && mentionsErasedTypeParam(type, ctx, 0) ) { - contextResolutions++; + if (!ctx.erasedMember || ctx.erasedMember.outerConsulted) contextResolutions++; return DYN; } // Readonly over a program class (Object.freeze's result type) is the @@ -2001,7 +2006,7 @@ function mapTypeInner(type: ts.Type, ctx: TypeMapperCtx): IrType | null { if (resolveTypeParam && type.flags & ts.TypeFlags.TypeParameter) { const bound = resolveTypeParam(type); if (bound) { - contextResolutions++; + if (!ctx.erasedMember?.own.has(type)) contextResolutions++; return bound; } } @@ -7309,10 +7314,21 @@ export function erasedGenericMemberType(t: ts.Type, ctx: TypeMapperCtx): IrType const sig = sigs[0]!; const own = new Set(sig.getTypeParameters()); const outer = ctx.resolveTypeParam; + // Resolving only the member's own parameters does not make the erasure + // context-sensitive, so a recursive record holding a generic method + // (`interface Sub extends Base { send(m: Msg): void }` reached + // from Base) still closes its knot. Consulting an enclosing binding does. + const erasedMember = { own, outerConsulted: false }; const erased = mapType(t, { ...ctx, erasingSignature: sig, - resolveTypeParam: (param) => (own.has(param) ? DYN : (outer?.(param) ?? null)), + erasedMember, + resolveTypeParam: (param) => { + if (own.has(param)) return DYN; + if (outer === undefined) return null; + erasedMember.outerConsulted = true; + return outer(param); + }, }); return erased?.kind === "func" ? erased : undefined; } diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/recursive-generic-method-members.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/recursive-generic-method-members.ts.json new file mode 100644 index 000000000..6c3590ab8 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/recursive-generic-method-members.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/recursive-generic-method-members.ts" + ], + "diags": [] +} diff --git a/tests/corpus/recursive-generic-method-members.ts b/tests/corpus/recursive-generic-method-members.ts new file mode 100644 index 000000000..1f4d8372d --- /dev/null +++ b/tests/corpus/recursive-generic-method-members.ts @@ -0,0 +1,58 @@ +// A recursive record that declares a generic method compiles. Erasing the +// method's own type parameters does not depend on any enclosing generic +// binding, so the record still closes its recursive knot: a parent +// interface whose child is a subinterface declaring a generic method, and a +// list node whose generic method sits beside a self-referencing field. + +interface Message { + content: string; + details?: T; +} + +interface TreeNode { + name: string; + child?: SendingNode; +} + +interface SendingNode extends TreeNode { + send(message: Message): string; +} + +interface ListNode { + value: number; + next: ListNode | null; + map(f: (value: number) => U): string; +} + +function describe(node: TreeNode): string { + const inner = node.child; + return inner ? `${node.name} > ${describe(inner)}` : node.name; +} + +function leaf(name: string): SendingNode { + return { name, send: () => `${name} got a message` }; +} + +const tree: SendingNode = { + name: "root", + child: { name: "middle", child: leaf("leaf"), send: () => "middle got a message" }, + send: () => "root got a message", +}; +console.log(describe(tree)); +console.log(tree.send({ content: "hello" })); +const middle = tree.child; +if (middle !== undefined) console.log(middle.send({ content: "hi", details: 42 })); + +function cons(value: number, next: ListNode | null): ListNode { + return { value, next, map: () => `node ${value}` }; +} + +let list: ListNode | null = cons(1, cons(2, cons(3, null))); +const values: number[] = []; +let labels = ""; +while (list !== null) { + values.push(list.value); + labels += `[${list.map((v) => v * 2)}]`; + list = list.next; +} +console.log(values.join(","), labels); diff --git a/tests/harness/corpus-costs/recursive-generic-method-members.ts.json b/tests/harness/corpus-costs/recursive-generic-method-members.ts.json new file mode 100644 index 000000000..1254aa6e4 --- /dev/null +++ b/tests/harness/corpus-costs/recursive-generic-method-members.ts.json @@ -0,0 +1 @@ +{ "plain": 4000 } From 177b2fde496334d47b025951cc6c1393866b2136 Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Sun, 25 Oct 2026 11:51:24 -0700 Subject: [PATCH 17/44] Cross package values at their declared type where the JS literal type lacks members An opted-in package's JavaScript types its own values. The lowering types a consumer's view of such a value from that JavaScript, and the declaration boundary (decl-projection.ts) casts to the declared type only where the JavaScript world answers `any`. A function that returns an object literal has a precise JavaScript type, so it never crossed, even where that literal type lacks members the declarations promise: - `{ [key: string]: string }` over `{ size: "12px" }`: the consumer read `settings["font"] ?? "x"` through the literal's fixed key set. The checker types the miss `any` (a JavaScript literal), but the keyed read took the declared fields' common type, so the miss trapped with "record has no key" and aborted the process. - an expando `out.max = ...` on a non-empty literal, which the JavaScript checker leaves out of the literal's type: reading `stats(xs).max` was refused with SC1090. A package value whose JavaScript type is an object literal that lacks a declared property or string index signature now crosses at its declared type, so missing keys read undefined through the index signature and expando members are read from the object. An index signature whose values do not project (a package interface) keeps the value untyped (`as any`), so its reads are the JavaScript reads. Only programs compiled with --npm-static whose consumers read such values are rewritten. Co-Authored-By: Claude Opus 5.5 --- .../compiler/src/frontend/decl-projection.ts | 62 +++++++++++++++++-- .../npm-static/declared-members-cli.ts | 21 +++++++ .../node_modules/declared-members/index.d.ts | 7 +++ .../node_modules/declared-members/index.js | 34 ++++++++++ .../declared-members/package.json | 6 ++ tests/harness/npm-static.test.ts | 12 ++++ 6 files changed, 138 insertions(+), 4 deletions(-) create mode 100644 tests/fixtures/npm-static/declared-members-cli.ts create mode 100644 tests/fixtures/npm-static/node_modules/declared-members/index.d.ts create mode 100644 tests/fixtures/npm-static/node_modules/declared-members/index.js create mode 100644 tests/fixtures/npm-static/node_modules/declared-members/package.json diff --git a/packages/compiler/src/frontend/decl-projection.ts b/packages/compiler/src/frontend/decl-projection.ts index 02a300996..a999559c2 100644 --- a/packages/compiler/src/frontend/decl-projection.ts +++ b/packages/compiler/src/frontend/decl-projection.ts @@ -519,6 +519,41 @@ export function planDeclProjection(input: DeclProjectionInput): DeclProjection | return undefined; } }; + const stringIndex = (checker: ts.TypeChecker, type: ts.Type): boolean => { + try { + return checker + .getIndexInfosOfType(type) + .some((info) => (info.keyType.flags & ts.TypeFlags.String) !== 0); + } catch { + return false; + } + }; + /** Whether the JavaScript world types a package value as an object + * literal that lacks a member its declaration promises: a property (an + * expando `out.max = ...` on a non-empty literal, which the JavaScript + * checker does not add to the literal's type) or a string index + * signature (`Record` over a literal). Lowering the value + * at the narrower literal type would drop the member or fix the key set, + * so it crosses at its declared type instead. */ + const missesDeclaredMembers = (jsType: ts.Type, declType: ts.Type): boolean => { + if (!(jsType.flags & ts.TypeFlags.Object) || !(declType.flags & ts.TypeFlags.Object)) + return false; + const literal = jsType.getSymbol(); + if (literal === undefined || (literal.flags & ts.SymbolFlags.ObjectLiteral) === 0) return false; + try { + if (declChecker.isArrayType(declType) || declChecker.isTupleType(declType)) return false; + if (stringIndex(declChecker, declType) && !stringIndex(jsChecker, jsType)) return true; + return declChecker + .getPropertiesOfType(declType) + .some( + (prop) => + !String(prop.escapedName).startsWith("__@") && + jsChecker.getPropertyOfType(jsType, String(prop.escapedName)) === undefined, + ); + } catch { + return false; + } + }; const namesPackageSymbol = (node: ts.Node): boolean => { let found = false; const visit = (n: ts.Node): void => { @@ -540,7 +575,7 @@ export function planDeclProjection(input: DeclProjectionInput): DeclProjection | const fileEdits: Edit[] = []; /** Candidate expressions (by js node) with their declared type, named * only for the casts that survive suppression. */ - const exprCandidates = new Map(); + const exprCandidates = new Map(); const exprOrder: ts.Node[] = []; /** Reads whose projectable receiver the JavaScript world types `any`: * kept only when nothing types the receiver. */ @@ -610,7 +645,14 @@ export function planDeclProjection(input: DeclProjectionInput): DeclProjection | return; } const jsType = typeAt(jsChecker, jsNode); - if (!isAnyish(jsType)) return; + if (jsType === null) return; + // A typed JavaScript value crosses only where its object type lacks a + // member the declaration promises (missesDeclaredMembers). + const typedJs = !isAnyish(jsType); + if (typedJs) { + const declType = typeAt(declChecker, declNode); + if (declType === null || !missesDeclaredMembers(jsType, declType)) return; + } // Package-produced? const d = declNode as ts.Expression; let produced = false; @@ -686,7 +728,18 @@ export function planDeclProjection(input: DeclProjectionInput): DeclProjection | // no contextual type } const declType = typeAt(declChecker, d); - if (declType === null || !printer.projectable(declType)) return; + if (declType === null) return; + if (!printer.projectable(declType)) { + // An index signature over values that do not project (package + // interfaces): the literal's fixed key set would still answer for + // keys it lacks, so the value stays untyped and every read is the + // JavaScript read. + if (typedJs && stringIndex(declChecker, declType)) { + exprCandidates.set(jsNode, "any"); + exprOrder.push(jsNode); + } + return; + } exprCandidates.set(jsNode, declType); exprOrder.push(jsNode); }; @@ -890,7 +943,8 @@ export function planDeclProjection(input: DeclProjectionInput): DeclProjection | ) continue; } - const alias = printer.project(exprCandidates.get(e)!); + const candidate = exprCandidates.get(e)!; + const alias = candidate === "any" ? "any" : printer.project(candidate); if (alias === null) continue; const start = ts.tokenStartOfNode(e); const stmt = e.parent; diff --git a/tests/fixtures/npm-static/declared-members-cli.ts b/tests/fixtures/npm-static/declared-members-cli.ts new file mode 100644 index 000000000..107ffc54d --- /dev/null +++ b/tests/fixtures/npm-static/declared-members-cli.ts @@ -0,0 +1,21 @@ +// A consumer reads members a package's declarations promise but its +// JavaScript adds in ways the inferred literal type misses: absent keys of +// index signatures read undefined, and expando properties read their values, +// as in Node. +import { counts, fonts, settings, stats } from "declared-members"; + +const key = process.argv.length > 10 ? "size" : "missing"; + +const s = settings(); +console.log(s["font"] ?? "default font", s["size"] ?? "default size", s.family); +console.log(s["font"] === undefined, typeof s["weight"], s.weight ?? "dot miss"); +console.log(s[key] ?? "dynamic miss", s[key], Object.keys(s)); + +const c = counts(); +console.log(c["chars"] ?? -1, c["lines"], c[key] === undefined); + +const st = stats([4, 9, 2]); +console.log(st.max, st.sum, st.count, st); + +const f = fonts(); +console.log(f["title"]?.name ?? "no title", f[key] === undefined, f["body"]?.name, f.code.name); diff --git a/tests/fixtures/npm-static/node_modules/declared-members/index.d.ts b/tests/fixtures/npm-static/node_modules/declared-members/index.d.ts new file mode 100644 index 000000000..8061fd5cb --- /dev/null +++ b/tests/fixtures/npm-static/node_modules/declared-members/index.d.ts @@ -0,0 +1,7 @@ +export declare function settings(): { [key: string]: string }; +export declare function counts(): Record; +export declare function stats(xs: number[]): { count: number; sum: number; max: number }; +export interface Font { + name: string; +} +export declare function fonts(): Record; diff --git a/tests/fixtures/npm-static/node_modules/declared-members/index.js b/tests/fixtures/npm-static/node_modules/declared-members/index.js new file mode 100644 index 000000000..381bd4a46 --- /dev/null +++ b/tests/fixtures/npm-static/node_modules/declared-members/index.js @@ -0,0 +1,34 @@ +"use strict"; +// Each function returns an object literal whose JavaScript-inferred type +// lacks members that index.d.ts promises. + +// An index signature over a literal built empty, then assigned. +function settings() { + const out = {}; + out.size = "12px"; + out.family = "serif"; + return out; +} + +// An index signature over a literal with fixed keys. +function counts() { + return { lines: 3, words: 12 }; +} + +// An expando on a non-empty literal: the checker's literal type has no `max`. +function stats(xs) { + const out = { count: xs.length, sum: 0 }; + for (const x of xs) out.sum += x; + out.max = Math.max(...xs); + return out; +} + +// An index signature whose values are a declared interface. +function fonts() { + return { body: { name: "serif" }, code: { name: "mono" } }; +} + +exports.settings = settings; +exports.counts = counts; +exports.stats = stats; +exports.fonts = fonts; diff --git a/tests/fixtures/npm-static/node_modules/declared-members/package.json b/tests/fixtures/npm-static/node_modules/declared-members/package.json new file mode 100644 index 000000000..4a42ebdf2 --- /dev/null +++ b/tests/fixtures/npm-static/node_modules/declared-members/package.json @@ -0,0 +1,6 @@ +{ + "name": "declared-members", + "version": "1.0.0", + "main": "index.js", + "types": "index.d.ts" +} diff --git a/tests/harness/npm-static.test.ts b/tests/harness/npm-static.test.ts index f3a6225c4..076733739 100644 --- a/tests/harness/npm-static.test.ts +++ b/tests/harness/npm-static.test.ts @@ -1207,6 +1207,18 @@ describe(`npm-static pilots${sanitize ? " (sanitized)" : ""}`, () => { expect(nativeRes.exitCode).toBe(nodeRes.exitCode); }, 180_000); + test("package values cross at the members their declarations promise", async () => { + const entry = join(pilotRoot, "declared-members-cli.ts"); + const binary = await buildStatic(entry, ["declared-members"]); + const [nodeRes, nativeRes] = await Promise.all([ + runBinary("node", [entry]), + runBinary(binary, []), + ]); + expect(nativeRes.stdout).toEqual(nodeRes.stdout); + expect(comparableStderr(nativeRes.stderr)).toEqual(nodeRes.stderr); + expect(nativeRes.exitCode).toBe(nodeRes.exitCode); + }, 180_000); + test("the published Effect Function subpath initializes and runs statically", async () => { const entry = join(pilotRoot, "effect-function-cli.ts"); const { coverage } = analyze(entry, { npmStatic: ["effect"] }); From a4bc01bafbbf5f0b9b2c8c346b1c3bae3cac6d20 Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Sun, 25 Oct 2026 11:52:19 -0700 Subject: [PATCH 18/44] Count shorthand property reads of an initializer-less JavaScript let An initializer-less JavaScript `let` whose writes all agree gets a scalar slot when every read already has that flow type (inferredEvolvingLetType). The read scan resolved each identifier with getSymbolAtLocation, which answers the property symbol for a shorthand property name, so the `width` in `return { width }` was not seen as a read. A function that assigns the let on one path only (`let width; if (tty) width = 80; return { width };`) then stored it in a number slot: the path that never assigns returned 80 or 0 instead of undefined. The capture scan missed a closure that reads the let through a shorthand the same way. Both scans now resolve a shorthand name to the binding it reads, so such a let keeps its checked-dynamic slot and reads undefined until assigned, as in Node. TypeScript lets are unaffected (the inference applies to JavaScript only). Co-Authored-By: Claude Opus 5.5 --- .../src/frontend/lowering/lower-stmts.ts | 14 ++++--- .../evolving-let-shorthand-reads.js.json | 6 +++ tests/corpus/evolving-let-shorthand-reads.js | 41 +++++++++++++++++++ .../evolving-let-shorthand-reads.js.json | 1 + 4 files changed, 56 insertions(+), 6 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/evolving-let-shorthand-reads.js.json create mode 100644 tests/corpus/evolving-let-shorthand-reads.js create mode 100644 tests/harness/corpus-costs/evolving-let-shorthand-reads.js.json diff --git a/packages/compiler/src/frontend/lowering/lower-stmts.ts b/packages/compiler/src/frontend/lowering/lower-stmts.ts index d6d8c1cc7..0563f6165 100644 --- a/packages/compiler/src/frontend/lowering/lower-stmts.ts +++ b/packages/compiler/src/frontend/lowering/lower-stmts.ts @@ -5422,6 +5422,12 @@ function inferredEvolvingLetType(lowerer: Lowerer, decl: ts.VariableDeclaration) const scope = decl.parent?.parent?.parent; if (symbol === undefined || (!ts.isBlock(scope) && !ts.isSourceFile(scope))) return null; + // A shorthand property's name (`{ width }`) resolves to the property + // symbol, but it reads the binding: the checker's shorthand-value symbol. + const refersToBinding = (node: ts.Identifier): boolean => + (ts.isShorthandPropertyAssignment(node.parent) && node.parent.name === node + ? lowerer.checker.getShorthandAssignmentValueSymbol(node.parent) + : lowerer.checker.getSymbolAtLocation(node)) === symbol; const isPlainAssignmentTarget = (node: ts.Identifier): boolean => { let target: ts.Expression = node; while (ts.isParenthesizedExpression(target.parent) && target.parent.expression === target) { @@ -5437,11 +5443,7 @@ function inferredEvolvingLetType(lowerer: Lowerer, decl: ts.VariableDeclaration) const captures = (fn: ts.Node): boolean => { let captured = false; ts.walkPreorder(fn, (node) => { - if ( - node !== fn && - ts.isIdentifier(node) && - lowerer.checker.getSymbolAtLocation(node) === symbol - ) { + if (node !== fn && ts.isIdentifier(node) && refersToBinding(node)) { captured = true; return "stop"; } @@ -5493,7 +5495,7 @@ function inferredEvolvingLetType(lowerer: Lowerer, decl: ts.VariableDeclaration) if ( ts.isIdentifier(node) && node !== decl.name && - lowerer.checker.getSymbolAtLocation(node) === symbol && + refersToBinding(node) && !isPlainAssignmentTarget(node) ) { reads.push(node); diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/evolving-let-shorthand-reads.js.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/evolving-let-shorthand-reads.js.json new file mode 100644 index 000000000..26d07a9f9 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/evolving-let-shorthand-reads.js.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/evolving-let-shorthand-reads.js" + ], + "diags": [] +} diff --git a/tests/corpus/evolving-let-shorthand-reads.js b/tests/corpus/evolving-let-shorthand-reads.js new file mode 100644 index 000000000..201171bcc --- /dev/null +++ b/tests/corpus/evolving-let-shorthand-reads.js @@ -0,0 +1,41 @@ +// An initializer-less JavaScript `let` read through a shorthand property +// (`{ width }`) still reads undefined on the paths that never assign it, as +// npm packages do when they build an options or size object. +function size(tty) { + let width; + if (tty) width = 80; + return { width }; +} + +/** + * @param {boolean} tty + * @returns {{ width: number | undefined }} + */ +function documented(tty) { + /** @type {number | undefined} */ + let width; + if (tty) width = 80; + return { width }; +} + +function captured(tty) { + let width; + const read = () => ({ width }); + const before = read().width; + if (tty) width = 80; + return [before, read().width]; +} + +function nested(flags) { + let columns; + let rows; + if (flags.columns) columns = 120; + if (flags.rows) rows = 40; + return { size: { columns, rows } }; +} + +console.log(size(true).width, size(false).width); +console.log(documented(true).width, documented(false).width); +console.log(JSON.stringify(size(false)), "width" in size(false)); +console.log(captured(true), captured(false)); +console.log(nested({ columns: true }), nested({ rows: true }), nested({})); diff --git a/tests/harness/corpus-costs/evolving-let-shorthand-reads.js.json b/tests/harness/corpus-costs/evolving-let-shorthand-reads.js.json new file mode 100644 index 000000000..f93ba161d --- /dev/null +++ b/tests/harness/corpus-costs/evolving-let-shorthand-reads.js.json @@ -0,0 +1 @@ +{ "plain": 3700, "san": 2800 } From 317d724a7b439cf8056eee530bf9cf9458227b16 Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Sun, 25 Oct 2026 10:46:52 -0700 Subject: [PATCH 19/44] Keep bound alias instantiations on the lowering pass The compiler is compiled by scriptc and keeps caches on their owners instead of weak collections (compiler-weak-collections.test.ts). Co-Authored-By: Claude Opus 5.5 --- .../frontend/lowering/generic-functions.ts | 25 ++----------------- .../compiler/src/frontend/lowering/lowerer.ts | 15 ++++++++--- 2 files changed, 14 insertions(+), 26 deletions(-) diff --git a/packages/compiler/src/frontend/lowering/generic-functions.ts b/packages/compiler/src/frontend/lowering/generic-functions.ts index 7d7124027..a2eda839a 100644 --- a/packages/compiler/src/frontend/lowering/generic-functions.ts +++ b/packages/compiler/src/frontend/lowering/generic-functions.ts @@ -921,27 +921,6 @@ function bindDefaultTypeParams( }); } -/** Generic alias references of a call-keyed instance's declared signature - * (`Catalog`), each paired with the same alias as the call's resolved - * signature instantiates it (`Catalog<{ g1: … }, "acme">`). The checker - * keeps the alias symbolic inside the generic body and offers no way to - * instantiate it with the bound arguments, but alias instantiations are - * interned: the body's `x as Catalog` is the very type the signature - * declares, so the pair recorded during unification is that re-instantiation. - * Keyed by the instance's ts-level bindings map, which every scope that - * lowers the instance installs (typeParamTsBindings). */ -const boundAliasInstances = new WeakMap, Map>(); - -/** The concrete instantiation of a generic alias reference inside the - * instance whose ts-level bindings are `tsBindings` (boundAliasInstances), - * or null when its signature never instantiated that reference. */ -export function boundAliasInstance( - tsBindings: Map, - type: ts.Type, -): ts.Type | null { - return boundAliasInstances.get(tsBindings)?.get(type) ?? null; -} - /** The structural half of binding inference: unify the DECLARED signature * (whose types mention the type parameters) against a TARGET signature * with the substitution applied — a call's resolved signature, or the @@ -1062,8 +1041,8 @@ function unifySignatureBindings( iAliasArgs.length === dAliasArgs.length ) { if (tsBindings && declared !== inst) { - let pairs = boundAliasInstances.get(tsBindings); - if (!pairs) boundAliasInstances.set(tsBindings, (pairs = new Map())); + let pairs = lowerer.boundAliasInstances.get(tsBindings); + if (!pairs) lowerer.boundAliasInstances.set(tsBindings, (pairs = new Map())); // A body forwarding its own alias (`inner(g)` returning `Catalog` inside an instance) resolves through the enclosing one. if (!pairs.has(declared)) pairs.set(declared, lowerer.boundAliasTsResolver(inst) ?? inst); diff --git a/packages/compiler/src/frontend/lowering/lowerer.ts b/packages/compiler/src/frontend/lowering/lowerer.ts index 52f3567fb..216851c9c 100644 --- a/packages/compiler/src/frontend/lowering/lowerer.ts +++ b/packages/compiler/src/frontend/lowering/lowerer.ts @@ -456,7 +456,6 @@ import { inferTypeParamBindings, lowerGenericInstance, genericLocalClassBinding, - boundAliasInstance, } from "./generic-functions.js"; import { bindingNeverReassigned } from "./binding-analysis.js"; import type { InstantiationPath } from "./instantiation-path.js"; @@ -2234,6 +2233,16 @@ export class Lowerer { * the body needs — indexed accesses (`T[K]` needs K's literal key) and * keyed record reads (`o[k]` where k's type is a literal-bound K). */ typeParamTsBindings: Map | null = null; + /** Generic alias references of a call-keyed instance's declared signature + * (`Catalog`), each paired with the same alias as the call's + * resolved signature instantiates it (`Catalog<{ g1: … }, "acme">`), + * keyed by the instance's ts-level bindings (typeParamTsBindings). The + * checker keeps the alias symbolic inside the generic body and offers no + * way to instantiate it with the bound arguments, but alias + * instantiations are interned: the body's `x as Catalog` is the + * very type the signature declares, so the pair recorded during binding + * unification is that re-instantiation. */ + readonly boundAliasInstances = new Map, Map>(); /** Non-null while an IMPLICIT-ANY instance body lowers (npm-static JS; * see generic-functions.ts): bound param symbol → * the call site's checker type, consulted by typeOf for identifier @@ -9554,10 +9563,10 @@ export class Lowerer { /** The concrete instantiation of a generic alias reference (`Catalog`) the current call-keyed instance's signature declares - * (boundAliasInstance). Inert outside generic function instantiation. */ + * (boundAliasInstances). Inert outside generic function instantiation. */ readonly boundAliasTsResolver = (t: ts.Type): ts.Type | null => { if (!this.typeParamTsBindings || t.getAliasTypeArguments().length === 0) return null; - return boundAliasInstance(this.typeParamTsBindings, t); + return this.boundAliasInstances.get(this.typeParamTsBindings)?.get(t) ?? null; }; irTypeOf(node: ts.Node): IrType { From 6000485a17155e71dba5a33c482a347309aadbba Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Sun, 25 Oct 2026 14:04:10 -0500 Subject: [PATCH 20/44] Report either decryption error of a wrong passphrase in corpus 8183 With a fresh key and IV each run, Node's OpenSSL now and then accepts the padding of a wrong-passphrase decryption and rejects the decrypted structure instead: ERR_OSSL_UNSUPPORTED rather than ERR_OSSL_BAD_DECRYPT, in about 1 run in 100. The program now prints both as one decryption error. --- tests/corpus/8183-key-passphrase-encoding.cjs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/corpus/8183-key-passphrase-encoding.cjs b/tests/corpus/8183-key-passphrase-encoding.cjs index e52fa6c1c..ca817f386 100644 --- a/tests/corpus/8183-key-passphrase-encoding.cjs +++ b/tests/corpus/8183-key-passphrase-encoding.cjs @@ -9,12 +9,17 @@ const hex = Buffer.from(pass).toString("hex"); const pem = privateKey.export({ format: "pem", type: "pkcs8", cipher: "aes-256-cbc", passphrase: hex, encoding: "hex" }); const der = privateKey.export({ format: "der", type: "pkcs8", cipher: "aes-128-cbc", passphrase: pass }); +// A wrong passphrase usually fails the decryption's padding check +// (ERR_OSSL_BAD_DECRYPT); with a fresh key and IV each run, Node's +// OpenSSL now and then accepts the padding and rejects the decrypted +// structure instead (ERR_OSSL_UNSUPPORTED). Both are reported as one. function attempt(label, options) { try { const key = c.createPrivateKey(options); console.log(label, key.asymmetricKeyType, key.equals(privateKey)); } catch (e) { - console.log(label, "throws", e.code); + const wrong = e.code === "ERR_OSSL_BAD_DECRYPT" || e.code === "ERR_OSSL_UNSUPPORTED"; + console.log(label, "throws", wrong ? "a decryption error" : e.code); } } From a2be7d8bc03ca9be9a3e1f941bbc4d759e449bd3 Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Sun, 25 Oct 2026 12:05:14 -0700 Subject: [PATCH 21/44] Answer Object.prototype's methods on typed records Records inherit toString, toLocaleString, valueOf, hasOwnProperty, isPrototypeOf and propertyIsEnumerable. Calls of these methods by name run an own callable entry when the record's type allows one and the record holds it, and otherwise the built-in with the record as receiver; toLocaleString, valueOf and isPrototypeOf on records were refused, and toString ignored an own entry. Keyed reads of records of functions answer the built-in as a native function value when the key misses: bound to the record when called at once, with an undefined `this` otherwise. The backend fuses the read with its fallback, so hits cost nothing extra. Records of other value types keep reading undefined for these names; the limitations page says so. Co-Authored-By: Claude Opus 5.5 --- .../record-identity-and-property-order.mdx | 7 + .../compiler/src/backend/llvm/expr-control.ts | 22 +- .../compiler/src/backend/llvm/expr-records.ts | 40 +- .../lowering/expressions/element-access.ts | 13 +- .../lowering/expressions/property-access.ts | 7 + .../src/frontend/lowering/lower-calls.ts | 4 + .../src/frontend/lowering/record-inherited.ts | 815 ++++++++++++++++++ ...8360-record-prototype-method-calls.ts.json | 6 + ...61-record-function-table-inherited.ts.json | 6 + .../8360-record-prototype-method-calls.ts | 56 ++ .../8361-record-function-table-inherited.ts | 40 + ...8360-record-prototype-method-calls.ts.json | 1 + ...61-record-function-table-inherited.ts.json | 1 + 13 files changed, 1014 insertions(+), 4 deletions(-) create mode 100644 packages/compiler/src/frontend/lowering/record-inherited.ts create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8360-record-prototype-method-calls.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8361-record-function-table-inherited.ts.json create mode 100644 tests/corpus/8360-record-prototype-method-calls.ts create mode 100644 tests/corpus/8361-record-function-table-inherited.ts create mode 100644 tests/harness/corpus-costs/8360-record-prototype-method-calls.ts.json create mode 100644 tests/harness/corpus-costs/8361-record-function-table-inherited.ts.json diff --git a/docs/content/limitations/record-identity-and-property-order.mdx b/docs/content/limitations/record-identity-and-property-order.mdx index dd180cb7b..fc0b2be92 100644 --- a/docs/content/limitations/record-identity-and-property-order.mdx +++ b/docs/content/limitations/record-identity-and-property-order.mdx @@ -25,3 +25,10 @@ Index-signature records list declared fields before the other keys. Optional fields and properties added through a record's `unknown` view can be deleted. A required field has no absent state, so deleting it through the view throws `TypeError: Cannot delete property 'x' of #` and leaves the record unchanged, where Node.js removes the property. + +Records inherit `toString`, `toLocaleString`, `valueOf`, `hasOwnProperty`, `isPrototypeOf` and `propertyIsEnumerable` from `Object.prototype`. +A call of one of these methods by name runs the built-in with the record as its receiver, unless the record holds an own function under that name. +A keyed read such as `handlers[name]` of a record whose values are functions answers the built-in when the record lacks the key, so truthiness, `typeof` and calls match Node.js. +A keyed read of a record whose values are not functions, such as `counts["toString"]` on a `Record`, answers `undefined` where Node.js answers the inherited function, so `counts[word] ?? 0` counts such a word as new. +Calling a built-in read from a record of functions throws a `TypeError` when its answer does not fit the record's function type, for example `valueOf` read from a `Record number>`. +Other inherited names, such as `constructor`, read `undefined` from typed records. diff --git a/packages/compiler/src/backend/llvm/expr-control.ts b/packages/compiler/src/backend/llvm/expr-control.ts index 2f7c31813..5ffa54eb5 100644 --- a/packages/compiler/src/backend/llvm/expr-control.ts +++ b/packages/compiler/src/backend/llvm/expr-control.ts @@ -2,11 +2,12 @@ import { emitBorrowedInput } from "./borrowed-inputs.js"; /* Focused LLVM expression emission extracted from emitter.ts. */ import { InternalCompilerError } from "../../errors.js"; import { undefinedArmTag } from "../../ir/analysis.js"; -import { isRefCounted, isUnitType, typeEquals, type IrExpr } from "../../ir/ir.js"; +import { isRefCounted, isUnitType, typeEquals, type IrExpr, type IrType } from "../../ir/ir.js"; import { DYN_KIND } from "./dyn.js"; import { LlvmUnsupportedError } from "./unsupported.js"; import type { LlvmEmitterContext, ExprOf, LlValue } from "./expr-context.js"; import { f64Lit } from "./common.js"; +import { emitRecordKeyGetOr } from "./expr-records.js"; import { NULLABLE_NULL, SHAPE_WORD_INDEX, shapeWordHeader } from "./nullable-unions.js"; import { emitUnionTag } from "./union-repr.js"; @@ -397,6 +398,13 @@ export function emitControlExpr( throw new InternalCompilerError("llvm emitter bug: nullish union lacks unit arms"); const fused = emitNullishOptionalChain(host, e); if (fused) return fused; + if ( + e.left.kind === "recordKeyGet" && + e.left.missNaN !== true && + typeEquals(e.type, e.left.type) && + storesNoUnit(host, e.left) + ) + return emitRecordKeyGetOr(host, e.left, e.right); if ( !typeEquals(e.type, e.left.type) && (e.left.kind === "varRef" || @@ -700,3 +708,15 @@ function emitUnionTagSet(host: LlvmEmitterContext, set: TagSet): string { B.line(`${t} = phi i1 [ true, %${yes} ], [ false, %${no} ]`); return t; } + +/** Every value a keyed read can hit is present (no stored undefined or + * null), so only a missing key yields the read's unit arm. */ +function storesNoUnit(host: LlvmEmitterContext, e: ExprOf<"recordKeyGet">): boolean { + const shape = host.recordShape(e.shapeId); + const holdsUnit = (t: IrType): boolean => + isUnitType(t) || + t.kind === "dyn" || + (t.kind === "union" && (host.unionsById.get(t.unionId)?.arms ?? []).some(isUnitType)); + if (shape.indexValue !== undefined && holdsUnit(shape.indexValue)) return false; + return e.overflowOnly === true || !shape.fields.some((f) => holdsUnit(f.type)); +} diff --git a/packages/compiler/src/backend/llvm/expr-records.ts b/packages/compiler/src/backend/llvm/expr-records.ts index 59ab625e5..86f20fd45 100644 --- a/packages/compiler/src/backend/llvm/expr-records.ts +++ b/packages/compiler/src/backend/llvm/expr-records.ts @@ -338,6 +338,39 @@ export function emitRegexIntrinsic( } } +/** `r[k] ?? d` in its pass-through shape: a missing key evaluates `d` + * where the read would store undefined, so a hit costs nothing extra. */ +export function emitRecordKeyGetOr( + host: LlvmEmitterContext, + e: IrExpr & { kind: "recordKeyGet" }, + fallback: IrExpr, +): LlValue { + const B = host.B; + const obj = host.emitExpr(e.obj); + const k = host.emitExpr(e.key); + const ty = host.llType(e.type); + const slot = B.slot(); + B.entryAllocas.push(`${slot} = alloca ${ty}`); + const join = B.newLabel("rkg.j"); + keyedRecordReadInto( + host, + slot, + join, + obj.name, + k.name, + e.shapeId, + e.type, + e.overflowOnly === true, + e.loc, + false, + () => host.emitBranchInto(slot, fallback), + ); + B.startBlock(join); + const t = B.tmp(); + B.line(`${t} = load ${ty}, ptr ${slot}`); + return host.own({ name: t, type: e.type }); +} + export function emitRecordKeyGet( host: LlvmEmitterContext, e: IrExpr & { kind: "recordKeyGet" }, @@ -485,6 +518,7 @@ export function keyedRecordReadInto( overflowOnly: boolean, loc?: SrcLoc, missNaN = false, + missInto: (() => void) | null = null, ): void { const B = host.B; const shape = host.recordShape(shapeId); @@ -687,8 +721,10 @@ export function keyedRecordReadInto( } } if (def !== null && resultType.kind === "union" && undefTag >= 0) { - // The miss path: the result union's undefined arm. - B.line(`store ptr ${host.unitInstanceRef(resultType.unionId, undefTag)}, ptr ${slot}`); + // The miss path: the result union's undefined arm, or the caller's + // own value for a miss (missInto stores it into the slot). + if (missInto !== null) missInto(); + else B.line(`store ptr ${host.unitInstanceRef(resultType.unionId, undefTag)}, ptr ${slot}`); B.br(join); return; } diff --git a/packages/compiler/src/frontend/lowering/expressions/element-access.ts b/packages/compiler/src/frontend/lowering/expressions/element-access.ts index a2f2ebdb6..565d0f843 100644 --- a/packages/compiler/src/frontend/lowering/expressions/element-access.ts +++ b/packages/compiler/src/frontend/lowering/expressions/element-access.ts @@ -69,6 +69,7 @@ import { InternalCompilerError } from "../../../errors.js"; import { dynUndefinedExpr, nodeThrowExpr, numLit, strLit, varRef } from "../../../ir/build.js"; import { npmStaticPackageOfPath } from "../../npm-static.js"; import { isSafeToRepeat } from "./evaluation-safety.js"; +import { withInheritedMethods } from "../record-inherited.js"; import { lowerClassPrototypeComputedAssignment } from "../class-prototypes.js"; import { isClassCallback } from "../class-callbacks.js"; import { isKeyedFieldAccess } from "./keyed-field-targets.js"; @@ -1444,7 +1445,17 @@ export function lowerRecordKeyRead( loc, }; if (nanCoded) return read; - return includeUndefined || optionalRead ? read : lowerer.maybeNarrow(read, expr); + const answered = withInheritedMethods(lowerer, read, litKey, isCalledAccess(expr)); + return includeUndefined || optionalRead ? answered : lowerer.maybeNarrow(answered, expr); +} + +/** The access is the callee of a call (`o[k](…)`, `o[k]?.(…)`, `o[k]!(…)`), + * which passes the object as `this`. */ +function isCalledAccess(node: ts.Expression): boolean { + let child: ts.Expression = node; + while (ts.isParenthesizedExpression(child.parent) || ts.isNonNullExpression(child.parent)) + child = child.parent; + return ts.isCallExpression(child.parent) && child.parent.expression === child; } /** ToPropertyKey for represented record keys. Missing array-derived string diff --git a/packages/compiler/src/frontend/lowering/expressions/property-access.ts b/packages/compiler/src/frontend/lowering/expressions/property-access.ts index 35bb16cd3..f213b6cbf 100644 --- a/packages/compiler/src/frontend/lowering/expressions/property-access.ts +++ b/packages/compiler/src/frontend/lowering/expressions/property-access.ts @@ -115,6 +115,7 @@ import { dynUndefinedExpr, nodeThrowExpr, numLit, strLit, varRef } from "../../. import { typedArrayLacksMember } from "../builtins/typed-array-members.js"; import { TEMPORAL_GLOBAL } from "../../builtin-modules.js"; import { tryLowerExpression } from "./try-lower-expression.js"; +import { lowerRecordInheritedMember } from "../record-inherited.js"; import { DATE_METHOD_HINT } from "../builtins/dates.js"; import { cjsClassExprWholeExportOf, @@ -3188,6 +3189,12 @@ export function lowerFieldRead(lowerer: Lowerer, expr: ts.PropertyAccessExpressi loc: locOf(expr), }; } + // Object.prototype's methods read as values: an own entry of the + // member's function type, else the built-in. + if (actual?.type.kind === "record") { + const inherited = lowerRecordInheritedMember(lowerer, actual, expr); + if (inherited) return inherited; + } // A member the checker declares on the receiver, stored in a pure // index-signature record (an object literal that spread one, see // dictSpreadLiteralShape): the key is always present, so the keyed diff --git a/packages/compiler/src/frontend/lowering/lower-calls.ts b/packages/compiler/src/frontend/lowering/lower-calls.ts index c8f02562e..aeb1993d4 100644 --- a/packages/compiler/src/frontend/lowering/lower-calls.ts +++ b/packages/compiler/src/frontend/lowering/lower-calls.ts @@ -280,6 +280,7 @@ import { } from "./generic-functions.js"; import { codeGenerationCalleeOf, lowerCodeGenerationCall } from "./code-generation.js"; +import { lowerRecordInheritedCall } from "./record-inherited.js"; const ZLIB_STREAM_METHODS: ReadonlySet = new Set(["flush", "params", "close", "reset"]); @@ -1643,6 +1644,9 @@ export function lowerCall(lowerer: Lowerer, expr: ts.CallExpression): IrExpr { // Union receivers whose every arm has a text — the ngrok // `(chunk: Buffer | string) => chunk.toString()` idiom. lowerUnionToStringCall(lowerer, expr, expr.expression) ?? + // Object.prototype's methods called by name on records: an own + // callable entry, else the built-in with the record as receiver. + lowerRecordInheritedCall(lowerer, expr, expr.expression) ?? // Object.prototype.toString's default answer on records and // override-free program classes — "[object Object]", folded. lowerDefaultToStringCall(lowerer, expr, expr.expression) ?? diff --git a/packages/compiler/src/frontend/lowering/record-inherited.ts b/packages/compiler/src/frontend/lowering/record-inherited.ts new file mode 100644 index 000000000..2c01ccd0a --- /dev/null +++ b/packages/compiler/src/frontend/lowering/record-inherited.ts @@ -0,0 +1,815 @@ +/* Object.prototype's methods on typed records. + * + * A record is an ordinary object, so `r.toString()`, `r.hasOwnProperty(k)` + * and the other Object.prototype methods answer from its prototype unless + * the record holds an own entry under that name. Method calls with a + * constant name call an own entry when the record's type lets one be + * callable and otherwise run the built-in natively with the record as its + * receiver. Keyed reads of records whose values are functions or checked + * values answer the built-in as a native function value on the miss path, + * so truthiness, typeof and calls behave as in Node; hits are untouched. + * Records of other value types keep reading undefined for these names + * (a documented limitation). No dynamic-tier unit is involved: every + * built-in is a small lifted function over the record. */ +import * as ts from "../ts7/adapter.js"; +import { dynUndefinedExpr, nodeThrowExpr, varRef } from "../../ir/build.js"; +import { + BOOL, + DYN, + STRING, + UNDEFINED_T, + isUnitType, + typeEquals, + typeKey, + type IrExpr, + type IrLocal, + type IrStmt, + type IrType, + type SrcLoc, +} from "../../ir/ir.js"; +import type { Lowerer } from "./lowerer.js"; +import { locOf } from "../program.js"; +import { keyedHelperName } from "./stable-names.js"; +import { isSafeToRepeat } from "./expressions/evaluation-safety.js"; +import { nullProtoRecordShape, recordNullProtoTest } from "./null-prototype.js"; +import { recordHasOwnHelper } from "./lower-calls.js"; +import { OTHER_ARM_FAILURE_MESSAGE } from "./coercions/unions.js"; + +/** Object.prototype's methods a record answers from its prototype. */ +export const RECORD_INHERITED_METHODS: ReadonlySet = new Set([ + "toString", + "toLocaleString", + "valueOf", + "hasOwnProperty", + "isPrototypeOf", + "propertyIsEnumerable", +]); + +type FuncType = IrType & { kind: "func" }; + +/** Declares a hidden local in the function being built: the lowerer's + * current function, or a lifted helper's own local list. */ +type DeclareLocal = (name: string, type: IrType) => string; + +function liftedLocals(): { declare: DeclareLocal; locals: IrLocal[] } { + const locals: IrLocal[] = []; + return { + locals, + declare: (name, type) => { + const id = `${name}.${locals.length}`; + locals.push({ id, name, type, mutable: false }); + return id; + }, + }; +} + +/** The form a read of `type` gives a built-in method value: the union's one + * function arm (beside an undefined arm), a checked value, or null when the + * type cannot hold a function. */ +function methodValueForm(lowerer: Lowerer, type: IrType): FuncType | "dyn" | null { + if (type.kind === "dyn") return "dyn"; + if (type.kind !== "union") return null; + const arms = lowerer.unions.get(type.unionId)?.arms ?? []; + if (!arms.some((arm) => arm.kind === "undefinedT")) return null; + const funcs = arms.filter((arm): arm is FuncType => arm.kind === "func"); + if (funcs.length !== 1 || funcs[0]!.restAbi !== undefined) return null; + return funcs[0]!; +} + +/** A value of `type` may be a function that a call can enter. */ +function callableType(lowerer: Lowerer, type: IrType): boolean { + if (type.kind === "func" || type.kind === "dyn") return true; + if (type.kind !== "union") return false; + return (lowerer.unions.get(type.unionId)?.arms ?? []).some( + (arm) => arm.kind === "func" || arm.kind === "dyn", + ); +} + +/** A keyed record read (`r[k]`, `r["toString"]`) whose miss answers the + * inherited Object.prototype method for those names. Hits keep the + * ordinary read; only the miss path tests the key. `bound`: the read is + * the callee of a call (`r[k](…)`, `r[k]?.(…)`), whose `this` is the + * record; any other read yields the method as a plain value, whose calls + * see an undefined `this` as in Node. */ +export function withInheritedMethods( + lowerer: Lowerer, + read: IrExpr & { kind: "recordKeyGet" }, + litKey: string | null, + bound: boolean, +): IrExpr { + if (litKey !== null && !RECORD_INHERITED_METHODS.has(litKey)) return read; + if (read.missNaN === true) return read; + const form = methodValueForm(lowerer, read.type); + const shape = lowerer.shapes.get(read.shapeId); + if (form === null || !shape || shape.tuple) return read; + const loc = read.loc; + const stmts: IrStmt[] = []; + const stable = (value: IrExpr, name: string): IrExpr => { + if (isSafeToRepeat(value) || value.kind === "strLit") return value; + const local = lowerer.declareHiddenLocal(name, value.type); + stmts.push({ kind: "varDecl", localId: local.id, init: value, loc }); + return varRef(local.id, value.type, loc); + }; + const obj = stable(read.obj, "%inheritedRecv"); + const key = stable(read.key, "%inheritedKey"); + const helper = inheritedReadHelper(lowerer, read.shapeId, read.type, form, bound, loc); + const fallback: IrExpr = { kind: "call", callee: helper, args: [obj, key], type: read.type, loc }; + if ( + form !== "dyn" && + read.type.kind === "union" && + !(lowerer.unions.get(read.type.unionId)?.arms ?? []).some((arm) => arm.kind === "nullT") + ) { + // `r[k] ?? inherited(r, k)`: the hit is the read's own value, untouched. + const coalesced: IrExpr = { + kind: "nullish", + left: { ...read, obj, key }, + right: fallback, + type: read.type, + loc, + }; + return stmts.length === 0 + ? coalesced + : { kind: "seqExpr", stmts, result: coalesced, type: read.type, loc }; + } + const value = lowerer.declareHiddenLocal("%inheritedRead", read.type); + stmts.push({ kind: "varDecl", localId: value.id, init: { ...read, obj, key }, loc }); + const found = varRef(value.id, read.type, loc); + let missing: IrExpr; + if (form === "dyn") { + missing = { kind: "dynTest", test: "undefined", value: found, type: BOOL, loc } as IrExpr; + } else { + const unionId = (read.type as IrType & { kind: "union" }).unionId; + const tag = (lowerer.unions.get(unionId)?.arms ?? []).findIndex( + (arm) => arm.kind === "undefinedT", + ); + missing = { kind: "unionIsTag", unionId, tag, negated: false, value: found, type: BOOL, loc }; + } + return { + kind: "seqExpr", + stmts, + result: { + kind: "ternary", + cond: missing, + then: fallback, + else_: found, + type: read.type, + loc, + }, + type: read.type, + loc, + }; +} + +/** `%rec.inherited.(o, k)`: the inherited method `k` names as a value + * of `readType` (bound to `o`), or undefined. */ +function inheritedReadHelper( + lowerer: Lowerer, + shapeId: string, + readType: IrType, + form: FuncType | "dyn", + bound: boolean, + loc: SrcLoc, +): string { + const key = `rec.inherited:${bound ? "bound" : "value"}:${shapeId}:${typeKey(readType)}`; + const existing = lowerer.arrHofHelpers.get(key); + if (existing) return existing; + const name = keyedHelperName(lowerer, "%rec.inherited", key); + lowerer.arrHofHelpers.set(key, name); + const recT: IrType = { kind: "record", shapeId }; + const o = varRef("o.0", recT, loc); + const k = varRef("k.0", STRING, loc); + const fnType: FuncType = + form === "dyn" ? ({ kind: "func", params: [DYN], ret: DYN } as FuncType) : form; + const undefinedValue: IrExpr = + form === "dyn" + ? dynUndefinedExpr(loc) + : lowerer.coerceToExpected( + { kind: "unitLit", unit: "undefined", type: UNDEFINED_T, loc }, + readType, + ); + const named = (method: string): IrExpr => ({ + kind: "strEq", + negated: false, + left: k, + right: { kind: "strLit", value: method, type: STRING, loc }, + type: BOOL, + loc, + }); + const methods = [...RECORD_INHERITED_METHODS]; + let anyName: IrExpr = named(methods[0]!); + for (const method of methods.slice(1)) + anyName = { kind: "logical", op: "||", left: anyName, right: named(method), type: BOOL, loc }; + const body: IrStmt[] = [ + // Most misses name no inherited method: one comparison chain, nothing else. + { + kind: "if", + cond: { kind: "unary", op: "!", operand: anyName, type: BOOL, loc }, + then: [{ kind: "return", value: { ...undefinedValue }, loc }], + else_: null, + loc, + }, + // An own key holding undefined shadows the prototype like any own key. + { + kind: "if", + cond: { + kind: "call", + callee: recordHasOwnHelper(lowerer, shapeId, loc), + args: [o, k], + type: BOOL, + loc, + }, + then: [{ kind: "return", value: { ...undefinedValue }, loc }], + else_: null, + loc, + }, + ]; + if (nullProtoRecordShape(lowerer, recT)) + body.push({ + kind: "if", + cond: recordNullProtoTest(o, shapeId, loc), + then: [{ kind: "return", value: { ...undefinedValue }, loc }], + else_: null, + loc, + }); + for (const method of methods) { + const value: IrExpr = bound + ? { + kind: "call", + callee: boundMethodFactory(lowerer, shapeId, method, fnType, loc), + args: [o], + type: fnType, + loc, + } + : { + kind: "closure", + fnName: unboundMethodImpl(lowerer, method, fnType, loc), + captures: [], + type: fnType, + loc, + }; + body.push({ + kind: "if", + cond: named(method), + then: [ + { + kind: "return", + value: lowerer.coerceToExpected(value, form === "dyn" ? DYN : readType), + loc, + }, + ], + else_: null, + loc, + }); + } + body.push({ kind: "return", value: { ...undefinedValue }, loc }); + lowerer.liftedFns.push({ + name, + params: [ + { localId: "o.0", name: "o", type: recT }, + { localId: "k.0", name: "k", type: STRING }, + ], + returnType: readType, + locals: [ + { id: "o.0", name: "o", type: recT, mutable: false }, + { id: "k.0", name: "k", type: STRING, mutable: false }, + ], + body, + loc, + }); + return name; +} + +/** `%rec.bind.(o)`: the built-in `method` bound to `o` as a closure. */ +function boundMethodFactory( + lowerer: Lowerer, + shapeId: string, + method: string, + fnType: FuncType, + loc: SrcLoc, +): string { + const key = `rec.bind:${shapeId}:${method}:${typeKey(fnType)}`; + const existing = lowerer.arrHofHelpers.get(key); + if (existing) return existing; + const name = keyedHelperName(lowerer, "%rec.bind", key); + lowerer.arrHofHelpers.set(key, name); + const recT: IrType = { kind: "record", shapeId }; + lowerer.liftedFns.push({ + name, + params: [{ localId: "o.0", name: "o", type: recT }], + returnType: fnType, + locals: [{ id: "o.0", name: "o", type: recT, mutable: false, boxed: true }], + body: [ + { + kind: "return", + value: { + kind: "closure", + fnName: inheritedMethodImpl(lowerer, shapeId, method, fnType, loc), + captures: ["o.0"], + type: fnType, + loc, + }, + loc, + }, + ], + loc, + }); + return name; +} + +/** The lifted body of one built-in bound to its record, with the read's + * function signature: the built-in's answer converted to the signature's + * return type, or the checked-conversion TypeError where it cannot be. */ +function inheritedMethodImpl( + lowerer: Lowerer, + shapeId: string, + method: string, + fnType: FuncType, + loc: SrcLoc, +): string { + const key = `rec.inheritedImpl:${shapeId}:${method}:${typeKey(fnType)}`; + const existing = lowerer.arrHofHelpers.get(key); + if (existing) return existing; + const name = keyedHelperName(lowerer, `%rec.${method}`, key); + lowerer.arrHofHelpers.set(key, name); + const recT: IrType = { kind: "record", shapeId }; + const o = varRef("o.0", recT, loc); + const params = fnType.params.map((type, i) => ({ localId: `p${i}.0`, name: `p${i}`, type })); + const args = params.map((p) => varRef(p.localId, p.type, loc)); + const body: IrStmt[] = []; + const hidden = liftedLocals(); + const answer = builtinAnswer(lowerer, hidden.declare, o, shapeId, method, args, fnType.ret, loc); + body.push(...returnAs(lowerer, answer, fnType.ret, loc)); + lowerer.liftedFns.push({ + name, + params, + returnType: fnType.ret, + captures: [{ localId: "o.0", name: "o", type: recT }], + locals: [ + { id: "o.0", name: "o", type: recT, mutable: false, boxed: true }, + ...params.map((p) => ({ id: p.localId, name: p.name, type: p.type, mutable: false })), + ...hidden.locals, + ], + body, + loc, + }); + return name; +} + +/** One built-in as a plain function value (no receiver): its answer for an + * undefined `this`, converted like inheritedMethodImpl's. */ +function unboundMethodImpl( + lowerer: Lowerer, + method: string, + fnType: FuncType, + loc: SrcLoc, +): string { + const key = `rec.unboundImpl:${method}:${typeKey(fnType)}`; + const existing = lowerer.arrHofHelpers.get(key); + if (existing) return existing; + const name = keyedHelperName(lowerer, `%obj.${method}`, key); + lowerer.arrHofHelpers.set(key, name); + const params = fnType.params.map((type, i) => ({ localId: `p${i}.0`, name: `p${i}`, type })); + const arg: IrExpr | undefined = params[0] && varRef(params[0].localId, params[0].type, loc); + const coerce = "Cannot convert undefined or null to object"; + let answer: IrExpr; + switch (method) { + case "toString": + answer = { kind: "strLit", value: "[object Undefined]", type: STRING, loc }; + break; + case "toLocaleString": + answer = nodeThrowExpr( + 1, + "", + "Object.prototype.toLocaleString called on null or undefined", + STRING, + loc, + ); + break; + case "isPrototypeOf": { + // A primitive argument answers false before `this` converts. + const no: IrExpr = { kind: "boolLit", value: false, type: BOOL, loc }; + const kind = arg?.type.kind; + if ( + arg === undefined || + isUnitType(arg.type) || + kind === "string" || + kind === "f64" || + kind === "bool" || + kind === "bigint" + ) + answer = no; + else if (kind === "dyn") + answer = { + kind: "ternary", + cond: { + kind: "logical", + op: "||", + left: { kind: "dynTest", test: "object", value: arg, type: BOOL, loc }, + right: { kind: "dynTest", test: "function", value: arg, type: BOOL, loc }, + type: BOOL, + loc, + } as IrExpr, + then: nodeThrowExpr(1, "", coerce, BOOL, loc), + else_: no, + type: BOOL, + loc, + }; + else answer = nodeThrowExpr(1, "", coerce, BOOL, loc); + break; + } + default: + answer = nodeThrowExpr(1, "", coerce, method === "valueOf" ? DYN : BOOL, loc); + } + const body: IrStmt[] = + answer.kind === "libCall" && answer.fn === "error.nodeThrow" + ? [ + { + kind: "return", + value: nodeThrowExpr( + 1, + "", + (answer.args[2] as IrExpr & { kind: "strLit" }).value, + fnType.ret, + loc, + ), + loc, + }, + ] + : returnAs(lowerer, answer, fnType.ret, loc); + lowerer.liftedFns.push({ + name, + params, + returnType: fnType.ret, + locals: params.map((p) => ({ id: p.localId, name: p.name, type: p.type, mutable: false })), + body, + loc, + }); + return name; +} + +/** The built-in's answer with `o` as its receiver. `want` is the type the + * caller needs (valueOf answers the record itself). */ +function builtinAnswer( + lowerer: Lowerer, + declare: DeclareLocal, + o: IrExpr, + shapeId: string, + method: string, + args: readonly IrExpr[], + want: IrType, + loc: SrcLoc, +): IrExpr { + switch (method) { + case "toString": + return { kind: "strLit", value: "[object Object]", type: STRING, loc }; + case "toLocaleString": + // Object.prototype.toLocaleString calls this.toString(). + return callOwnOrBuiltin( + lowerer, + declare, + o, + shapeId, + "toString", + [], + STRING, + "toString", + loc, + ); + case "valueOf": + return want.kind === "dyn" ? lowerer.coerceToExpected(o, DYN) : o; + case "isPrototypeOf": + // No object is created with a typed record as its prototype. + return { kind: "boolLit", value: false, type: BOOL, loc }; + default: { + const key = propertyKeyOf(lowerer, args[0], loc); + return { + kind: "call", + callee: recordHasOwnHelper(lowerer, shapeId, loc), + args: [o, key], + type: BOOL, + loc, + }; + } + } +} + +/** ToPropertyKey of a built-in's key argument (absent: "undefined"). */ +function propertyKeyOf(lowerer: Lowerer, arg: IrExpr | undefined, loc: SrcLoc): IrExpr { + if (arg === undefined || isUnitType(arg.type)) + return { + kind: "strLit", + value: arg?.type.kind === "nullT" ? "null" : "undefined", + type: STRING, + loc, + }; + if (arg.type.kind === "string") return arg; + if (arg.type.kind === "f64" || arg.type.kind === "bool") + return { kind: "toString", operand: arg, type: STRING, loc }; + return { + kind: "libCall", + fn: "dyn.toStringCoerce", + args: [lowerer.coerceToExpected(arg, DYN)], + type: STRING, + loc, + }; +} + +/** `value` returned as `ret`: as is, wrapped into a union arm or a checked + * value, discarded for void, or the checked-conversion TypeError. */ +function returnAs(lowerer: Lowerer, value: IrExpr, ret: IrType, loc: SrcLoc): IrStmt[] { + const converted = convertTo(lowerer, value, ret); + if (converted !== null) return [{ kind: "return", value: converted, loc }]; + if (ret.kind === "void") + return [ + { kind: "exprStmt", expr: value, loc }, + { kind: "return", value: null, loc }, + ]; + return [ + { kind: "exprStmt", expr: value, loc }, + { kind: "return", value: nodeThrowExpr(1, "", OTHER_ARM_FAILURE_MESSAGE, ret, loc), loc }, + ]; +} + +function convertTo(lowerer: Lowerer, value: IrExpr, want: IrType): IrExpr | null { + if (typeEquals(value.type, want)) return value; + if (want.kind === "dyn" && lowerer.dynConvertible(value.type)) + return lowerer.coerceToExpected(value, DYN); + if (want.kind === "union" && lowerer.armTag(want.unionId, value.type) >= 0) + return lowerer.coerceToExpected(value, want); + return null; +} + +/** `o.method(...args)` on a record: an own callable entry when the record + * holds one, else the built-in. `calleeText` names the callee for Node's + * "is not a function". */ +function callOwnOrBuiltin( + lowerer: Lowerer, + declare: DeclareLocal, + o: IrExpr, + shapeId: string, + method: string, + args: readonly IrExpr[], + want: IrType, + calleeText: string, + loc: SrcLoc, +): IrExpr { + const shape = lowerer.shapes.get(shapeId)!; + const field = shape.fields.find((f) => f.name === method); + const ownType = field ? field.type : shape.indexValue; + let builtin = builtinAnswer(lowerer, declare, o, shapeId, method, args, want, loc); + const recT: IrType = { kind: "record", shapeId }; + if (nullProtoRecordShape(lowerer, recT)) + builtin = { + kind: "ternary", + cond: recordNullProtoTest(o, shapeId, loc), + then: nodeThrowExpr(1, "", `${calleeText} is not a function`, builtin.type, loc), + else_: builtin, + type: builtin.type, + loc, + }; + const builtinAsWant = convertTo(lowerer, builtin, want) ?? builtin; + if (ownType === undefined || !callableType(lowerer, ownType)) return builtinAsWant; + const own: IrExpr = field + ? { kind: "recordGet", obj: o, shapeId, field: method, type: field.type, loc } + : { + kind: "recordKeyGet", + obj: o, + shapeId, + key: { kind: "strLit", value: method, type: STRING, loc }, + overflowOnly: true, + type: lowerer.withUndefinedArmOf(shape.indexValue!) ?? shape.indexValue!, + loc, + }; + const called = callOwn(lowerer, declare, own, args, builtinAsWant.type, calleeText, loc); + return { + kind: "ternary", + cond: { + kind: "call", + callee: recordHasOwnHelper(lowerer, shapeId, loc), + args: [o, { kind: "strLit", value: method, type: STRING, loc }], + type: BOOL, + loc, + }, + then: called, + else_: builtinAsWant, + type: builtinAsWant.type, + loc, + }; +} + +/** Call the own entry `own` (present, of any stored type) with `args`. A + * non-function throws Node's "is not a function"; a function whose + * parameters or result cannot carry the call throws the checked-conversion + * TypeError (the program's types claimed otherwise). */ +function callOwn( + lowerer: Lowerer, + declare: DeclareLocal, + own: IrExpr, + args: readonly IrExpr[], + want: IrType, + calleeText: string, + loc: SrcLoc, +): IrExpr { + const notFunction = nodeThrowExpr(1, "", `${calleeText} is not a function`, want, loc); + const lie = nodeThrowExpr(1, "", OTHER_ARM_FAILURE_MESSAGE, want, loc); + const localId = declare("%ownMethod", own.type); + const value = varRef(localId, own.type, loc); + const enter = (fn: IrExpr): IrExpr => { + if (fn.type.kind === "dyn") { + const result: IrExpr = { + kind: "dynCall", + callee: fn, + calleeName: calleeText, + args: args.map((a) => lowerer.coerceToExpected(a, DYN)), + type: DYN, + loc, + } as IrExpr; + return convertTo(lowerer, result, want) ?? sequence(result, lie, loc); + } + const sig = fn.type as FuncType; + if (sig.restAbi !== undefined) return lie; + const passed: IrExpr[] = []; + for (let i = 0; i < sig.params.length; i++) { + const param = sig.params[i]!; + const arg = args[i]; + const given = + arg === undefined + ? convertTo( + lowerer, + { kind: "unitLit", unit: "undefined", type: UNDEFINED_T, loc }, + param, + ) + : convertTo(lowerer, arg, param); + if (given === null) return lie; + passed.push(given); + } + const result: IrExpr = { kind: "callValue", callee: fn, args: passed, type: sig.ret, loc }; + if (sig.ret.kind === "void") return sequence(result, lie, loc); + return convertTo(lowerer, result, want) ?? sequence(result, lie, loc); + }; + let call: IrExpr; + if (own.type.kind === "func" || own.type.kind === "dyn") call = enter(value); + else if (own.type.kind === "union") { + const unionId = own.type.unionId; + const arms = lowerer.unions.get(unionId)?.arms ?? []; + call = notFunction; + for (let tag = arms.length - 1; tag >= 0; tag--) { + const arm = arms[tag]!; + if (arm.kind !== "func" && arm.kind !== "dyn") continue; + call = { + kind: "ternary", + cond: { kind: "unionIsTag", unionId, tag, negated: false, value, type: BOOL, loc }, + then: enter({ kind: "unionNarrow", unionId, tag, value, type: arm, loc }), + else_: call, + type: want, + loc, + }; + } + } else call = notFunction; + return { + kind: "seqExpr", + stmts: [{ kind: "varDecl", localId, init: own, loc }], + result: call, + type: want, + loc, + }; +} + +function sequence(first: IrExpr, result: IrExpr, loc: SrcLoc): IrExpr { + return { + kind: "seqExpr", + stmts: [{ kind: "exprStmt", expr: first, loc }], + result, + type: result.type, + loc, + }; +} + +/** `r.toString()`, `r.hasOwnProperty(k)` and the other Object.prototype + * methods called by name on a typed record (the checker resolved the + * member to Object.prototype's). Null for other calls. */ +export function lowerRecordInheritedCall( + lowerer: Lowerer, + call: ts.CallExpression, + access: ts.PropertyAccessExpression, +): IrExpr | null { + const method = access.name.text; + if ( + call.questionDotToken || + access.questionDotToken || + !RECORD_INHERITED_METHODS.has(method) || + call.arguments.nodes.some(ts.isSpreadElement) || + !lowerer.isStdlibMember(access) + ) + return null; + const recvT = lowerer.mapTypeOf(lowerer.typeOf(access.expression)); + if (recvT?.kind !== "record") return null; + const shape = lowerer.shapes.get(recvT.shapeId); + if (!shape || shape.tuple) return null; + const want = + method === "valueOf" + ? (lowerer.mapTypeOf(lowerer.typeOf(call)) ?? recvT) + : method === "toString" || method === "toLocaleString" + ? STRING + : BOOL; + // Without a possible own function, toString and the own-key tests keep + // their existing lowerings (with the special record views they know). + if ( + (method === "hasOwnProperty" || method === "propertyIsEnumerable" || method === "toString") && + !shape.fields.some((f) => f.name === method && callableType(lowerer, f.type)) && + !(shape.indexValue && callableType(lowerer, shape.indexValue)) + ) + return null; + const loc = locOf(call); + const stmts: IrStmt[] = []; + const receiver = lowerer.lowerExpr(access.expression); + if (receiver.type.kind !== "record") return null; + const recvLocal = lowerer.declareHiddenLocal("%protoRecv", receiver.type); + stmts.push({ kind: "varDecl", localId: recvLocal.id, init: receiver, loc }); + const o = varRef(recvLocal.id, receiver.type, loc); + const args: IrExpr[] = []; + for (const node of call.arguments.nodes) { + const arg = lowerer.lowerExpr(node); + const local = lowerer.declareHiddenLocal("%protoArg", arg.type); + stmts.push({ kind: "varDecl", localId: local.id, init: arg, loc }); + args.push(varRef(local.id, arg.type, loc)); + } + const result = callOwnOrBuiltin( + lowerer, + (name, type) => lowerer.declareHiddenLocal(name, type).id, + o, + receiver.type.shapeId, + method, + args, + want, + ts.getTextOfNode(access), + loc, + ); + return { kind: "seqExpr", stmts, result, type: result.type, loc }; +} + +/** `r.toString` and the other Object.prototype methods read by name (not + * called) on an index-signature record: an own entry of the member's + * function type when the record holds one, else the built-in as a plain + * function value. Null when the checker's member type is not a function. */ +export function lowerRecordInheritedMember( + lowerer: Lowerer, + receiver: IrExpr, + expr: ts.PropertyAccessExpression, +): IrExpr | null { + const method = expr.name.text; + if (!RECORD_INHERITED_METHODS.has(method) || !lowerer.isStdlibMember(expr)) return null; + if (receiver.type.kind !== "record") return null; + const want = lowerer.mapTypeOf(lowerer.typeOf(expr)); + if (want?.kind !== "func" || want.restAbi !== undefined) return null; + const shapeId = receiver.type.shapeId; + const shape = lowerer.shapes.get(shapeId); + if (!shape || shape.tuple) return null; + const loc = locOf(expr); + const builtin: IrExpr = { + kind: "closure", + fnName: unboundMethodImpl(lowerer, method, want, loc), + captures: [], + type: want, + loc, + }; + const recvLocal = lowerer.declareHiddenLocal("%protoRecv", receiver.type); + const o = varRef(recvLocal.id, receiver.type, loc); + const stmts: IrStmt[] = [{ kind: "varDecl", localId: recvLocal.id, init: receiver, loc }]; + let result: IrExpr = builtin; + if (shape.indexValue && typeEquals(shape.indexValue, want)) { + const ownType = lowerer.withUndefinedArmOf(want); + if (ownType?.kind === "union") { + const read: IrExpr = { + kind: "recordKeyGet", + obj: o, + shapeId, + key: { kind: "strLit", value: method, type: STRING, loc }, + overflowOnly: true, + type: ownType, + loc, + }; + const tag = lowerer.armTag(ownType.unionId, want); + const ownLocal = lowerer.declareHiddenLocal("%ownMethod", ownType); + const own = varRef(ownLocal.id, ownType, loc); + stmts.push({ kind: "varDecl", localId: ownLocal.id, init: read, loc }); + result = { + kind: "ternary", + cond: { + kind: "unionIsTag", + unionId: ownType.unionId, + tag, + negated: false, + value: own, + type: BOOL, + loc, + }, + then: { kind: "unionNarrow", unionId: ownType.unionId, tag, value: own, type: want, loc }, + else_: builtin, + type: want, + loc, + }; + } + } + return { kind: "seqExpr", stmts, result, type: want, loc }; +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8360-record-prototype-method-calls.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8360-record-prototype-method-calls.ts.json new file mode 100644 index 000000000..2bbaaaa88 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8360-record-prototype-method-calls.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/8360-record-prototype-method-calls.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8361-record-function-table-inherited.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8361-record-function-table-inherited.ts.json new file mode 100644 index 000000000..a9d8bf4b0 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8361-record-function-table-inherited.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/8361-record-function-table-inherited.ts" + ], + "diags": [] +} diff --git a/tests/corpus/8360-record-prototype-method-calls.ts b/tests/corpus/8360-record-prototype-method-calls.ts new file mode 100644 index 000000000..980e4b5d2 --- /dev/null +++ b/tests/corpus/8360-record-prototype-method-calls.ts @@ -0,0 +1,56 @@ +// Object.prototype's methods called by name on typed records: the built-in +// with the record as its receiver, or an own entry that shadows it. +interface Point { + x: number; + y: number; +} +const counts: Record = { apple: 2, pear: 0 }; +const point: Point = { x: 1, y: 2 }; +const handlers: Record number> = { double: (n) => n * 2 }; +const own: Record string> = { + toString: () => "own toString", + valueOf: () => "own valueOf", +}; + +function line(...parts: unknown[]): void { + console.log(parts.map(String).join(" | ")); +} + +line("toString", counts.toString(), point.toString(), handlers.toString(), String(counts)); +line("toLocaleString", counts.toLocaleString(), point.toLocaleString()); +line("valueOf", counts.valueOf() === counts, point.valueOf() === point); +line( + "hasOwnProperty", + counts.hasOwnProperty("apple"), + counts.hasOwnProperty("pear"), + counts.hasOwnProperty("plum"), + counts.hasOwnProperty("toString"), + point.hasOwnProperty("x"), + point.hasOwnProperty("z"), + handlers.hasOwnProperty("double"), + handlers.hasOwnProperty("valueOf"), +); +line("propertyIsEnumerable", counts.propertyIsEnumerable("apple"), point.propertyIsEnumerable("toString")); +line("isPrototypeOf", counts.isPrototypeOf(point), point.isPrototypeOf(counts)); + +// Own entries shadow the built-ins; toLocaleString calls the own toString. +line("own", own.toString(), own.toLocaleString(), own.valueOf(), own.hasOwnProperty("toString")); +delete own["toString"]; +line("own deleted", own.toString(), own.toLocaleString()); +own.toString = () => "own again"; +line("own again", own.toString(), `${String(own.toString())}`); + +// Keys added at run time are own keys too. +const later: Record = {}; +for (const word of "toString valueOf plain toString".split(" ")) { + later[word] = (later.hasOwnProperty(word) ? later[word]! : 0) + 1; +} +line("later", JSON.stringify(later), later.hasOwnProperty("toString"), later.hasOwnProperty("constructor")); + +// A null-prototype record has no inherited methods. +const groups = Object.groupBy([1, 2, 3], (n): string => (n % 2 ? "odd" : "even")); +try { + line("groups", groups.toString()); +} catch (e) { + line("groups threw", (e as Error).name + ": " + (e as Error).message); +} diff --git a/tests/corpus/8361-record-function-table-inherited.ts b/tests/corpus/8361-record-function-table-inherited.ts new file mode 100644 index 000000000..eecf74f02 --- /dev/null +++ b/tests/corpus/8361-record-function-table-inherited.ts @@ -0,0 +1,40 @@ +// Records of functions used as dispatch tables meet Object.prototype's method +// names: a keyed read answers the inherited method, bound to the record when +// the read is called at once and a plain function value otherwise. +type Command = (arg: string) => unknown; +const commands: Record = { + echo: (arg) => arg, + shout: (arg) => arg.toUpperCase(), +}; +const inputs = process.argv.length > 99 ? [] : ["echo", "toString", "valueOf", "hasOwnProperty", "isPrototypeOf", "propertyIsEnumerable", "toLocaleString", "shout", "nope"]; +function show(v: unknown): string { + return v === commands ? "" : typeof v + ":" + String(v); +} +for (const name of inputs) { + const handler = commands[name]; + if (!handler) { + console.log(name, "unknown handler"); + continue; + } + console.log(name, typeof handler, "direct", show(commands[name](name === "hasOwnProperty" ? "echo" : "x"))); + console.log(name, "optional", show(commands[name]?.("shout"))); + try { + console.log(name, "detached", show(handler("echo"))); + } catch (e) { + console.log(name, "detached threw", (e as Error).name + ": " + (e as Error).message); + } +} +console.log(typeof commands["toString"], typeof commands["valueOf"], commands["toString"] === undefined, commands["toString"] === commands["toString"]); +console.log(String(commands["toString"]("x")), commands["hasOwnProperty"]("shout"), commands["hasOwnProperty"]("nope")); +const literal: Record = {}; +console.log(Object.keys(literal).length, literal["isPrototypeOf"]?.("x"), typeof literal.toString); + +// A word count whose words are not Object.prototype names keeps `?? 0`. +const freq: Record = {}; +for (const w of "a rose is a rose is a rose".split(" ")) freq[w] = (freq[w] ?? 0) + 1; +console.log(JSON.stringify(freq), freq["tulip"] ?? 0); + +// Read by name without a call: a plain function value. +const counts: Record = { a: 1 }; +const named = counts.toString; +console.log(typeof counts.toString, typeof commands.valueOf, String(named())); diff --git a/tests/harness/corpus-costs/8360-record-prototype-method-calls.ts.json b/tests/harness/corpus-costs/8360-record-prototype-method-calls.ts.json new file mode 100644 index 000000000..1b2d59a51 --- /dev/null +++ b/tests/harness/corpus-costs/8360-record-prototype-method-calls.ts.json @@ -0,0 +1 @@ +{ "plain": 3200, "san": 3900 } diff --git a/tests/harness/corpus-costs/8361-record-function-table-inherited.ts.json b/tests/harness/corpus-costs/8361-record-function-table-inherited.ts.json new file mode 100644 index 000000000..1b2d59a51 --- /dev/null +++ b/tests/harness/corpus-costs/8361-record-function-table-inherited.ts.json @@ -0,0 +1 @@ +{ "plain": 3200, "san": 3900 } From f4825e05a7ad5fc48d6bf188ba4d272ec6401d2a Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Sun, 25 Oct 2026 11:33:12 -0700 Subject: [PATCH 22/44] Replace a package's declaration files or supply its declarations with type patches A scriptc.typePatches entry can now replace a declaration file of the package (or of its @types twin, written @types//) with a local .d.ts ("files"), or give the package a declaration entry ("types"). A supplied entry answers the package's types file and the declarations TypeScript probes beside its JavaScript entries, and exists for the program host, the resolver and npm-static's untyped-package test, so a package without declarations becomes a typed package. Declaration patches still apply on top. A files target the installed copy does not ship is SC8031. Co-Authored-By: Claude Opus 5.5 --- docs/content/docs/dependencies.mdx | 19 +++ packages/compiler/src/frontend/npm-static.ts | 3 +- packages/compiler/src/frontend/resolve.ts | 4 +- .../compiler/src/frontend/ts7/program-host.ts | 4 +- .../src/frontend/type-patches/layer.ts | 157 +++++++++++++++--- packages/compiler/src/project-config.test.ts | 44 ++++- packages/compiler/src/project-config.ts | 68 +++++++- .../npm/cases/type-patch-files/main.ts.json | 25 +++ .../cases/type-patch-types-entry/main.ts.json | 16 ++ packages/compiler/test/type-patches.test.ts | 55 ++++++ .../node_modules/@effect/platform-node | 1 + tests/fixtures/effect4/node_modules/effect | 1 + .../npm/cases/type-patch-files/main.ts | 10 ++ .../node_modules/eventlog/dist/entry.d.ts | 8 + .../node_modules/eventlog/dist/index.d.ts | 4 + .../node_modules/eventlog/dist/index.js | 17 ++ .../node_modules/eventlog/package.json | 1 + .../npm/cases/type-patch-files/package.json | 13 ++ .../type-patch-files/type-patches/entry.d.ts | 7 + .../npm/cases/type-patch-types-entry/main.ts | 9 + .../node_modules/pointjs/lib/point.js | 17 ++ .../node_modules/pointjs/package.json | 1 + .../cases/type-patch-types-entry/package.json | 10 ++ .../type-patches/pointjs.d.ts | 4 + tests/harness/type-patches.test.ts | 48 +++++- 25 files changed, 513 insertions(+), 33 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/type-patch-files/main.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/type-patch-types-entry/main.ts.json create mode 120000 tests/fixtures/effect4/node_modules/@effect/platform-node create mode 120000 tests/fixtures/effect4/node_modules/effect create mode 100644 tests/fixtures/npm/cases/type-patch-files/main.ts create mode 100644 tests/fixtures/npm/cases/type-patch-files/node_modules/eventlog/dist/entry.d.ts create mode 100644 tests/fixtures/npm/cases/type-patch-files/node_modules/eventlog/dist/index.d.ts create mode 100644 tests/fixtures/npm/cases/type-patch-files/node_modules/eventlog/dist/index.js create mode 100644 tests/fixtures/npm/cases/type-patch-files/node_modules/eventlog/package.json create mode 100644 tests/fixtures/npm/cases/type-patch-files/package.json create mode 100644 tests/fixtures/npm/cases/type-patch-files/type-patches/entry.d.ts create mode 100644 tests/fixtures/npm/cases/type-patch-types-entry/main.ts create mode 100644 tests/fixtures/npm/cases/type-patch-types-entry/node_modules/pointjs/lib/point.js create mode 100644 tests/fixtures/npm/cases/type-patch-types-entry/node_modules/pointjs/package.json create mode 100644 tests/fixtures/npm/cases/type-patch-types-entry/package.json create mode 100644 tests/fixtures/npm/cases/type-patch-types-entry/type-patches/pointjs.d.ts diff --git a/docs/content/docs/dependencies.mdx b/docs/content/docs/dependencies.mdx index ec1eb0601..9b660c76f 100644 --- a/docs/content/docs/dependencies.mdx +++ b/docs/content/docs/dependencies.mdx @@ -172,6 +172,25 @@ Each key names a package and a semver range of the installed versions the patch A declaration is named by its path: `Options.name` for a member of a type, `parse()` for a function's return type, `parse(text)` for a parameter, `list()[]` for an array's elements, and `Env[string]` for an index signature. `export=()` names the return type of the function the module exports, `default` its default export, and a file prefix such as `index.js:parse()` names a declaration inside one of the package's files. One patch applies to the package's own declarations, its JSDoc, and its `@types/` package. The patched type then holds everywhere, so your code has to handle the `undefined` that a widened declaration allows. An optional `"note"` string can link the upstream issue. +For larger fixes, replace a whole declaration file, or give a package without declarations its own: + +```json:package.json +{ + "scriptc": { + "typePatches": { + "some-client@^3.2.0": { + "files": { "dist/types/options.d.ts": "./type-patches/options.d.ts" } + }, + "untyped-helper@1.x": { + "types": "./type-patches/untyped-helper.d.ts" + } + } + } +} +``` + +`files` maps a declaration file inside the package, or inside its `@types/` package written as `@types//`, to a local `.d.ts` file. `types` supplies the package's declaration entry. Paths of local files are relative to the `package.json`, and a replaced file can import the package's other declaration files with the same relative paths as the file it replaces. + Patches change only declarations, never the package's code, and values are still checked against the patched types. A build warns when a patch doesn't apply as written: when no range covers the installed version, when a declaration path matches nothing, when the declaration already has the type, or when a path is ambiguous. Report wrong declarations to the package too, so the patch can be removed later. ## Provenance sources (experimental) diff --git a/packages/compiler/src/frontend/npm-static.ts b/packages/compiler/src/frontend/npm-static.ts index ed442be79..8e26e3058 100644 --- a/packages/compiler/src/frontend/npm-static.ts +++ b/packages/compiler/src/frontend/npm-static.ts @@ -23,7 +23,7 @@ import { trackedRealpath, } from "./input-tracker.js"; import { npmBuiltinAliasOf } from "./builtin-modules.js"; -import { packageSourceText } from "./type-patches/layer.js"; +import { packageSourceText, typePatchSuppliesTypes } from "./type-patches/layer.js"; let activePackages: ReadonlySet = new Set(); let declarationOverloads: ReadonlyMap = new Map(); @@ -374,6 +374,7 @@ function packageIsUntyped(path: string): boolean { /* no package.json — keep probing */ } if (untyped && trackedExists(`${pkgDir}/index.d.ts`)) untyped = false; + if (untyped && typePatchSuppliesTypes(pkgDir)) untyped = false; if (untyped) { // the @types twin, hoisted anywhere up the realm chain const mangled = mangledTypesName(dirName); diff --git a/packages/compiler/src/frontend/resolve.ts b/packages/compiler/src/frontend/resolve.ts index 5b035b6cb..86252499f 100644 --- a/packages/compiler/src/frontend/resolve.ts +++ b/packages/compiler/src/frontend/resolve.ts @@ -37,9 +37,11 @@ import { trackedRealpath, } from "./input-tracker.js"; import { isRelativeSpecifier, packageNameOfSpecifier } from "./workspace-registry.js"; +import { typePatchFileExists } from "./type-patches/layer.js"; function isFile(path: string): boolean { - return trackedFileExists(path); + // A declaration entry a project type patch supplies (type-patches/layer.ts). + return trackedFileExists(path) || typePatchFileExists(path); } function isDirectory(path: string): boolean { diff --git a/packages/compiler/src/frontend/ts7/program-host.ts b/packages/compiler/src/frontend/ts7/program-host.ts index 9e8f51bb3..303092acf 100644 --- a/packages/compiler/src/frontend/ts7/program-host.ts +++ b/packages/compiler/src/frontend/ts7/program-host.ts @@ -46,7 +46,7 @@ import { trackedRealpath, } from "../input-tracker.js"; import { packageShimDeclarationText } from "../package-shims.js"; -import { typePatchedText } from "../type-patches/layer.js"; +import { typePatchedText, typePatchFileExists } from "../type-patches/layer.js"; import { ambientDtsPath, fallbackDtsPath, overridesDtsPath } from "../dts-paths.js"; import { reusedLibDiagnostics } from "./lib-diagnostics.js"; import { ts7Diagnostic } from "./diagnostic.js"; @@ -190,7 +190,7 @@ export class Ts7Host { fileExists: (fileName) => { if (virtualFiles.has(tsgoPath(fileName))) return true; if (shadow !== null && shadow.hideFile(fileName)) return false; - return trackedFileExists(fileName); + return trackedFileExists(fileName) || typePatchFileExists(fileName); }, directoryExists: (path) => trackedDirectoryExists(path), realpath: (path) => diff --git a/packages/compiler/src/frontend/type-patches/layer.ts b/packages/compiler/src/frontend/type-patches/layer.ts index 48ef6ecf0..5fb41f968 100644 --- a/packages/compiler/src/frontend/type-patches/layer.ts +++ b/packages/compiler/src/frontend/type-patches/layer.ts @@ -27,7 +27,7 @@ import { dirname, relative } from "node:path"; import type { ScrDiagnostic } from "../../diagnostics/diagnostic.js"; import { typePatchDiag } from "../../diagnostics/diagnostic.js"; import { readProjectConfig, type TypePatchEntry } from "../../project-config.js"; -import { trackedReadFile } from "../input-tracker.js"; +import { trackedFileExists, trackedReadFile } from "../input-tracker.js"; import type { SourceFile } from "../ts7/syntax.js"; import { applyPatchEdits, patchFile, type PatchMatch, type PatchRequest } from "./locate.js"; import { parseSelector, parseTypeText, type Selector, type TypePatchOp } from "./selector.js"; @@ -49,6 +49,11 @@ interface CopyInfo { root: string; /** Requests whose entry's range the version satisfies. */ requests: Request[]; + /** Replaced declaration files: package-relative path → local file. */ + files: Map; + /** Declaration entries a `types` patch supplies: package-relative path + * → local file (existing entries are replaced, missing ones appear). */ + entries: Map; } interface MatchRecord extends PatchMatch { @@ -61,9 +66,11 @@ interface State { manifest: string; manifestText: string; parse: TypePatchParse; - byPackage: Map; + byPackage: Map; copies: Map; files: Map; + /** `files` targets that do not exist in a copy. */ + missingFiles: { entry: TypePatchEntry; root: string; rel: string }[]; /** Package copies whose version no patch range covers. */ unmatchedCopies: Map; /** Requests applicable to a copy this load read files of. */ @@ -84,22 +91,23 @@ export function configureTypePatches(entryPath: string, parse: TypePatchParse): state = null; return; } - const byPackage = new Map(); + const byPackage = new Map(); let id = 0; for (const entry of config.typePatches) { + const pkg = byPackage.get(entry.pkg) ?? { entries: [], requests: [] }; + byPackage.set(entry.pkg, pkg); + pkg.entries.push(entry); for (const decl of entry.declarations) { const selector = parseSelector(decl.selector); const op = parseTypeText(decl.type); if ("error" in selector || "error" in op) continue; // reported as SC8004 - const list = byPackage.get(entry.pkg) ?? []; - list.push({ + pkg.requests.push({ id: id++, selector: selector as Selector, op: op as TypePatchOp, entry, offset: decl.offset, }); - byPackage.set(entry.pkg, list); } } state = { @@ -109,6 +117,7 @@ export function configureTypePatches(entryPath: string, parse: TypePatchParse): byPackage, copies: new Map(), files: new Map(), + missingFiles: [], unmatchedCopies: new Map(), touched: new Set(), matches: [], @@ -145,7 +154,7 @@ function versionAt(dir: string): string | null { /** The installed package copy a file belongs to, with the requests that * apply to it; null for files outside node_modules or of unpatched * packages. */ -function copyOf(s: State, path: string): { copy: CopyInfo; rel: string } | null { +function copyOf(s: State, path: string, touch = true): { copy: CopyInfo; rel: string } | null { const marker = "/node_modules/"; const at = path.lastIndexOf(marker); if (at < 0) return null; @@ -159,33 +168,120 @@ function copyOf(s: State, path: string): { copy: CopyInfo; rel: string } | null let copy = s.copies.get(root); if (copy === undefined) { copy = null; - const pkg = name.startsWith("@types/") ? unmangleTypesName(name) : name; - const requests = s.byPackage.get(pkg); - if (requests !== undefined) { + const twin = name.startsWith("@types/"); + const pkg = twin ? unmangleTypesName(name) : name; + const patches = s.byPackage.get(pkg); + if (patches !== undefined) { // A twin describes the runtime package installed beside it. - const version = name.startsWith("@types/") + const version = twin ? (versionAt(path.slice(0, at + marker.length) + pkg) ?? versionAt(root)) : versionAt(root); - const applicable = requests.filter( - (r) => version !== null && satisfies(version, r.entry.range), + const applicable = patches.entries.filter( + (e) => version !== null && satisfies(version, e.range), ); - // Later entries win on the same selector. - const bySelector = new Map(); - for (const r of applicable) bySelector.set(r.selector.text, r); - const chosen = [...bySelector.values()]; - if (chosen.length === 0) { - const entries = [...new Set(requests.map((r) => r.entry))]; - s.unmatchedCopies.set(root, { pkg, version, entries }); + if (applicable.length === 0) { + s.unmatchedCopies.set(root, { pkg, version, entries: patches.entries }); + } else { + // Later entries win on the same selector, file or entry. + const bySelector = new Map(); + for (const r of patches.requests) + if (applicable.includes(r.entry)) bySelector.set(r.selector.text, r); + const files = new Map(); + let types: string | null = null; + const prefix = `${name}/`; + for (const e of applicable) { + for (const [target, local] of e.files) { + // `@types//` names the twin's file; plain paths the + // package's own. + const forTwin = target.startsWith("@types/"); + if (forTwin !== twin) continue; + const rel = forTwin + ? target.startsWith(prefix) + ? target.slice(prefix.length) + : null + : target; + if (rel === null) continue; + if (!trackedFileExists(`${root}/${rel}`)) { + s.missingFiles.push({ entry: e, root, rel }); + continue; + } + files.set(rel, local); + } + if (e.types !== null && !twin) types = e.types; + } + copy = { + pkg, + version, + root, + requests: [...bySelector.values()], + files, + entries: types === null ? new Map() : declarationEntries(root, types), + }; } - copy = chosen.length === 0 ? null : { pkg, version, root, requests: chosen }; } s.copies.set(root, copy); } if (copy === null) return null; - for (const r of copy.requests) s.touched.add(r); + if (touch) for (const r of copy.requests) s.touched.add(r); return { copy, rel }; } +/** The declaration entry paths of a package a `types` patch stands for: + * its `types`/`typings` file and every declaration TypeScript probes + * beside a JavaScript entry (`main`, the `exports` targets of "."). */ +function declarationEntries(root: string, local: string): Map { + const out = new Map(); + let pkg: Record = {}; + try { + pkg = JSON.parse(trackedReadFile(`${root}/package.json`) ?? "{}") as Record; + } catch { + pkg = {}; + } + const add = (target: unknown): void => { + if (typeof target !== "string") return; + const rel = target.replace(/^\.\//, ""); + if (/\.d\.[cm]?ts$/.test(rel)) out.set(rel, local); + else if (/\.[cm]?js$/.test(rel)) out.set(rel.replace(/\.([cm]?)js$/, ".d.$1ts"), local); + else if (!/\.\w+$/.test(rel)) out.set(`${rel}.d.ts`, local); + }; + add(pkg["types"]); + add(pkg["typings"]); + add(typeof pkg["main"] === "string" ? pkg["main"] : "index.js"); + const exportsField = pkg["exports"]; + const dot = + exportsField !== null && typeof exportsField === "object" && "." in exportsField + ? (exportsField as Record)["."] + : exportsField; + const visit = (v: unknown): void => { + if (typeof v === "string") add(v); + else if (v !== null && typeof v === "object") for (const x of Object.values(v)) visit(x); + }; + // Subpath exports ("./x") are not the entry. + if (dot !== null && typeof dot === "object" && Object.keys(dot).some((k) => k.startsWith("."))) + return out; + visit(dot); + return out; +} + +/** True for a declaration entry a `types` patch supplies that does not + * exist on disk (a package without declarations gains one). */ +export function typePatchFileExists(fileName: string): boolean { + const s = state; + if (s === null) return false; + const path = fileName.split("\\").join("/"); + if (!/\.d\.[cm]?ts$/.test(path)) return false; + const located = copyOf(s, path, false); + return located !== null && located.copy.entries.has(located.rel); +} + +/** True when a `types` patch gives the package at `pkgDir` declarations. */ +export function typePatchSuppliesTypes(pkgDir: string): boolean { + const s = state; + if (s === null) return false; + const located = copyOf(s, `${pkgDir.split("\\").join("/")}/package.json`, false); + return located !== null && located.copy.entries.size > 0; +} + /** Names a request's root can appear under in a file's text. */ function rootWords(sel: Selector): string[] { if (sel.root === "export=") return ["exports", "export"]; @@ -205,7 +301,12 @@ export function typePatchedText(fileName: string): string | undefined { let result: string | null = null; const located = copyOf(s, path); if (located !== null) { - const text = trackedReadFile(path); + // A replaced file or a supplied entry reads as the local file; the + // declaration patches then apply to it like to the package's own. + const local = located.copy.files.get(located.rel) ?? located.copy.entries.get(located.rel); + const replaced = local === undefined ? null : trackedReadFile(local); + const text = replaced ?? trackedReadFile(path); + if (replaced !== null) result = replaced; const requests = located.copy.requests.filter( (r) => (r.selector.file === null || r.selector.file === located.rel) && @@ -314,6 +415,16 @@ export function typePatchDiagnostics(): { ); } } + for (const missing of s.missingFiles) { + out.push( + typePatchDiag( + "SC8031", + `type patch '${missing.entry.key}': ${missing.rel} is not a file of ${missing.entry.pkg} at ${relativeToManifest(s, missing.root)}`, + "name a declaration file the installed package ships; the package may have moved it", + loc(s, missing.entry.offset), + ), + ); + } for (const r of s.touched) { const mine = s.matches.filter((m) => m.id === r.id); // A selector that matched somewhere (the .d.ts) is not a problem where diff --git a/packages/compiler/src/project-config.test.ts b/packages/compiler/src/project-config.test.ts index 04a5d2e0b..51d9520a0 100644 --- a/packages/compiler/src/project-config.test.ts +++ b/packages/compiler/src/project-config.test.ts @@ -214,7 +214,7 @@ test("typePatches: malformed keys, ranges, selectors and types are SC8004", () = "A.c": "string\nnumber", "A.d": "| undefined", }, - files: {}, + patches: {}, }, "other@^1.0.0": [], }, @@ -225,7 +225,7 @@ test("typePatches: malformed keys, ranges, selectors and types are SC8004", () = `package.json 'scriptc.typePatches.ms@^banana' has the version range "^banana", which is not a semver range`, "package.json 'scriptc.typePatches.@types/node@*' patches @types/node, whose declarations ship with scriptc or TypeScript", "package.json 'scriptc.typePatches.typescript@5' patches typescript, whose declarations ship with scriptc or TypeScript", - "package.json 'scriptc.typePatches.lib@^1.0.0.files' is not a known type patch setting", + "package.json 'scriptc.typePatches.lib@^1.0.0.patches' is not a known type patch setting", "package.json 'scriptc.typePatches.lib@^1.0.0.declarations.1bad' is not a selector: it must start with a declared name, 'export=' or 'default'", "package.json 'scriptc.typePatches.lib@^1.0.0.declarations.A.b' must map to a type string, not 3", "package.json 'scriptc.typePatches.lib@^1.0.0.declarations.A.c' has a type that must be one line", @@ -241,3 +241,43 @@ test("typePatches: malformed keys, ranges, selectors and types are SC8004", () = }); expect(partial.typePatches[0]!.declarations.map((d) => d.selector)).toEqual(["A.d"]); }); + +test("typePatches: files and types name local declaration files", () => { + const root = mkdtempSync(join(tmpdir(), "scriptc-project-config-")); + try { + mkdirSync(join(root, "patches")); + writeFileSync(join(root, "patches", "entry.d.ts"), "export {};\n"); + const text = JSON.stringify({ + scriptc: { + typePatches: { + "lib@^1.0.0": { + files: { + "dist/entry.d.ts": "./patches/entry.d.ts", + "@types/lib/index.d.ts": "patches/entry.d.ts", + "dist/index.js": "./patches/entry.d.ts", + "../outside.d.ts": "./patches/entry.d.ts", + "dist/other.d.ts": "./patches/missing.d.ts", + }, + types: "./patches/entry.d.ts", + }, + "other@^1.0.0": { types: "./patches/entry.ts" }, + }, + }, + }); + const c = parseProjectConfig(join(root, "package.json"), text); + expect(c.diagnostics.map((d) => d.message)).toEqual([ + "package.json 'scriptc.typePatches.lib@^1.0.0.files.dist/index.js' must name a declaration file inside the package", + "package.json 'scriptc.typePatches.lib@^1.0.0.files.../outside.d.ts' must name a declaration file inside the package", + `package.json 'scriptc.typePatches.lib@^1.0.0.files.dist/other.d.ts' names "./patches/missing.d.ts", which does not exist`, + `package.json 'scriptc.typePatches.other@^1.0.0.types' names "./patches/entry.ts", which is not a declaration file`, + ]); + const entry = c.typePatches.find((e) => e.pkg === "lib")!; + expect([...entry.files]).toEqual([ + ["dist/entry.d.ts", join(root, "patches/entry.d.ts")], + ["@types/lib/index.d.ts", join(root, "patches/entry.d.ts")], + ]); + expect(entry.types).toBe(join(root, "patches/entry.d.ts")); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); diff --git a/packages/compiler/src/project-config.ts b/packages/compiler/src/project-config.ts index c43a6bd2c..4aad9cbc3 100644 --- a/packages/compiler/src/project-config.ts +++ b/packages/compiler/src/project-config.ts @@ -27,7 +27,7 @@ * to their own readers. */ import { dirname, join, resolve } from "node:path"; import { projectConfigDiag, type ScrDiagnostic } from "./diagnostics/diagnostic.js"; -import { trackedReadFile } from "./frontend/input-tracker.js"; +import { trackedFileExists, trackedReadFile } from "./frontend/input-tracker.js"; import { parseSelector, parseTypeText } from "./frontend/type-patches/selector.js"; import { isValidRange } from "./frontend/type-patches/semver.js"; @@ -79,6 +79,13 @@ export interface TypePatchEntry { pkg: string; range: string; declarations: TypePatchDeclaration[]; + /** `files`: declaration files of the package (or of its `@types/` twin, + * written `@types//`) replaced by local files, by + * package-relative path. The local paths are absolute. */ + files: Map; + /** `types`: a local declaration file that becomes the package's + * declaration entry (absolute), or null. */ + types: string | null; /** Offset of the key in the manifest. */ offset: number; } @@ -341,7 +348,8 @@ export function parseProjectConfig(manifest: string, text: string): ProjectConfi return config; } -const TYPE_PATCH_KEYS = ["declarations", "note"]; +const TYPE_PATCH_KEYS = ["declarations", "files", "types", "note"]; +const DECLARATION_FILE = /\.d\.[cm]?ts$/; /** Packages whose declarations ship with scriptc or TypeScript: a wrong * one is a bug to report, not a project-local patch. */ const UNPATCHABLE = /^(?:@types\/node|typescript|@typescript\/.*)$/; @@ -404,8 +412,64 @@ function readTypePatches( pkg, range, declarations: [], + files: new Map(), + types: null, offset: locate.offsetOf(["scriptc", "typePatches", key]), }; + const base = dirname(config.manifest!); + const localFile = (path: string[], v: unknown): string | null => { + if (typeof v !== "string" || v === "") { + problem( + path, + `must be the path of a local declaration file, not ${shown(v)}`, + `write "./type-patches/${pkg}.d.ts"`, + ); + return null; + } + const abs = resolve(base, v); + if (!DECLARATION_FILE.test(abs)) { + problem( + path, + `names "${v}", which is not a declaration file`, + "use a .d.ts, .d.mts or .d.cts file", + ); + return null; + } + if (!trackedFileExists(abs)) { + problem( + path, + `names "${v}", which does not exist`, + "give the path relative to this package.json", + ); + return null; + } + return abs; + }; + const files = value["files"]; + if (files !== undefined && !isRecord(files)) { + problem( + ["typePatches", key, "files"], + `must be an object mapping declaration files of the package to local files, not ${shown(files)}`, + `write "files": { "dist/index.d.ts": "./type-patches/${pkg}.d.ts" }`, + ); + } else if (files !== undefined) { + for (const target of Object.keys(files)) { + const path = ["typePatches", key, "files", target]; + const rel = target.replace(/^\.\//, ""); + if (!DECLARATION_FILE.test(rel) || rel.startsWith("/") || rel.split("/").includes("..")) { + problem( + path, + "must name a declaration file inside the package", + `write a package-relative .d.ts path such as "dist/index.d.ts", or "@types//index.d.ts" for its @types package`, + ); + continue; + } + const abs = localFile(path, files[target]); + if (abs !== null) entry.files.set(rel, abs); + } + } + if (value["types"] !== undefined) + entry.types = localFile(["typePatches", key, "types"], value["types"]); let valid = true; for (const k of Object.keys(value)) { if (!TYPE_PATCH_KEYS.includes(k)) { diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/type-patch-files/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/type-patch-files/main.ts.json new file mode 100644 index 000000000..4a15a1975 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/type-patch-files/main.ts.json @@ -0,0 +1,25 @@ +{ + "order": [ + "/tests/fixtures/npm/cases/type-patch-files/main.ts" + ], + "diags": [ + { + "code": "SC0001", + "message": "Property 'toUpperCase' does not exist on type 'number'.", + "loc": { + "file": "/tests/fixtures/npm/cases/type-patch-files/main.ts", + "start": 346, + "end": 357 + } + }, + { + "code": "SC0001", + "message": "This comparison appears to be unintentional because the types 'number' and 'string' have no overlap.", + "loc": { + "file": "/tests/fixtures/npm/cases/type-patch-files/main.ts", + "start": 429, + "end": 447 + } + } + ] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/type-patch-types-entry/main.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/type-patch-types-entry/main.ts.json new file mode 100644 index 000000000..bc1622724 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/fixtures/npm/cases/type-patch-types-entry/main.ts.json @@ -0,0 +1,16 @@ +{ + "order": [ + "/tests/fixtures/npm/cases/type-patch-types-entry/main.ts" + ], + "diags": [ + { + "code": "SC0001", + "message": "Could not find a declaration file for module 'pointjs'. '/tests/fixtures/npm/cases/type-patch-types-entry/node_modules/pointjs/lib/point.js' implicitly has an 'any' type.\n Try `npm i --save-dev @types/pointjs` if it exists or add a new declaration (.d.ts) file containing `declare module 'pointjs';`", + "loc": { + "file": "/tests/fixtures/npm/cases/type-patch-types-entry/main.ts", + "start": 246, + "end": 255 + } + } + ] +} diff --git a/packages/compiler/test/type-patches.test.ts b/packages/compiler/test/type-patches.test.ts index 729a0da37..116715db5 100644 --- a/packages/compiler/test/type-patches.test.ts +++ b/packages/compiler/test/type-patches.test.ts @@ -12,7 +12,9 @@ import { typePatchDiagnostics, typePatchedSpans, typePatchedText, + typePatchFileExists, typePatchMatches, + typePatchSuppliesTypes, } from "../src/frontend/type-patches/layer.js"; const fixtures = join(import.meta.dirname, "../../../tests/fixtures"); @@ -122,3 +124,56 @@ test("an @types twin is matched by the runtime package's installed version", () rmSync(root, { recursive: true, force: true }); } }); + +test("files replaces a declaration file and declarations still apply on top", () => { + const root = join(fixtures, "npm/cases/type-patch-files"); + configureTypePatches(join(root, "main.ts"), parse); + const entry = typePatchedText(join(root, "node_modules/eventlog/dist/entry.d.ts")); + expect(entry).toBe(readFileSync(join(root, "type-patches/entry.d.ts"), "utf8")); + expect(typePatchedText(join(root, "node_modules/eventlog/dist/index.d.ts"))).toContain( + "{ count: number; last: string | undefined }", + ); + expect(typePatchDiagnostics()!.diagnostics).toEqual([]); +}); + +test("types gives a package without declarations an entry beside its main", () => { + const root = join(fixtures, "npm/cases/type-patch-types-entry"); + configureTypePatches(join(root, "main.ts"), parse); + const pkg = join(root, "node_modules/pointjs"); + expect(typePatchFileExists(join(pkg, "lib/point.d.ts"))).toBe(true); + expect(typePatchFileExists(join(pkg, "index.d.ts"))).toBe(false); + expect(typePatchFileExists(join(pkg, "lib/other.d.ts"))).toBe(false); + expect(typePatchSuppliesTypes(pkg)).toBe(true); + expect(typePatchedText(join(pkg, "lib/point.d.ts"))).toBe( + readFileSync(join(root, "type-patches/pointjs.d.ts"), "utf8"), + ); + // Without patches nothing exists that is not on disk. + configureTypePatches(join(tmpdir(), "none", "main.ts"), parse); + expect(typePatchFileExists(join(pkg, "lib/point.d.ts"))).toBe(false); + expect(typePatchSuppliesTypes(pkg)).toBe(false); +}); + +test("a files target the installed package does not ship warns", () => { + const root = mkdtempSync(join(tmpdir(), "scriptc-type-patches-")); + try { + const twin = join(fixtures, "npm/cases/type-patch-files"); + writeFileSync(join(root, "local.d.ts"), "export type Entry = {};\n"); + writeFileSync( + join(root, "package.json"), + JSON.stringify({ + scriptc: { + typePatches: { "eventlog@^2.0.0": { files: { "dist/gone.d.ts": "./local.d.ts" } } }, + }, + }), + ); + configureTypePatches(join(root, "main.ts"), parse); + expect(typePatchedText(join(twin, "node_modules/eventlog/dist/index.d.ts"))).toBeUndefined(); + const diags = typePatchDiagnostics()!.diagnostics; + expect(diags.map((d) => d.code)).toEqual(["SC8031"]); + expect(diags[0]!.message).toMatch( + /^type patch 'eventlog@\^2\.0\.0': dist\/gone\.d\.ts is not a file of eventlog at .*type-patch-files\/node_modules\/eventlog$/, + ); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); diff --git a/tests/fixtures/effect4/node_modules/@effect/platform-node b/tests/fixtures/effect4/node_modules/@effect/platform-node new file mode 120000 index 000000000..aaa396705 --- /dev/null +++ b/tests/fixtures/effect4/node_modules/@effect/platform-node @@ -0,0 +1 @@ +../../../../../node_modules/.pnpm/@effect+platform-node@4.0.0_effect@4.0.0_redis@6.3.0/node_modules/@effect/platform-node \ No newline at end of file diff --git a/tests/fixtures/effect4/node_modules/effect b/tests/fixtures/effect4/node_modules/effect new file mode 120000 index 000000000..1b933cdef --- /dev/null +++ b/tests/fixtures/effect4/node_modules/effect @@ -0,0 +1 @@ +../../../../node_modules/.pnpm/effect@4.0.0/node_modules/effect \ No newline at end of file diff --git a/tests/fixtures/npm/cases/type-patch-files/main.ts b/tests/fixtures/npm/cases/type-patch-files/main.ts new file mode 100644 index 000000000..32e041ce0 --- /dev/null +++ b/tests/fixtures/npm/cases/type-patch-files/main.ts @@ -0,0 +1,10 @@ +// A package declaration file replaced by a local one (scriptc.typePatches +// files), plus a declaration patch on another file of the same package. +import { parseLog, summarize } from "eventlog"; + +const entries = parseLog("10 info ada started\n12 warn - disk low\n15 error lin crashed hard\n"); +for (const e of entries) console.log(e.at, e.level.toUpperCase(), e.user ?? "(system)", e.message); +console.log(entries.filter((e) => e.level !== "info").length, JSON.stringify(entries[1])); +const all = summarize(entries); +const none = summarize([]); +console.log(all.count, all.last ?? "-", none.count, none.last ?? "-"); diff --git a/tests/fixtures/npm/cases/type-patch-files/node_modules/eventlog/dist/entry.d.ts b/tests/fixtures/npm/cases/type-patch-files/node_modules/eventlog/dist/entry.d.ts new file mode 100644 index 000000000..0a42b298f --- /dev/null +++ b/tests/fixtures/npm/cases/type-patch-files/node_modules/eventlog/dist/entry.d.ts @@ -0,0 +1,8 @@ +// Out of date: the JavaScript reports levels as strings and has no +// `user` for system entries. +export type Entry = { + at: number; + level: number; + user: string; + message: string; +}; diff --git a/tests/fixtures/npm/cases/type-patch-files/node_modules/eventlog/dist/index.d.ts b/tests/fixtures/npm/cases/type-patch-files/node_modules/eventlog/dist/index.d.ts new file mode 100644 index 000000000..c16d59d46 --- /dev/null +++ b/tests/fixtures/npm/cases/type-patch-files/node_modules/eventlog/dist/index.d.ts @@ -0,0 +1,4 @@ +export type { Entry } from "./entry"; +import type { Entry } from "./entry"; +export declare function parseLog(text: string): Entry[]; +export declare function summarize(entries: Entry[]): { count: number; last: string }; diff --git a/tests/fixtures/npm/cases/type-patch-files/node_modules/eventlog/dist/index.js b/tests/fixtures/npm/cases/type-patch-files/node_modules/eventlog/dist/index.js new file mode 100644 index 000000000..4c10d88f4 --- /dev/null +++ b/tests/fixtures/npm/cases/type-patch-files/node_modules/eventlog/dist/index.js @@ -0,0 +1,17 @@ +"use strict"; +function parseLog(text) { + const rows = String(text) + .split("\n") + .filter(Boolean) + .map((line) => { + const [at, level, user, ...rest] = line.split(" "); + return { at: Number(at), level: level, user: user === "-" ? undefined : user, message: rest.join(" ") }; + }); + // Entries travel as JSON (the log format's own wire form). + return JSON.parse(JSON.stringify(rows)); +} +function summarize(entries) { + return { count: entries.length, last: entries.length === 0 ? undefined : entries[entries.length - 1].message }; +} +exports.parseLog = parseLog; +exports.summarize = summarize; diff --git a/tests/fixtures/npm/cases/type-patch-files/node_modules/eventlog/package.json b/tests/fixtures/npm/cases/type-patch-files/node_modules/eventlog/package.json new file mode 100644 index 000000000..27ebf4625 --- /dev/null +++ b/tests/fixtures/npm/cases/type-patch-files/node_modules/eventlog/package.json @@ -0,0 +1 @@ +{ "name": "eventlog", "version": "2.3.1", "main": "dist/index.js", "types": "dist/index.d.ts" } diff --git a/tests/fixtures/npm/cases/type-patch-files/package.json b/tests/fixtures/npm/cases/type-patch-files/package.json new file mode 100644 index 000000000..92a38c108 --- /dev/null +++ b/tests/fixtures/npm/cases/type-patch-files/package.json @@ -0,0 +1,13 @@ +{ + "name": "type-patch-files-case", + "private": true, + "type": "module", + "scriptc": { + "typePatches": { + "eventlog@^2.0.0": { + "files": { "dist/entry.d.ts": "./type-patches/entry.d.ts" }, + "declarations": { "summarize().last": "| undefined" } + } + } + } +} diff --git a/tests/fixtures/npm/cases/type-patch-files/type-patches/entry.d.ts b/tests/fixtures/npm/cases/type-patch-files/type-patches/entry.d.ts new file mode 100644 index 000000000..380338e96 --- /dev/null +++ b/tests/fixtures/npm/cases/type-patch-files/type-patches/entry.d.ts @@ -0,0 +1,7 @@ +// The project's replacement for eventlog's dist/entry.d.ts. +export type Entry = { + at: number; + level: "info" | "warn" | "error"; + user?: string; + message: string; +}; diff --git a/tests/fixtures/npm/cases/type-patch-types-entry/main.ts b/tests/fixtures/npm/cases/type-patch-types-entry/main.ts new file mode 100644 index 000000000..3d0ab5ac9 --- /dev/null +++ b/tests/fixtures/npm/cases/type-patch-types-entry/main.ts @@ -0,0 +1,9 @@ +// A package without declarations, typed by a local declaration file +// (scriptc.typePatches types): user code is checked against it, and the +// values cross the declaration boundary as native records. +import { centroid, point, type Point } from "pointjs"; + +const points: Point[] = [point(0, 0), point(4, 0), point(4, 3)]; +const c = centroid(points); +console.log(c === null ? "none" : `${c.x.toFixed(2)},${c.y.toFixed(2)}`); +console.log(centroid([]) === null, JSON.stringify(points.map((p) => p.x + p.y))); diff --git a/tests/fixtures/npm/cases/type-patch-types-entry/node_modules/pointjs/lib/point.js b/tests/fixtures/npm/cases/type-patch-types-entry/node_modules/pointjs/lib/point.js new file mode 100644 index 000000000..f324e9acb --- /dev/null +++ b/tests/fixtures/npm/cases/type-patch-types-entry/node_modules/pointjs/lib/point.js @@ -0,0 +1,17 @@ +"use strict"; +// Plain JavaScript without declarations. +function point(x, y) { + return { x: x, y: y }; +} +function centroid(points) { + if (points.length === 0) return null; + var sx = 0; + var sy = 0; + for (var i = 0; i < points.length; i++) { + sx += points[i].x; + sy += points[i].y; + } + return point(sx / points.length, sy / points.length); +} +exports.point = point; +exports.centroid = centroid; diff --git a/tests/fixtures/npm/cases/type-patch-types-entry/node_modules/pointjs/package.json b/tests/fixtures/npm/cases/type-patch-types-entry/node_modules/pointjs/package.json new file mode 100644 index 000000000..917576875 --- /dev/null +++ b/tests/fixtures/npm/cases/type-patch-types-entry/node_modules/pointjs/package.json @@ -0,0 +1 @@ +{ "name": "pointjs", "version": "1.4.0", "main": "lib/point.js" } diff --git a/tests/fixtures/npm/cases/type-patch-types-entry/package.json b/tests/fixtures/npm/cases/type-patch-types-entry/package.json new file mode 100644 index 000000000..06846a01d --- /dev/null +++ b/tests/fixtures/npm/cases/type-patch-types-entry/package.json @@ -0,0 +1,10 @@ +{ + "name": "type-patch-types-entry-case", + "private": true, + "type": "module", + "scriptc": { + "typePatches": { + "pointjs@1.x": { "types": "./type-patches/pointjs.d.ts" } + } + } +} diff --git a/tests/fixtures/npm/cases/type-patch-types-entry/type-patches/pointjs.d.ts b/tests/fixtures/npm/cases/type-patch-types-entry/type-patches/pointjs.d.ts new file mode 100644 index 000000000..30ae94a5e --- /dev/null +++ b/tests/fixtures/npm/cases/type-patch-types-entry/type-patches/pointjs.d.ts @@ -0,0 +1,4 @@ +// The project's declarations for pointjs, which ships none. +export type Point = { x: number; y: number }; +export declare function point(x: number, y: number): Point; +export declare function centroid(points: Point[]): Point | null; diff --git a/tests/harness/type-patches.test.ts b/tests/harness/type-patches.test.ts index ecd8ee923..816418a96 100644 --- a/tests/harness/type-patches.test.ts +++ b/tests/harness/type-patches.test.ts @@ -4,8 +4,10 @@ * value (the shape of commander's HelpContext.helpWidth), and ms's @return * tags, compile natively and byte-match Node once patched; without the * patches the same declarations throw where Node runs. Patches that do not - * apply as written warn (SC8030-SC8033). The `.d.ts` and `@types` cases - * run in npm.test.ts (type-patch-dts, type-patch-types-twin), and + * apply as written warn (SC8030-SC8033). The `.d.ts`, `@types`, replaced + * file and supplied entry cases run in npm.test.ts (type-patch-dts, + * type-patch-types-twin, type-patch-files, type-patch-types-entry); here + * their unpatched copies show the patches are what makes them compile. And * commander-calc carries the HelpContext.helpWidth patch. */ import { execFile } from "node:child_process"; import { cpSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; @@ -121,6 +123,48 @@ describe("type patches", () => { }, ); + test( + "without its patch, code using the out-of-date declaration file does not type-check", + { timeout: 600_000 }, + async () => { + const copy = unpatched(join(repoRoot, "tests/fixtures/npm/cases/type-patch-files")); + try { + mkdirSync(outDir, { recursive: true }); + const result = await compile(join(copy, "main.ts"), { + outPath: join(outDir, "files-unpatched"), + outDir, + sanitize, + }); + expect(result.ok).toBe(false); + if (!result.ok) + expect(result.diagnostics.map((d) => d.message)).toContain( + "Property 'toUpperCase' does not exist on type 'number'.", + ); + } finally { + rmSync(copy, { recursive: true, force: true }); + } + }, + ); + + test( + "without its patch, a package without declarations is not compiled for a typed import", + { timeout: 600_000 }, + async () => { + const copy = unpatched(join(repoRoot, "tests/fixtures/npm/cases/type-patch-types-entry")); + try { + mkdirSync(outDir, { recursive: true }); + const result = await compile(join(copy, "main.ts"), { + outPath: join(outDir, "types-entry-unpatched"), + outDir, + sanitize, + }); + expect(result.ok).toBe(false); + } finally { + rmSync(copy, { recursive: true, force: true }); + } + }, + ); + test( "patches that do not apply as written warn and the build goes on", { timeout: 600_000 }, From be0e13aa1ef057c04ab397aa801a00b7577a1378 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Sun, 25 Oct 2026 14:28:23 -0500 Subject: [PATCH 23/44] Run paired row sums two columns per step A paired row-sum nest now also takes two columns per step: each row adds the first column's term and then the second's, and an odd column count runs its last column on its own, so every sum keeps its order of additions. The nest runs paired when all four bounds are integers in [0, 2^25 - 1]; it reads them masked to that range, which proves both counters' ranges, so its counted loops need no run-time guard or second copy of the body. Corpus 8355 covers the column counts and bounds, and corpus 8356 covers worker loops over shared views that run past a view's end or write overlapping views of one buffer. --- packages/compiler/src/ir/row-pairs.test.ts | 37 +++- packages/compiler/src/ir/row-pairs.ts | 200 +++++++++++++----- .../corpus/8355-row-pair-columns.ts.json | 6 + .../8356-shared-view-loop-versions.ts.json | 6 + tests/corpus/8355-row-pair-columns.ts | 73 +++++++ .../corpus/8356-shared-view-loop-versions.ts | 93 ++++++++ 6 files changed, 352 insertions(+), 63 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8355-row-pair-columns.ts.json create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8356-shared-view-loop-versions.ts.json create mode 100644 tests/corpus/8355-row-pair-columns.ts create mode 100644 tests/corpus/8356-shared-view-loop-versions.ts diff --git a/packages/compiler/src/ir/row-pairs.test.ts b/packages/compiler/src/ir/row-pairs.test.ts index f9db5b89d..8ef01c665 100644 --- a/packages/compiler/src/ir/row-pairs.test.ts +++ b/packages/compiler/src/ir/row-pairs.test.ts @@ -93,12 +93,39 @@ test("a row-sum nest runs two rows per pass behind its bounds and overlap test", // from the array the term reads. expect(JSON.stringify(go.init)).toContain('"fn":"bytes.disjoint"'); expect(branch.else_).toBeNull(); - const pairs = branch.then[0] as IrStmt & { kind: "for" }; + // The four bounds, masked to the range the test proved, then the pairs. + expect(branch.then.slice(0, 4)).toMatchObject( + Array(4).fill({ kind: "varDecl", init: { op: "&", right: { value: 2 ** 25 - 1 } } }), + ); + const pairs = branch.then[4] as IrStmt & { kind: "for" }; expect(pairs.body.filter((s) => s.kind === "bytesSet")).toHaveLength(2); - const inner = pairs.body[2] as IrStmt & { kind: "for" }; - // One read of u[j] for both rows, then the two sums. - expect(inner.body).toHaveLength(3); + const [, , inner, last] = pairs.body as [ + IrStmt, + IrStmt, + IrStmt & { kind: "for" }, + IrStmt & { kind: "if" }, + ]; + const column = inner.init as IrStmt & { kind: "varDecl" }; + // Two columns per step: for each, one read of u[j] for both rows, then + // the two sums; the second column reads `column + 1`. + expect(inner.body).toHaveLength(6); expect(inner.body[0]).toMatchObject({ kind: "varDecl", init: { kind: "bytesIntrinsic" } }); + expect(inner.body[3]).toMatchObject({ + kind: "varDecl", + init: { + kind: "bytesIntrinsic", + args: [{ op: "+", left: { localId: column.localId }, right: { value: 1 } }], + }, + }); + expect(inner.cond).toMatchObject({ op: "<", right: { op: "-", right: { value: 1 } } }); + expect(inner.update).toMatchObject({ value: { op: "+", right: { value: 2 } } }); + // The last column of an odd count, `to - 1`. + expect(last.then).toHaveLength(3); + expect(last.then[0]).toMatchObject({ + kind: "varDecl", + init: { args: [{ op: "-", right: { value: 1 } }] }, + }); + expect(last.else_).toBeNull(); // The second row's term reads `row + 1`. const row = pairs.init?.kind === "varDecl" ? pairs.init.localId : ""; expect((inner.body[2] as IrStmt & { kind: "assign" }).value).toMatchObject({ @@ -130,7 +157,7 @@ test("a row-sum nest runs two rows per pass behind its bounds and overlap test", expect(top.body).toHaveLength(4); expect( result.functions[0]!.locals.map((l) => l.id).filter((id) => id.startsWith("%pair.")), - ).toHaveLength(7); + ).toHaveLength(13); }); test("nests whose term calls, writes or reads the sum keep their loops", () => { diff --git a/packages/compiler/src/ir/row-pairs.ts b/packages/compiler/src/ir/row-pairs.ts index 2e189f42e..e6b3889a5 100644 --- a/packages/compiler/src/ir/row-pairs.ts +++ b/packages/compiler/src/ir/row-pairs.ts @@ -8,10 +8,17 @@ import { type IrStmt, } from "./ir.js"; import { everyExpr, everyExprChild, mapExprChildren, transformStmtList } from "./traverse.js"; +import { PRODUCT_COUNTER_BOUND } from "./integer-loops.js"; /** Inner-loop expressions above this many nodes keep the loop as written. */ const MAX_TERM_NODES = 64; +/** The largest bound a paired nest runs with. Counters below + * PRODUCT_COUNTER_BOUND keep products of their sums exact, so the nest's + * counted loops take integer counters without a run-time guard (and its + * copy of the body). */ +const SMALL_BOUND = PRODUCT_COUNTER_BOUND - 1; + /** The number operators a term may use: they run no other code and cannot * throw. */ const PURE_BINARY = new Set([ @@ -210,7 +217,8 @@ const bin = ( }); /** Runs the row sums of `match` two rows per pass when the test holds and - * as written otherwise. */ + * as written otherwise. Each pass takes two columns per step, then the + * last column when their count is odd. */ function pairRows( s: IrStmt & { kind: "for" }, match: RowSum, @@ -225,42 +233,93 @@ function pairRows( sum = fresh("sum"), next = fresh("next"), column = fresh("column"); - const atColumn = (r: IrExpr & { kind: "varRef" }): IrExpr => - r.localId === match.column ? { ...r, localId: column } : { ...r }; // Element reads that do not depend on the row are read once per column // and shared by both rows. - const shared = new Map(); - const elements: IrStmt[] = []; + const sharedReads: IrExpr[] = []; const collect = (e: IrExpr): boolean => { if (e.kind === "bytesIntrinsic" && e.method === "get" && !readsLocal(e, match.row)) { - const id = fresh("element"); - shared.set(e, id); - elements.push({ kind: "varDecl", localId: id, init: copy(e, atColumn), loc }); + sharedReads.push(e); return true; } return everyExprChild(e, collect, () => true); }; collect(match.term); - // The second row's term reads `row + 1`, so its counter sums stay visible - // to the counted-loop analysis. - const termAt = (rowExpr: () => IrExpr): IrExpr => - copy( - match.term, - (r) => (r.localId === match.row ? rowExpr() : atColumn(r)), - (node) => { - const id = shared.get(node); - return id === undefined ? null : { kind: "varRef", localId: id, type: node.type, loc }; + /** Both rows' additions at the column `columnAt()`: its shared element + * reads, then each row's sum. The second row's term reads `row + 1`, and + * the second column's `column + 1`, so their counter sums stay visible to + * the counted-loop analysis. */ + const step = (columnAt: () => IrExpr): IrStmt[] => { + const at = (r: IrExpr & { kind: "varRef" }): IrExpr => + r.localId === match.column ? columnAt() : { ...r }; + const shared = new Map(); + const elements: IrStmt[] = sharedReads.map((read) => { + const id = fresh("element"); + shared.set(read, id); + return { kind: "varDecl", localId: id, init: copy(read, at), loc }; + }); + const termAt = (rowExpr: () => IrExpr): IrExpr => + copy( + match.term, + (r) => (r.localId === match.row ? rowExpr() : at(r)), + (node) => { + const id = shared.get(node); + return id === undefined ? null : { kind: "varRef", localId: id, type: node.type, loc }; + }, + ); + return [ + ...elements, + { + kind: "assign", + localId: sum, + value: bin( + "+", + ref(sum), + termAt(() => ref(row)), + loc, + ), + loc, }, - ); + { + kind: "assign", + localId: next, + value: bin( + "+", + ref(next), + termAt(() => bin("+", ref(row), num(1), loc)), + loc, + ), + loc, + }, + ]; + }; const innerCond = match.inner.cond as IrExpr & { kind: "bin" }; + const columnStart = (match.inner.init as IrStmt & { kind: "varDecl" }).init!; + // The bounds, as the paired nest reads them (see `small` below). + const low = fresh("low"), + high = fresh("high"), + first = fresh("first"), + last = fresh("last"); + const bounds: IrStmt[] = ( + [ + [low, match.start], + [high, match.end], + [first, columnStart], + [last, innerCond.right], + ] as const + ).map(([id, bound]) => ({ + kind: "varDecl", + localId: id, + init: bin("&", again(bound), num(SMALL_BOUND), loc), + loc, + })); const pairs: IrStmt = { kind: "for", - init: { kind: "varDecl", localId: row, init: again(match.start), loc }, + init: { kind: "varDecl", localId: row, init: ref(low), loc }, cond: { kind: "bin", op: "<", left: ref(row), - right: bin("-", again(match.end), num(1), loc), + right: bin("-", ref(high), num(1), loc), type: BOOL, loc, }, @@ -268,41 +327,40 @@ function pairRows( body: [ { kind: "varDecl", localId: sum, init: { ...match.init }, loc }, { kind: "varDecl", localId: next, init: { ...match.init }, loc }, + // Both column bounds are int32, so `column < to - 1` is + // `column + 1 < to`. Each row adds its columns in order. { kind: "for", - init: { - kind: "varDecl", - localId: column, - init: again((match.inner.init as IrStmt & { kind: "varDecl" }).init!), + init: { kind: "varDecl", localId: column, init: ref(first), loc }, + cond: { + ...innerCond, + left: ref(column), + right: bin("-", ref(last), num(1), loc), + }, + update: { kind: "assign", localId: column, value: bin("+", ref(column), num(2), loc), loc }, + body: [...step(() => ref(column)), ...step(() => bin("+", ref(column), num(1), loc))], + loc, + }, + // An odd column count leaves the last column, `to - 1`. + { + kind: "if", + cond: { + kind: "logical", + op: "&&", + left: { ...innerCond, left: ref(first), right: ref(last) }, + right: { + kind: "bin", + op: "===", + left: bin("&", bin("-", ref(last), ref(first), loc), num(1), loc), + right: num(1), + type: BOOL, + loc, + }, + type: BOOL, loc, }, - cond: { ...innerCond, left: ref(column), right: again(innerCond.right) }, - update: { kind: "assign", localId: column, value: bin("+", ref(column), num(1), loc), loc }, - body: [ - ...elements, - { - kind: "assign", - localId: sum, - value: bin( - "+", - ref(sum), - termAt(() => ref(row)), - loc, - ), - loc, - }, - { - kind: "assign", - localId: next, - value: bin( - "+", - ref(next), - termAt(() => bin("+", ref(row), num(1), loc)), - loc, - ), - loc, - }, - ], + then: step(() => bin("-", ref(last), num(1), loc)), + else_: null, loc, }, { kind: "bytesSet", arr: { ...match.out }, index: ref(row), value: ref(sum), loc }, @@ -316,9 +374,12 @@ function pairRows( ], loc, }; - // Both bounds are int32, so `row < end - 1` is `row + 1 < end`, and the - // nest has at least two rows. The overlap tests run only then, so they never - // read an array the program assigns only when there are rows. Rows are + // Every bound is an integer in [0, SMALL_BOUND], so `row < end - 1` is + // `row + 1 < end` (and the same for the columns), and the masks in + // `bounds` leave the bounds unchanged while proving both counters' ranges. + // The nest has at least two rows. The column bounds' tests and the + // overlap tests run only then, so they never read an array the program + // assigns only when there are rows. Rows are // independent when the written array shares no byte with an array the // term reads; an array that is not there answers false. const int32 = (e: IrExpr): IrExpr => ({ @@ -329,9 +390,22 @@ function pairRows( type: BOOL, loc, }); + const compare = (op: "<=" | ">=", e: IrExpr, value: number): IrExpr => ({ + kind: "bin", + op, + left: again(e), + right: num(value), + type: BOOL, + loc, + }); + const small = (e: IrExpr): IrExpr[] => [ + int32(e), + compare(">=", e, 0), + compare("<=", e, SMALL_BOUND), + ]; const tests: IrExpr[] = [ - int32(match.start), - int32(match.end), + ...small(match.start), + ...small(match.end), { kind: "bin", op: "<", @@ -340,6 +414,8 @@ function pairRows( type: BOOL, loc, }, + ...small(columnStart), + ...small(innerCond.right), ...match.reads.map((read): IrExpr => ({ kind: "libCall", fn: "bytes.disjoint", @@ -394,7 +470,7 @@ function pairRows( }, loc, }, - { kind: "if", cond: { ...paired }, then: [pairs], else_: null, loc }, + { kind: "if", cond: { ...paired }, then: [...bounds, pairs], else_: null, loc }, rest, ], loc, @@ -413,7 +489,15 @@ function pairRows( * store into shares no byte with any array the term reads, so no row reads * another row's result. An odd row count runs its last row as written. * Otherwise the nest runs as written. Two independent sums take turns in - * the inner loop, so neither waits on the other's addition. */ + * the inner loop, so neither waits on the other's addition. + * + * The inner loop takes two columns per step, each row adding the first + * column's term and then the second's, and the last column on its own + * when the count is odd, so the additions keep their order. Both column + * bounds must also be int32 (so `column < to - 1` is `column + 1 < to`). + * Neighboring rows and columns share counter sums (`row + 1 + column` and + * `row + (column + 1)`), and the step's divisions have no dependence + * between them. */ export function pairRowSums(mod: IrModule): IrModule { let changed = false; const functions = mod.functions.map((fn): IrFunction => { diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8355-row-pair-columns.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8355-row-pair-columns.ts.json new file mode 100644 index 000000000..900de6fb0 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8355-row-pair-columns.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/8355-row-pair-columns.ts" + ], + "diags": [] +} diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8356-shared-view-loop-versions.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8356-shared-view-loop-versions.ts.json new file mode 100644 index 000000000..76cd99065 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8356-shared-view-loop-versions.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/8356-shared-view-loop-versions.ts" + ], + "diags": [] +} diff --git a/tests/corpus/8355-row-pair-columns.ts b/tests/corpus/8355-row-pair-columns.ts new file mode 100644 index 000000000..96ba26153 --- /dev/null +++ b/tests/corpus/8355-row-pair-columns.ts @@ -0,0 +1,73 @@ +// Row sums whose inner loop runs two columns per step: even, odd, zero, one +// and two columns, column ranges that start above zero or at or past their +// end, fractional, negative, NaN and very large column bounds, a column +// bound read from an array's length, terms that read two elements per +// column, and row bounds at and past 2^25. Every sum adds its columns in +// order, so each has the bits of the loop as written. +function rows(u: Float64Array, v: Float64Array, start: number, end: number, from: number, to: number): void { + for (let i = start; i < end; i++) { + let t = 0; + for (let j = from; j < to; j++) t += u[j]! / ((((i + j) * (i + j + 1)) >>> 1) + i + 1); + v[i] = t; + } +} + +function pairs(u: Float64Array, w: Float32Array, v: Float64Array, end: number): void { + for (let i = 0; i < end; i++) { + let t = -0.25; + for (let j = 0; j < u.length; j++) t += u[j]! * w[j + 1]! - (i + 1) / (j + 3); + v[i] = t; + } +} + +const show = (label: string, v: Float64Array): void => { + const parts: string[] = []; + for (const x of v) parts.push(Object.is(x, -0) ? "-0" : x.toPrecision(17)); + console.log(label, parts.join(" ")); +}; + +const n = 9; +const u = new Float64Array(n).map((_, k) => 1 / (k + 1) + k); +for (const [from, to] of [ + [0, n], + [0, n - 1], + [0, 0], + [0, 1], + [0, 2], + [3, n], + [4, n], + [5, 5], + [7, 2], + [0.5, 6], + [0, 6.5], + [-2, 5], + [-0, 4], + [NaN, 4], + [0, NaN], + [0, n + 3], + [2 ** 25, 2 ** 25 + 3], +]) { + const v = new Float64Array(6); + rows(u, v, 0, 6, from!, to!); + show(`columns ${from} ${to}`, v); +} +for (const [start, end] of [ + [2 ** 25 - 3, 2 ** 25 + 1], + [2 ** 25, 2 ** 25 + 2], + [1, 6], +]) { + const v = new Float64Array(8); + rows(u, v, start!, end!, 0, n); + show(`rows ${start} ${end}`, v); +} + +const w = new Float32Array(n + 1).map((_, k) => (k % 4) - 1.5); +for (const end of [0, 1, 4, 5]) { + const v = new Float64Array(5); + pairs(u, w, v, end); + show(`two reads ${end}`, v); +} +const short = new Float64Array(4).map((_, k) => k * 2.5); +const out = new Float64Array(5); +pairs(short, w, out, 5); +show("two reads short", out); diff --git a/tests/corpus/8356-shared-view-loop-versions.ts b/tests/corpus/8356-shared-view-loop-versions.ts new file mode 100644 index 000000000..28f4307fb --- /dev/null +++ b/tests/corpus/8356-shared-view-loop-versions.ts @@ -0,0 +1,93 @@ +import { Worker, isMainThread, parentPort, workerData } from "node:worker_threads"; + +// Loops over typed-array views of shared memory in a worker program: row +// sums and element-wise passes whose accesses are in bounds, the same loops +// running past a view's end, a loop that mixes a shared view with one the +// thread owns, and views of one SharedArrayBuffer that overlap, written in +// an order that shows whether each read sees the previous write. The worker +// writes its results into shared memory, which the main thread reads after +// the worker reports. + +interface Job { + sab: SharedArrayBuffer; + n: number; +} + +function rowSums(u: Float64Array, v: Float64Array, start: number, end: number, n: number): void { + for (let i = start; i < end; i++) { + let t = 0; + for (let j = 0; j < n; j++) t = t + u[j]! / (((i + j) * (i + j + 1)) / 2 + i + 1); + v[i] = t; + } +} + +function scaled(src: Float64Array, dst: Float64Array, k: number, count: number): void { + for (let i = 0; i < count; i++) dst[i] = src[i]! * k + 1; +} + +function mixed(shared: Int32Array, owned: Int32Array, count: number): number { + let sum = 0; + for (let i = 0; i < count; i++) { + owned[i] = shared[i]! * 3; + shared[i] = owned[i]! - i; + sum = (sum + shared[i]! + owned[i]!) | 0; + } + return sum; +} + +function shift(a: Float64Array, b: Float64Array, count: number): void { + for (let i = 0; i < count; i++) a[i + 1] = b[i]! + 1; +} + +function checksum(values: Float64Array): string { + let h = 0; + for (let i = 0; i < values.length; i++) h = (Math.imul(h, 31) + Math.round(values[i]! * 1e6)) >>> 0; + return h.toString(16); +} + +function work({ sab, n }: Job): string[] { + const out: string[] = []; + const u = new Float64Array(sab, 0, n); + const v = new Float64Array(sab, n * 8, n); + for (let i = 0; i < n; i++) u[i] = 1 + (i % 5) / 4; + rowSums(u, v, 0, n, n); + out.push(`rows ${checksum(v)}`); + // Past the end of u: the reads beyond it are undefined, so the sums are NaN. + rowSums(u, v, 0, 4, n + 3); + out.push(`past ${v[0]} ${v[3]} ${checksum(v.subarray(4))}`); + scaled(u, v, 0.5, n); + out.push(`scaled ${checksum(v)}`); + scaled(u, v, 2, n + 2); + out.push(`scaled past ${checksum(v)}`); + const ints = new Int32Array(sab, 2 * n * 8, 32); + for (let i = 0; i < 32; i++) ints[i] = i * 7 - 50; + const own = new Int32Array(32); + out.push(`mixed ${mixed(ints, own, 32)} ${mixed(ints, own, 34)} ${ints[5]} ${own[31]}`); + // a[i + 1] lands on b[i + 1] when b starts one element after a. + const a = new Float64Array(sab, 0, 24); + const b = new Float64Array(sab, 8, 24); + for (let i = 0; i < 25; i++) u[i] = i; + shift(a, b, 20); + out.push(`overlap ${Array.from(a.subarray(0, 8)).join(",")}`); + shift(b, a, 20); + out.push(`overlap back ${Array.from(b.subarray(0, 8)).join(",")}`); + return out; +} + +if (isMainThread) { + const n = 40; + const sab = new SharedArrayBuffer(2 * n * 8 + 32 * 4); + const worker = new Worker(new URL(import.meta.url), { workerData: { sab, n } satisfies Job }); + worker.on("message", (lines: string[]) => { + for (const line of lines) console.log("worker", line); + const v = new Float64Array(sab, n * 8, n); + const ints = new Int32Array(sab, 2 * n * 8, 32); + console.log("main sees", checksum(v), ints[0], ints[31]); + // The same passes on the main thread, over a fresh buffer. + const again = new SharedArrayBuffer(2 * n * 8 + 32 * 4); + for (const line of work({ sab: again, n })) console.log("main", line); + }); + worker.on("exit", (code: number) => console.log("exit", code)); +} else { + parentPort!.postMessage(work(workerData as Job)); +} From 7d458303a632d9b3842e52cea373c3325cff3b19 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Sun, 25 Oct 2026 14:32:00 -0500 Subject: [PATCH 24/44] Check type patch files again on a cached configuration, and supply .jsx entries - The project configuration records the local files a type patch names, with whether each existed. A configuration read again from an unchanged package.json is reused only while every answer still holds, so creating or deleting a local declaration file between builds in one process validates again. - A types patch supplies the declaration TypeScript probes for a .jsx JavaScript entry (x.d.ts for x.jsx), as it does for .js, .mjs and .cjs. - Remove the two tests/fixtures/effect4/node_modules links the previous commit added; that directory is ignored and pnpm creates them. --- .../src/frontend/type-patches/layer.ts | 2 ++ packages/compiler/src/project-config.test.ts | 25 +++++++++++++++++ packages/compiler/src/project-config.ts | 16 +++++++++-- packages/compiler/test/type-patches.test.ts | 28 ++++++++++++++++++- .../node_modules/@effect/platform-node | 1 - tests/fixtures/effect4/node_modules/effect | 1 - 6 files changed, 68 insertions(+), 5 deletions(-) delete mode 120000 tests/fixtures/effect4/node_modules/@effect/platform-node delete mode 120000 tests/fixtures/effect4/node_modules/effect diff --git a/packages/compiler/src/frontend/type-patches/layer.ts b/packages/compiler/src/frontend/type-patches/layer.ts index 5fb41f968..9c156b68c 100644 --- a/packages/compiler/src/frontend/type-patches/layer.ts +++ b/packages/compiler/src/frontend/type-patches/layer.ts @@ -242,6 +242,8 @@ function declarationEntries(root: string, local: string): Map { const rel = target.replace(/^\.\//, ""); if (/\.d\.[cm]?ts$/.test(rel)) out.set(rel, local); else if (/\.[cm]?js$/.test(rel)) out.set(rel.replace(/\.([cm]?)js$/, ".d.$1ts"), local); + // TypeScript probes x.tsx and then x.d.ts for x.jsx. + else if (rel.endsWith(".jsx")) out.set(`${rel.slice(0, -".jsx".length)}.d.ts`, local); else if (!/\.\w+$/.test(rel)) out.set(`${rel}.d.ts`, local); }; add(pkg["types"]); diff --git a/packages/compiler/src/project-config.test.ts b/packages/compiler/src/project-config.test.ts index 51d9520a0..f12a8ea89 100644 --- a/packages/compiler/src/project-config.test.ts +++ b/packages/compiler/src/project-config.test.ts @@ -281,3 +281,28 @@ test("typePatches: files and types name local declaration files", () => { rmSync(root, { recursive: true, force: true }); } }); + +test("readProjectConfig checks local type patch files again for an unchanged package.json", () => { + const root = mkdtempSync(join(tmpdir(), "scriptc-project-config-")); + try { + writeFileSync( + join(root, "package.json"), + JSON.stringify({ scriptc: { typePatches: { "lib@^1.0.0": { types: "./lib.d.ts" } } } }), + ); + const entry = join(root, "main.ts"); + const missing = readProjectConfig(entry); + expect(missing.diagnostics.map((d) => d.message)).toEqual([ + `package.json 'scriptc.typePatches.lib@^1.0.0.types' names "./lib.d.ts", which does not exist`, + ]); + expect(readProjectConfig(entry)).toBe(missing); + writeFileSync(join(root, "lib.d.ts"), "export {};\n"); + const created = readProjectConfig(entry); + expect(created.diagnostics).toEqual([]); + expect(created.typePatches[0]!.types).toBe(join(root, "lib.d.ts")); + expect(readProjectConfig(entry)).toBe(created); + rmSync(join(root, "lib.d.ts")); + expect(readProjectConfig(entry).diagnostics.map((d) => d.code)).toEqual(["SC8004"]); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); diff --git a/packages/compiler/src/project-config.ts b/packages/compiler/src/project-config.ts index 4aad9cbc3..b87b67838 100644 --- a/packages/compiler/src/project-config.ts +++ b/packages/compiler/src/project-config.ts @@ -101,6 +101,10 @@ export interface ProjectConfig { wasm: string[]; /** `scriptc.typePatches`, in manifest order. */ typePatches: TypePatchEntry[]; + /** The local files the configuration checked for existence, with the + * answer. A configuration read again from an unchanged package.json is + * reused only while every answer still holds. */ + localFiles: { path: string; exists: boolean }[]; /** SC8004 for every unknown key or value. */ diagnostics: ScrDiagnostic[]; } @@ -153,6 +157,7 @@ export function emptyProjectConfig(): ProjectConfig { nodeIslands: defaultNodeIslands(), wasm: [], typePatches: [], + localFiles: [], diagnostics: [], }; } @@ -435,7 +440,9 @@ function readTypePatches( ); return null; } - if (!trackedFileExists(abs)) { + const exists = trackedFileExists(abs); + config.localFiles.push({ path: abs, exists }); + if (!exists) { problem( path, `names "${v}", which does not exist`, @@ -546,7 +553,12 @@ export function readProjectConfig(entryPath: string): ProjectConfig { const manifest = join(dir, "package.json"); const text = trackedReadFile(manifest); if (text !== null) { - if (cachedConfig !== null && cachedManifest === manifest && cachedText === text) + if ( + cachedConfig !== null && + cachedManifest === manifest && + cachedText === text && + cachedConfig.localFiles.every((file) => trackedFileExists(file.path) === file.exists) + ) return cachedConfig; // A manifest without a `scriptc` key (none spelled out, none behind a // \u escape) has the empty configuration; builds of such projects diff --git a/packages/compiler/test/type-patches.test.ts b/packages/compiler/test/type-patches.test.ts index 116715db5..ddcd5b88d 100644 --- a/packages/compiler/test/type-patches.test.ts +++ b/packages/compiler/test/type-patches.test.ts @@ -2,7 +2,7 @@ * packages: commander's HelpContext.helpWidth @typedef property and ms's * @return tags, patched from the fixtures' package.json; installed-version * selection; and the warnings for patches that do not apply as written. */ -import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, expect, test } from "vitest"; @@ -153,6 +153,32 @@ test("types gives a package without declarations an entry beside its main", () = expect(typePatchSuppliesTypes(pkg)).toBe(false); }); +test("types gives a package whose main is a .jsx file an entry beside it", () => { + const root = mkdtempSync(join(tmpdir(), "scriptc-type-patches-")); + try { + const pkg = join(root, "node_modules/viewjs"); + mkdirSync(join(pkg, "lib"), { recursive: true }); + writeFileSync( + join(pkg, "package.json"), + JSON.stringify({ name: "viewjs", version: "1.0.0", main: "lib/view.jsx" }), + ); + writeFileSync(join(pkg, "lib/view.jsx"), "export const view = 1;\n"); + writeFileSync(join(root, "viewjs.d.ts"), "export declare const view: number;\n"); + writeFileSync( + join(root, "package.json"), + JSON.stringify({ scriptc: { typePatches: { "viewjs@^1.0.0": { types: "./viewjs.d.ts" } } } }), + ); + configureTypePatches(join(root, "main.ts"), parse); + expect(typePatchFileExists(join(pkg, "lib/view.d.ts"))).toBe(true); + expect(typePatchSuppliesTypes(pkg)).toBe(true); + expect(typePatchedText(join(pkg, "lib/view.d.ts"))).toBe( + "export declare const view: number;\n", + ); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + test("a files target the installed package does not ship warns", () => { const root = mkdtempSync(join(tmpdir(), "scriptc-type-patches-")); try { diff --git a/tests/fixtures/effect4/node_modules/@effect/platform-node b/tests/fixtures/effect4/node_modules/@effect/platform-node deleted file mode 120000 index aaa396705..000000000 --- a/tests/fixtures/effect4/node_modules/@effect/platform-node +++ /dev/null @@ -1 +0,0 @@ -../../../../../node_modules/.pnpm/@effect+platform-node@4.0.0_effect@4.0.0_redis@6.3.0/node_modules/@effect/platform-node \ No newline at end of file diff --git a/tests/fixtures/effect4/node_modules/effect b/tests/fixtures/effect4/node_modules/effect deleted file mode 120000 index 1b933cdef..000000000 --- a/tests/fixtures/effect4/node_modules/effect +++ /dev/null @@ -1 +0,0 @@ -../../../../node_modules/.pnpm/effect@4.0.0/node_modules/effect \ No newline at end of file From 6d34f4f0a5c23fe2001d8c454af7de5217e52104 Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Sun, 25 Oct 2026 12:41:21 -0700 Subject: [PATCH 25/44] Leave withUnitArm unchanged for pending placeholders The lowerer's optional-union cache relies on withUnitArm answering for a placeholder before it completes and recomputing after the revision moves (lowerer.test.ts). Only the optional-parameter site needs to keep the placeholder, so the refusal goes back out. Co-Authored-By: Claude Opus 5.5 --- packages/compiler/src/frontend/type-mapper.test.ts | 14 -------------- packages/compiler/src/frontend/type-mapper.ts | 3 --- 2 files changed, 17 deletions(-) diff --git a/packages/compiler/src/frontend/type-mapper.test.ts b/packages/compiler/src/frontend/type-mapper.test.ts index 6e7e339d6..163ba7b9e 100644 --- a/packages/compiler/src/frontend/type-mapper.test.ts +++ b/packages/compiler/src/frontend/type-mapper.test.ts @@ -53,20 +53,6 @@ describe("union arm lookup", () => { }); }); -describe("recursive union placeholders", () => { - test("refuse a unit arm until their arms are known", () => { - const unions = new UnionRegistry(); - const recursive = {} as Type; - const union = unions.recursiveRef(recursive, () => "Recursive"); - const placeholder: IrType = { kind: "union", unionId: union }; - expect(unions.isPendingPlaceholder(union)).toBe(true); - expect(withUndefinedArm(placeholder, unions)).toBeNull(); - unions.finalizeRecursive(recursive, [{ kind: "record", shapeId: "r1" }, UNDEFINED_T]); - expect(unions.isPendingPlaceholder(union)).toBe(false); - expect(withUndefinedArm(placeholder, unions)).toEqual(placeholder); - }); -}); - describe("nullable collection union builders", () => { test.each([mapOf(STRING, F64), setOf(STRING)])( "optional %j fields and generator results share the same union", diff --git a/packages/compiler/src/frontend/type-mapper.ts b/packages/compiler/src/frontend/type-mapper.ts index 63eed378d..d819d30a2 100644 --- a/packages/compiler/src/frontend/type-mapper.ts +++ b/packages/compiler/src/frontend/type-mapper.ts @@ -6799,9 +6799,6 @@ export function withUnitArm( if (t.kind === "union") { const def = unions.get(t.unionId); if (!def) return null; - // A recursive placeholder's arms arrive when its outer frame finishes; - // extending its (still empty) arm list would drop every arm but the unit. - if (unions.isPendingPlaceholder(t.unionId)) return null; if (def.arms.some((a) => a.kind === kind)) return t; const arms = [...def.arms, unit]; // Date and Promise arms beside other data arms have no union home. From 7222c3e900320b99c906b61f0a2987ed7e008d13 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Sun, 25 Oct 2026 14:44:05 -0500 Subject: [PATCH 26/44] Skip module-pass work that changes nothing in the functions it walks The LLVM emitter's module passes walk every function, and on large programs most of them find nothing to change in most functions. Each pass now avoids work whose result it does not use, with the same output: - scalarizeLiteralResults built, for every function, a set of all locals and all module globals before looking for a producer call. It now collects the names a fresh binding must avoid when it makes the first one. - specializeCallbackCalls walked every body five times: rebindings, node counts and definitions, call keys, untouched bodies, and callers. One walk now records each body's node count, rebindings, written values and helper calls in walk order; the later steps read those records and walk only the bodies the rewrite changed and the copies it made. - holdCharLocalsAsUnits skips its two object walks in functions without a candidate local, and its mention count where no candidate is declared from a charAt or an optional read (the only ones it can hold). - resolveSelfCaptures only asks CaptureFlow whether bindings are unchanged, so its flow collects the facts of the bodies those questions reach instead of scanning every body. - copyStableCaptures reads the function references of the bodies it kept from its CaptureFlow facts instead of walking them again, and makes the set of local ids a copy must avoid only when it makes a copy. - versionLoopBoundParameters counts call sites only when a function has loop-bound parameters. - pairRowSums and inlineExpressionFunctions look for a row sum or an inlinable call before rebuilding a body. --- .../src/ir/callback-specialization.ts | 168 ++++++++++++------ packages/compiler/src/ir/capture-values.ts | 32 +++- packages/compiler/src/ir/char-unit-locals.ts | 10 +- .../compiler/src/ir/expression-inlining.ts | 19 +- .../compiler/src/ir/parameter-versions.ts | 9 +- packages/compiler/src/ir/row-pairs.ts | 18 +- packages/compiler/src/ir/scalar-records.ts | 12 +- packages/compiler/src/ir/self-captures.ts | 3 +- 8 files changed, 195 insertions(+), 76 deletions(-) diff --git a/packages/compiler/src/ir/callback-specialization.ts b/packages/compiler/src/ir/callback-specialization.ts index f4421f34b..ba579061c 100644 --- a/packages/compiler/src/ir/callback-specialization.ts +++ b/packages/compiler/src/ir/callback-specialization.ts @@ -33,6 +33,22 @@ const REBINDING_KINDS: readonly IrNodeKind[] = [ "closure", ]; +/** The rebindings one function's body makes: locals it writes after + * declaring them, and the closures it creates over captured bindings. */ +interface FunctionRebindings { + ids: Set; + sites: { owner: string; fnName: string; captures: string[] }[]; +} + +/** Records `node`'s rebinding into `into` (owner `fnName`). */ +function noteRebinding(fnName: string, node: IrExpr | IrStmt, into: FunctionRebindings): void { + if (node.kind === "assign" || node.kind === "forOf") into.ids.add(node.localId); + else if (node.kind === "tryCatch" && node.catchLocalId !== null) into.ids.add(node.catchLocalId); + else if (node.kind === "assignExpr" || node.kind === "incDec") into.ids.add(node.localId); + else if (node.kind === "closure" && node.captures.length) + into.sites.push({ owner: fnName, fnName: node.fnName, captures: node.captures }); +} + function rebindings( mod: { readonly functions: readonly IrFunction[] }, nodes?: IrNodeIndex, @@ -41,21 +57,31 @@ function rebindings( /** Bindings of each function that a closure it creates writes. */ closureWritten: Map>; } { + const scan = nodes?.scan(REBINDING_KINDS); + const each = mod.functions.map((fn) => { + const own: FunctionRebindings = { ids: new Set(), sites: [] }; + eachNodeOfKinds(fn, scan, (node) => noteRebinding(fn.name, node, own)); + return own; + }); + return closeRebindings(mod, each); +} + +/** The module's rebindings from each function's own (`each`, in module + * order): + * a binding a closure writes through its environment is written by the + * function that created the closure, transitively. */ +function closeRebindings( + mod: { readonly functions: readonly IrFunction[] }, + each: readonly FunctionRebindings[], +): { written: Map>; closureWritten: Map> } { const written = new Map>(); const closureWritten = new Map>(); const sites: { owner: string; fnName: string; captures: string[] }[] = []; - const scan = nodes?.scan(REBINDING_KINDS); - for (const fn of mod.functions) { - const ids = new Set(); - eachNodeOfKinds(fn, scan, (node) => { - if (node.kind === "assign" || node.kind === "forOf") ids.add(node.localId); - else if (node.kind === "tryCatch" && node.catchLocalId !== null) ids.add(node.catchLocalId); - else if (node.kind === "assignExpr" || node.kind === "incDec") ids.add(node.localId); - else if (node.kind === "closure" && node.captures.length) - sites.push({ owner: fn.name, fnName: node.fnName, captures: node.captures }); - }); - written.set(fn.name, ids); - } + mod.functions.forEach((fn, index) => { + const own = each[index]!; + written.set(fn.name, own.ids); + for (const site of own.sites) sites.push(site); + }); const functions = new Map(mod.functions.map((fn) => [fn.name, fn])); for (let changed = true; changed;) { changed = false; @@ -130,9 +156,10 @@ export function specializeCallbackCalls( symbolName: (name: string) => string = (name) => name, ): IrModule { const functions = new Map(mod.functions.map((fn) => [fn.name, fn])); - const { written, closureWritten } = rebindings(mod); const copies = new Map(); - const candidates = new Map(); + // Helpers that may take callbacks: their callback parameters are final + // once the walk below finds which of them the body rebinds. + const helpers = new Map(); for (const fn of mod.functions) { if ( !(fn.ownsPrototype || fn.name.startsWith("%arr.")) || @@ -142,19 +169,13 @@ export function specializeCallbackCalls( fn.captures !== undefined || fn.classCaptures?.length || fn.fastRead !== undefined || - fn.speculativeDispatch + fn.speculativeDispatch || + !fn.params.some((param) => param.type.kind === "func") ) continue; - const nodes = countNodes(fn); - if (nodes > MAX_FUNCTION_NODES) continue; - const rebound = written.get(fn.name)!; - const slots: number[] = []; - fn.params.forEach((param, index) => { - if (param.type.kind === "func" && !rebound.has(param.localId)) slots.push(index); - }); - if (slots.length) candidates.set(fn.name, { fn, slots, nodes }); + helpers.set(fn.name, fn); } - if (!candidates.size) return mod; + if (!helpers.size) return mod; // Function values, exports, entry points and methods: an original // reachable this way stays even when every direct call moves to a copy. @@ -162,16 +183,17 @@ export function specializeCallbackCalls( for (const cls of mod.classes ?? []) for (const method of ["constructor", ...(cls.methods ?? [])]) referenced.add(`%${cls.name}.${method}`); - // Single definitions of function-typed locals and globals. - const definitions = new Map>(); const globalIds = new Set(mod.globals?.map((global) => global.id)); const globalWrites = new Map(); - // The sum of countNodes over every function, counted by the walk below. - let totalNodes = 0; - for (const fn of mod.functions) { + // One walk of every body records what the steps below read: its node + // count (countNodes), its rebindings, the values written into each of its + // bindings, and its direct calls of helpers, in walk order. + const each = mod.functions.map((fn) => { let nodes = 0; + const own: FunctionRebindings = { ids: new Set(), sites: [] }; const values = new Map(); const counts = new Map(); + const calls: (IrExpr & { kind: "call" })[] = []; const record = (id: string, value: IrExpr | null, weight: number): void => { counts.set(id, (counts.get(id) ?? 0) + weight); if (value) { @@ -183,6 +205,7 @@ export function specializeCallbackCalls( everyStmtList(fn.body, { stmt: (stmt) => { nodes++; + noteRebinding(fn.name, stmt, own); if (stmt.kind === "varDecl") record(stmt.localId, stmt.init, stmt.init ? 1 : 0); if (stmt.kind === "assign") { if (globalIds.has(stmt.localId)) { @@ -199,15 +222,42 @@ export function specializeCallbackCalls( }, expr: (expr) => { nodes++; + noteRebinding(fn.name, expr, own); if (expr.kind === "assignExpr" || expr.kind === "incDec") { record(expr.localId, null, 2); if (globalIds.has(expr.localId)) globalWrites.set(expr.localId, []); } + if (expr.kind === "call" && helpers.has(expr.callee)) calls.push(expr); const name = exprFunctionReference(expr); if (name !== undefined) referenced.add(name); return true; }, }); + return { fn, nodes: Math.min(nodes, MAX_FUNCTION_NODES + 1), own, values, counts, calls }; + }); + const { written, closureWritten } = closeRebindings( + mod, + each.map((entry) => entry.own), + ); + const nodesOf = new Map(each.map((entry) => [entry.fn.name, entry.nodes])); + const candidates = new Map(); + for (const fn of helpers.values()) { + const nodes = nodesOf.get(fn.name)!; + if (nodes > MAX_FUNCTION_NODES) continue; + const rebound = written.get(fn.name)!; + const slots: number[] = []; + fn.params.forEach((param, index) => { + if (param.type.kind === "func" && !rebound.has(param.localId)) slots.push(index); + }); + if (slots.length) candidates.set(fn.name, { fn, slots, nodes }); + } + if (!candidates.size) return mod; + + // Single definitions of function-typed locals and globals. + const definitions = new Map>(); + // The sum of countNodes over every function. + let totalNodes = 0; + for (const { fn, nodes, values, counts } of each) { const single = new Map(); const locals = new Set(fn.locals.map((local) => local.id)); const shared = closureWritten.get(fn.name); @@ -221,7 +271,7 @@ export function specializeCallbackCalls( ) single.set(id, list[0]!); definitions.set(fn.name, single); - totalNodes += Math.min(nodes, MAX_FUNCTION_NODES + 1); + totalNodes += nodes; } const globalDefinitions = new Map(); for (const global of mod.globals ?? []) @@ -288,23 +338,18 @@ export function specializeCallbackCalls( // unknown callbacks or function-value entries. const keysOf = new Map>(); const unknown = new Set(); - for (const fn of mod.functions) - everyStmtList(fn.body, { - stmt: () => true, - expr: (expr) => { - if (expr.kind !== "call") return true; - const candidate = candidates.get(expr.callee); - if (!candidate) return true; - const key = callKey(fn, new Map(), expr, candidate); - if (key === "") unknown.add(expr.callee); - else if (key !== null) { - const keys = keysOf.get(expr.callee) ?? new Set(); - keys.add(key); - keysOf.set(expr.callee, keys); - } - return true; - }, - }); + for (const { fn, calls } of each) + for (const call of calls) { + const candidate = candidates.get(call.callee); + if (!candidate) continue; + const key = callKey(fn, new Map(), call, candidate); + if (key === "") unknown.add(call.callee); + else if (key !== null) { + const keys = keysOf.get(call.callee) ?? new Set(); + keys.add(key); + keysOf.set(call.callee, keys); + } + } const specialized = new Set(); for (const [name, keys] of keysOf) if (keys.size > 1 || unknown.has(name) || referenced.has(name)) specialized.add(name); @@ -419,12 +464,9 @@ export function specializeCallbackCalls( return owner.body.map(stmt); } - const functionsOut = mod.functions.map((fn) => { + const functionsOut = each.map(({ fn, calls }) => { // Most bodies call no specialized original; keep their nodes. - const untouched = everyStmtList(fn.body, { - stmt: () => true, - expr: (expr) => expr.kind !== "call" || !specialized.has(expr.callee), - }); + const untouched = calls.every((call) => !specialized.has(call.callee)); return untouched ? fn : { ...fn, body: rewriteBody(fn, new Map(), new Set(), false) }; }); for (let i = 0; i < pending.length; i++) { @@ -437,19 +479,29 @@ export function specializeCallbackCalls( // Drop originals no remaining code reaches: every direct call moved to a // copy and nothing names them as a value. const callers = new Map>(); - for (const fn of functionsOut) + const call = (callee: string, caller: string): void => { + if (!specialized.has(callee)) return; + const set = callers.get(callee) ?? new Set(); + set.add(caller); + callers.set(callee, set); + }; + functionsOut.forEach((fn, index) => { + // A body the rewrite kept has the calls the walk recorded, and its + // closures are already in `referenced`. + const kept = each[index]; + if (kept !== undefined && kept.fn === fn) { + for (const site of kept.calls) call(site.callee, fn.name); + return; + } everyStmtList(fn.body, { stmt: () => true, expr: (expr) => { - if (expr.kind === "call" && specialized.has(expr.callee)) { - const set = callers.get(expr.callee) ?? new Set(); - set.add(fn.name); - callers.set(expr.callee, set); - } + if (expr.kind === "call") call(expr.callee, fn.name); if (expr.kind === "closure" && specialized.has(expr.fnName)) referenced.add(expr.fnName); return true; }, }); + }); const live = new Set(); const work: string[] = []; for (const fn of functionsOut) diff --git a/packages/compiler/src/ir/capture-values.ts b/packages/compiler/src/ir/capture-values.ts index bcc587f1b..19ec41363 100644 --- a/packages/compiler/src/ir/capture-values.ts +++ b/packages/compiler/src/ir/capture-values.ts @@ -86,7 +86,12 @@ export function copyStableCaptures(mod: IrModule): IrModule { // A closure body whose every use became a direct call is only reachable // through its copy now. const referenced = new Set(moduleFunctionReferences(mod)); - for (const fn of functions) + functions.forEach((fn, index) => { + // A body the rewrite kept names what the flow recorded of it. + if (fn === mod.functions[index]) { + for (const name of flow.references(fn)) referenced.add(name); + return; + } everyStmtList(fn.body, { stmt: () => true, expr: (e) => { @@ -95,6 +100,7 @@ export function copyStableCaptures(mod: IrModule): IrModule { return true; }, }); + }); const clones = flow.directClones(); return { ...mod, @@ -251,6 +257,9 @@ interface FunctionFacts { captureIndex: Map; classCaptures: Set; params: Set; + /** Functions the body names: direct call callees and function values + * (exprFunctionReference). */ + references: Set; } interface CreationSite { @@ -273,17 +282,26 @@ export class CaptureFlow { /** Environment-free bodies called directly, and their copies' names. */ private readonly clones = new Map(); - constructor(functions: readonly IrFunction[]) { + /** With `scan` false, each body's facts are collected when a question + * first needs them, and only unchanged() may be asked: the other + * questions read creation sites, which come from scanning every body. */ + constructor(functions: readonly IrFunction[], scan = true) { for (const fn of functions) this.functions.set(fn.name, fn); this.receivers = new AmbientReceiverReaders(this.functions); // Creation sites are collected while scanning each body. - for (const fn of functions) this.factsOf(fn); + if (scan) for (const fn of functions) this.factsOf(fn); } directClones(): ReadonlyMap { return this.clones; } + /** The functions the body of `fn`, one of the functions this flow was + * made from, names (FunctionFacts.references). */ + references(fn: IrFunction): ReadonlySet { + return this.factsOf(fn).references; + } + /** A lifted body that captures nothing and can be entered like a declared * function: synchronous, never naming its own closure, and unable to * observe the ambient receiver a closure call would bracket. */ @@ -459,6 +477,7 @@ export class CaptureFlow { captureIndex: new Map(), classCaptures: new Set((fn.classCaptures ?? []).map((c) => c.localId)), params: new Set(fn.params.map((p) => p.localId)), + references: new Set(), }; (fn.captures ?? []).forEach((c, index) => { if (!facts.captureIndex.has(c.localId)) facts.captureIndex.set(c.localId, index); @@ -511,6 +530,8 @@ export class CaptureFlow { facts.writes.set(id, (facts.writes.get(id) ?? 0) + 1); }; const expr = (e: IrExpr): boolean => { + const referenced = e.kind === "call" ? e.callee : exprFunctionReference(e); + if (referenced !== undefined) facts.references.add(referenced); switch (e.kind) { case "varRef": facts.reads.set(e.localId, (facts.reads.get(e.localId) ?? 0) + 1); @@ -707,8 +728,11 @@ export class CaptureFlow { const splits = new Map(); const entry: IrStmt[] = []; const locals: IrLocal[] = []; - const used = new Set(fn.locals.map((l) => l.id)); + // The local ids a copy must avoid, collected for the first copy: most + // functions make none. + let used: Set | null = null; const fresh = (local: IrLocal): IrLocal => { + used ??= new Set(fn.locals.map((l) => l.id)); let k = 0; while (used.has(`${local.name}.${k}`)) k++; const id = `${local.name}.${k}`; diff --git a/packages/compiler/src/ir/char-unit-locals.ts b/packages/compiler/src/ir/char-unit-locals.ts index aff952909..fc1d5873b 100644 --- a/packages/compiler/src/ir/char-unit-locals.ts +++ b/packages/compiler/src/ir/char-unit-locals.ts @@ -300,6 +300,8 @@ function rewriteFunction(fn: IrFunction): IrFunction { // or as copies of another such local (a switch's discriminant temp). const reads = new Map(); const copyOf = new Map(); + // Whether a candidate string is declared from a charAt. + let charAtDeclared = false; const collect = (value: unknown): void => { if (Array.isArray(value)) for (const item of value) collect(item); else if (value !== null && typeof value === "object") { @@ -307,6 +309,7 @@ function rewriteFunction(fn: IrFunction): IrFunction { if (node.kind === "varDecl" && candidates.has(node.localId as string) && node.init) { const id = node.localId as string; const init = node.init as IrExpr; + if (locals.get(id)?.type.kind === "string" && isCharAt(init)) charAtDeclared = true; const read = optionalCharRead(init); if (read) reads.set(id, read); else if (init.kind === "varRef" && reads.has(init.localId)) { @@ -317,7 +320,8 @@ function rewriteFunction(fn: IrFunction): IrFunction { for (const key in node) if (key !== "loc") collect(node[key]); } }; - collect(fn.body); + // Both walks only record candidates: a function without one holds none. + if (candidates.size > 0) collect(fn.body); const localTags = (operand: IrExpr): OptionalCharRead | null => operand.kind === "varRef" ? (reads.get(operand.localId) ?? null) : null; @@ -344,7 +348,9 @@ function rewriteFunction(fn: IrFunction): IrFunction { for (const key in node) if (key !== "loc") count(node[key]); } }; - count(fn.body); + // Only candidates declared from a charAt or an optional read can be held + // (understoodMention), and count() only decides which are. + if (reads.size > 0 || charAtDeclared) count(fn.body); // A copy is held only with its source, and the source's mention in the // copy's initializer is understood only while the copy is held. const held = new Set(); diff --git a/packages/compiler/src/ir/expression-inlining.ts b/packages/compiler/src/ir/expression-inlining.ts index 377cfaf3f..1282cdb79 100644 --- a/packages/compiler/src/ir/expression-inlining.ts +++ b/packages/compiler/src/ir/expression-inlining.ts @@ -128,15 +128,24 @@ export function inlineExpressionFunctions(mod: IrModule): IrModule { const functions = mod.functions.map((fn) => { const locals = new Map(fn.locals.map((local) => [local.id, local])); const inLoops = callsInLoops(fn.body); - if (inLoops.size === 0) return fn; + // Each call the rewrite inlines: of an expression function, in a loop, + // with arguments the copy may repeat. + const inlinable = (e: IrExpr & { kind: "call" }): boolean => { + if (e.callee === fn.name) return false; + const target = expressions.get(e.callee); + return ( + target !== undefined && + target.fn.params.length === e.args.length && + e.args.every((arg) => duplicableArgument(arg, locals)) + ); + }; + if (![...inLoops].some((e) => e.kind === "call" && inlinable(e))) return fn; let inlined = false; const body = transformStmtList(fn.body, { stmt: (s) => s, expr: (e) => { - if (e.kind !== "call" || e.callee === fn.name || !inLoops.has(e)) return e; - const target = expressions.get(e.callee); - if (target === undefined || target.fn.params.length !== e.args.length) return e; - if (!e.args.every((arg) => duplicableArgument(arg, locals))) return e; + if (e.kind !== "call" || !inLoops.has(e) || !inlinable(e)) return e; + const target = expressions.get(e.callee)!; const args = new Map(target.fn.params.map((param, i) => [param.localId, e.args[i]!])); inlined = true; return copy(target.value, (ref) => { diff --git a/packages/compiler/src/ir/parameter-versions.ts b/packages/compiler/src/ir/parameter-versions.ts index 88232251c..5156d4bff 100644 --- a/packages/compiler/src/ir/parameter-versions.ts +++ b/packages/compiler/src/ir/parameter-versions.ts @@ -184,7 +184,8 @@ export function versionLoopBoundParameters( symbolName: (name: string) => string = (name) => name, ): IrModule { const candidates: { fn: IrFunction; params: Set }[] = []; - const sites = callSites(mod); + // Call sites are read only for functions with loop-bound parameters. + let sites: ReturnType | null = null; const functions = new Map(mod.functions.map((fn) => [fn.name, fn])); const members = new Set((mod.classes ?? []).flatMap((cls) => cls.methods ?? [])); for (const fn of mod.functions) { @@ -207,7 +208,11 @@ export function versionLoopBoundParameters( const local = locals.get(id); if (!local || local.boxed || local.tdz) params.delete(id); } - if (params.size === 0 || size(fn) > MAX_FUNCTION_NODES || keepsSingleBody(fn, sites, functions)) + if ( + params.size === 0 || + size(fn) > MAX_FUNCTION_NODES || + keepsSingleBody(fn, (sites ??= callSites(mod)), functions) + ) continue; candidates.push({ fn, params }); } diff --git a/packages/compiler/src/ir/row-pairs.ts b/packages/compiler/src/ir/row-pairs.ts index 2e189f42e..d5342d113 100644 --- a/packages/compiler/src/ir/row-pairs.ts +++ b/packages/compiler/src/ir/row-pairs.ts @@ -7,7 +7,13 @@ import { type IrModule, type IrStmt, } from "./ir.js"; -import { everyExpr, everyExprChild, mapExprChildren, transformStmtList } from "./traverse.js"; +import { + everyExpr, + everyExprChild, + everyStmtList, + mapExprChildren, + transformStmtList, +} from "./traverse.js"; /** Inner-loop expressions above this many nodes keep the loop as written. */ const MAX_TERM_NODES = 64; @@ -419,6 +425,16 @@ export function pairRowSums(mod: IrModule): IrModule { const functions = mod.functions.map((fn): IrFunction => { if (fn.async || fn.generator) return fn; const locals = new Map(fn.locals.map((local) => [local.id, local])); + // Most bodies hold no row sum: find one before copying the body. + let found = false; + everyStmtList(fn.body, { + expr: () => !found, + stmt: (s) => { + if (s.kind === "for" && matchRowSum(s, locals) !== null) found = true; + return !found; + }, + }); + if (!found) return fn; const added: IrLocal[] = []; const ids = new Set(fn.locals.map((local) => local.id)); const fresh = (name: string): string => { diff --git a/packages/compiler/src/ir/scalar-records.ts b/packages/compiler/src/ir/scalar-records.ts index 5d57cb3ec..088fd8461 100644 --- a/packages/compiler/src/ir/scalar-records.ts +++ b/packages/compiler/src/ir/scalar-records.ts @@ -554,17 +554,23 @@ export function scalarizeLiteralResults(mod: IrModule): IrModule { } if (producers.size === 0) return mod; let changed = false; + const globalIds = new Set((mod.globals ?? []).map((g) => g.id)); const functions = mod.functions.map((fn): IrFunction => { if (fn.async || fn.generator) return fn; const locals = new Map(fn.locals.map((l) => [l.id, l])); - const used = new Set([...locals.keys(), ...(mod.globals ?? []).map((g) => g.id)]); - collectLabels(fn.body, used); + // The names a fresh binding must avoid, collected when the first one is + // made: most functions inline no producer. + let used: Set | null = null; let next = 0; const fresh = (): string => { + if (used === null) { + used = new Set(locals.keys()); + collectLabels(fn.body, used); + } let id: string; do { id = `%literal.${next++}`; - } while (used.has(id)); + } while (used.has(id) || globalIds.has(id)); used.add(id); return id; }; diff --git a/packages/compiler/src/ir/self-captures.ts b/packages/compiler/src/ir/self-captures.ts index 74f634d4a..70242cb8b 100644 --- a/packages/compiler/src/ir/self-captures.ts +++ b/packages/compiler/src/ir/self-captures.ts @@ -107,7 +107,8 @@ export function resolveSelfCaptures(mod: IrModule): IrModule { if (creator.captures?.some((c) => c.localId === source)) return false; if (creator.classCaptures?.some((c) => c.localId === source)) return false; if (declarations.get(owner)!.get(source) !== 1) return false; - flow ??= new CaptureFlow(mod.functions); + // Only unchanged() is asked: the flow reads just the bodies it needs. + flow ??= new CaptureFlow(mod.functions, false); return flow.unchanged(owner, source, definition.writes); }); if (qualifies) { From 6b51aa883f99337b45a7e854f0c27eef1f6b2f65 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Sun, 25 Oct 2026 14:46:01 -0500 Subject: [PATCH 27/44] Keep -0 row and column starts out of the paired row-sum nest The paired nest reads its bounds masked to [0, 2^25 - 1], which turns a -0 start into +0, and a term can tell the two apart (1 / i on the first row). The bounds test now requires 1 / b > 0, which -0 fails, so such a nest runs as written. Corpus 8355 adds row and column starts of -0 with terms that see the sign. --- packages/compiler/src/ir/row-pairs.ts | 13 ++++++++----- tests/corpus/8355-row-pair-columns.ts | 25 +++++++++++++++++++++++-- 2 files changed, 31 insertions(+), 7 deletions(-) diff --git a/packages/compiler/src/ir/row-pairs.ts b/packages/compiler/src/ir/row-pairs.ts index e6b3889a5..d54f37b0b 100644 --- a/packages/compiler/src/ir/row-pairs.ts +++ b/packages/compiler/src/ir/row-pairs.ts @@ -203,7 +203,7 @@ function copy( } const bin = ( - op: "+" | "-" | "&" | "|", + op: "+" | "-" | "/" | "&" | "|", left: IrExpr, right: IrExpr, loc: IrExpr["loc"], @@ -374,9 +374,10 @@ function pairRows( ], loc, }; - // Every bound is an integer in [0, SMALL_BOUND], so `row < end - 1` is - // `row + 1 < end` (and the same for the columns), and the masks in - // `bounds` leave the bounds unchanged while proving both counters' ranges. + // Every bound is an integer in [0, SMALL_BOUND] and not -0, so `row < + // end - 1` is `row + 1 < end` (and the same for the columns), and the + // masks in `bounds` leave the bounds unchanged while proving both + // counters' ranges. // The nest has at least two rows. The column bounds' tests and the // overlap tests run only then, so they never read an array the program // assigns only when there are rows. Rows are @@ -398,9 +399,11 @@ function pairRows( type: BOOL, loc, }); + // `1 / e > 0` holds for +0 and every positive bound, but not for -0: + // masking would turn a -0 start into +0, which a term can tell apart. const small = (e: IrExpr): IrExpr[] => [ int32(e), - compare(">=", e, 0), + { kind: "bin", op: ">", left: bin("/", num(1), again(e), loc), right: num(0), type: BOOL, loc }, compare("<=", e, SMALL_BOUND), ]; const tests: IrExpr[] = [ diff --git a/tests/corpus/8355-row-pair-columns.ts b/tests/corpus/8355-row-pair-columns.ts index 96ba26153..dccd9ca6d 100644 --- a/tests/corpus/8355-row-pair-columns.ts +++ b/tests/corpus/8355-row-pair-columns.ts @@ -2,8 +2,9 @@ // and two columns, column ranges that start above zero or at or past their // end, fractional, negative, NaN and very large column bounds, a column // bound read from an array's length, terms that read two elements per -// column, and row bounds at and past 2^25. Every sum adds its columns in -// order, so each has the bits of the loop as written. +// column, row bounds at and past 2^25, and row and column starts of -0 +// with terms that tell -0 from +0. Every sum adds its columns in order, +// so each has the bits of the loop as written. function rows(u: Float64Array, v: Float64Array, start: number, end: number, from: number, to: number): void { for (let i = start; i < end; i++) { let t = 0; @@ -20,6 +21,16 @@ function pairs(u: Float64Array, w: Float32Array, v: Float64Array, end: number): } } +// 1 / (i * 4 - j) is -Infinity at a -0 row and +0 column, and +// 1 / (j - i * 4) at a -0 column and +0 row: the first pass sees the sign. +function signs(v: Float64Array, start: number, end: number, from: number, to: number): void { + for (let i = start; i < end; i++) { + let t = 0; + for (let j = from; j < to; j++) t += 1 / (i * 4 - j) + 1 / (j - i * 4) + 1 / (i * 4 + j + 1); + v[i] = t; + } +} + const show = (label: string, v: Float64Array): void => { const parts: string[] = []; for (const x of v) parts.push(Object.is(x, -0) ? "-0" : x.toPrecision(17)); @@ -71,3 +82,13 @@ const short = new Float64Array(4).map((_, k) => k * 2.5); const out = new Float64Array(5); pairs(short, w, out, 5); show("two reads short", out); +for (const [start, from] of [ + [0, 0], + [-0, 0], + [0, -0], + [-0, -0], +]) { + const v = new Float64Array(4); + signs(v, start!, 4, from!, 5); + show(`signs ${Object.is(start, -0) ? "-0" : start} ${Object.is(from, -0) ? "-0" : from}`, v); +} From f791c1d0061b946c4f1d2f24ca9673ca50e6efba Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Sun, 25 Oct 2026 14:47:24 -0500 Subject: [PATCH 28/44] Address review feedback --- .../src/frontend/lowering/record-inherited.ts | 144 +++++++++++++++--- .../8362-record-inherited-order.ts.json | 6 + tests/corpus/8362-record-inherited-order.ts | 40 +++++ .../record-inherited/member-signature.ts | 12 ++ .../8362-record-inherited-order.ts.json | 1 + tests/harness/record-inherited-member.test.ts | 32 ++++ 6 files changed, 211 insertions(+), 24 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8362-record-inherited-order.ts.json create mode 100644 tests/corpus/8362-record-inherited-order.ts create mode 100644 tests/fixtures/record-inherited/member-signature.ts create mode 100644 tests/harness/corpus-costs/8362-record-inherited-order.ts.json create mode 100644 tests/harness/record-inherited-member.test.ts diff --git a/packages/compiler/src/frontend/lowering/record-inherited.ts b/packages/compiler/src/frontend/lowering/record-inherited.ts index 2c01ccd0a..65d070f1d 100644 --- a/packages/compiler/src/frontend/lowering/record-inherited.ts +++ b/packages/compiler/src/frontend/lowering/record-inherited.ts @@ -104,14 +104,19 @@ export function withInheritedMethods( if (form === null || !shape || shape.tuple) return read; const loc = read.loc; const stmts: IrStmt[] = []; - const stable = (value: IrExpr, name: string): IrExpr => { - if (isSafeToRepeat(value) || value.kind === "strLit") return value; + const repeatable = (value: IrExpr): boolean => isSafeToRepeat(value) || value.kind === "strLit"; + const stable = (value: IrExpr, name: string, keep: boolean): IrExpr => { + if (keep) return value; const local = lowerer.declareHiddenLocal(name, value.type); stmts.push({ kind: "varDecl", localId: local.id, init: value, loc }); return varRef(local.id, value.type, loc); }; - const obj = stable(read.obj, "%inheritedRecv"); - const key = stable(read.key, "%inheritedKey"); + // The receiver is evaluated before the key: when computing the key runs + // code (it may reassign the receiver's variable), the receiver is held + // first, even when reading it again would be safe on its own. + const keyRepeatable = repeatable(read.key); + const obj = stable(read.obj, "%inheritedRecv", repeatable(read.obj) && keyRepeatable); + const key = stable(read.key, "%inheritedKey", keyRepeatable); const helper = inheritedReadHelper(lowerer, read.shapeId, read.type, form, bound, loc); const fallback: IrExpr = { kind: "call", callee: helper, args: [obj, key], type: read.type, loc }; if ( @@ -481,19 +486,52 @@ function builtinAnswer( // No object is created with a typed record as its prototype. return { kind: "boolLit", value: false, type: BOOL, loc }; default: { - const key = propertyKeyOf(lowerer, args[0], loc); - return { + // hasOwnProperty and propertyIsEnumerable: ToPropertyKey keeps a + // symbol a symbol, and a record's own keys are strings, so a symbol + // key is no entry of it. + const no: IrExpr = { kind: "boolLit", value: false, type: BOOL, loc }; + const arg = args[0]; + if (arg?.type.kind === "symbol") return sequence(arg, no, loc); + const has = (key: IrExpr): IrExpr => ({ kind: "call", callee: recordHasOwnHelper(lowerer, shapeId, loc), args: [o, key], type: BOOL, loc, + }); + if (arg === undefined || !mayBeSymbol(arg.type)) return has(propertyKeyOf(lowerer, arg, loc)); + const id = declare("%protoKey", DYN); + const held = varRef(id, DYN, loc); + return { + kind: "seqExpr", + stmts: [{ kind: "varDecl", localId: id, init: lowerer.coerceToExpected(arg, DYN), loc }], + result: { + kind: "ternary", + cond: { kind: "dynTest", test: "symbol", value: held, type: BOOL, loc } as IrExpr, + then: no, + else_: has(propertyKeyOf(lowerer, held, loc)), + type: BOOL, + loc, + }, + type: BOOL, + loc, }; } } } -/** ToPropertyKey of a built-in's key argument (absent: "undefined"). */ +/** Whether a value of `type` may be a symbol at run time. */ +function mayBeSymbol(type: IrType): boolean { + return !( + isUnitType(type) || + type.kind === "string" || + type.kind === "f64" || + type.kind === "bool" + ); +} + +/** ToPropertyKey of a built-in's key argument (absent: "undefined"), for + * keys that are not symbols. */ function propertyKeyOf(lowerer: Lowerer, arg: IrExpr | undefined, loc: SrcLoc): IrExpr { if (arg === undefined || isUnitType(arg.type)) return { @@ -552,6 +590,7 @@ function callOwnOrBuiltin( want: IrType, calleeText: string, loc: SrcLoc, + resolved?: { has: IrExpr; own: IrExpr }, ): IrExpr { const shape = lowerer.shapes.get(shapeId)!; const field = shape.fields.find((f) => f.name === method); @@ -569,7 +608,47 @@ function callOwnOrBuiltin( }; const builtinAsWant = convertTo(lowerer, builtin, want) ?? builtin; if (ownType === undefined || !callableType(lowerer, ownType)) return builtinAsWant; - const own: IrExpr = field + const own = resolved?.own ?? ownEntryRead(lowerer, o, shapeId, method, loc); + const called = callOwn(lowerer, declare, own, args, builtinAsWant.type, calleeText, loc); + return { + kind: "ternary", + cond: resolved?.has ?? ownEntryTest(lowerer, o, shapeId, method, loc), + then: called, + else_: builtinAsWant, + type: builtinAsWant.type, + loc, + }; +} + +/** Whether the record holds an own entry named `method`. */ +function ownEntryTest( + lowerer: Lowerer, + o: IrExpr, + shapeId: string, + method: string, + loc: SrcLoc, +): IrExpr { + return { + kind: "call", + callee: recordHasOwnHelper(lowerer, shapeId, loc), + args: [o, { kind: "strLit", value: method, type: STRING, loc }], + type: BOOL, + loc, + }; +} + +/** The record's own entry named `method`: its field, or its overflow entry + * (undefined when absent). */ +function ownEntryRead( + lowerer: Lowerer, + o: IrExpr, + shapeId: string, + method: string, + loc: SrcLoc, +): IrExpr { + const shape = lowerer.shapes.get(shapeId)!; + const field = shape.fields.find((f) => f.name === method); + return field ? { kind: "recordGet", obj: o, shapeId, field: method, type: field.type, loc } : { kind: "recordKeyGet", @@ -580,21 +659,6 @@ function callOwnOrBuiltin( type: lowerer.withUndefinedArmOf(shape.indexValue!) ?? shape.indexValue!, loc, }; - const called = callOwn(lowerer, declare, own, args, builtinAsWant.type, calleeText, loc); - return { - kind: "ternary", - cond: { - kind: "call", - callee: recordHasOwnHelper(lowerer, shapeId, loc), - args: [o, { kind: "strLit", value: method, type: STRING, loc }], - type: BOOL, - loc, - }, - then: called, - else_: builtinAsWant, - type: builtinAsWant.type, - loc, - }; } /** Call the own entry `own` (present, of any stored type) with `args`. A @@ -727,6 +791,25 @@ export function lowerRecordInheritedCall( const recvLocal = lowerer.declareHiddenLocal("%protoRecv", receiver.type); stmts.push({ kind: "varDecl", localId: recvLocal.id, init: receiver, loc }); const o = varRef(recvLocal.id, receiver.type, loc); + // `o.m(...)` reads the member before its arguments run: an argument that + // adds, deletes or replaces the own method does not change the callee. + const field = shape.fields.find((f) => f.name === method); + const ownType = field ? field.type : shape.indexValue; + let resolved: { has: IrExpr; own: IrExpr } | undefined; + if (ownType !== undefined && callableType(lowerer, ownType)) { + const shapeId = receiver.type.shapeId; + const hasLocal = lowerer.declareHiddenLocal("%protoHas", BOOL); + stmts.push({ + kind: "varDecl", + localId: hasLocal.id, + init: ownEntryTest(lowerer, o, shapeId, method, loc), + loc, + }); + const read = ownEntryRead(lowerer, o, shapeId, method, loc); + const ownLocal = lowerer.declareHiddenLocal("%protoOwn", read.type); + stmts.push({ kind: "varDecl", localId: ownLocal.id, init: read, loc }); + resolved = { has: varRef(hasLocal.id, BOOL, loc), own: varRef(ownLocal.id, read.type, loc) }; + } const args: IrExpr[] = []; for (const node of call.arguments.nodes) { const arg = lowerer.lowerExpr(node); @@ -744,6 +827,7 @@ export function lowerRecordInheritedCall( want, ts.getTextOfNode(access), loc, + resolved, ); return { kind: "seqExpr", stmts, result, type: result.type, loc }; } @@ -751,7 +835,10 @@ export function lowerRecordInheritedCall( /** `r.toString` and the other Object.prototype methods read by name (not * called) on an index-signature record: an own entry of the member's * function type when the record holds one, else the built-in as a plain - * function value. Null when the checker's member type is not a function. */ + * function value. An own entry of another type than the member's (a + * function of another signature) throws the checked-conversion TypeError: + * the program's types claimed the member. Null when the checker's member + * type is not a function. */ export function lowerRecordInheritedMember( lowerer: Lowerer, receiver: IrExpr, @@ -810,6 +897,15 @@ export function lowerRecordInheritedMember( loc, }; } + } else if (shape.indexValue && callableType(lowerer, shape.indexValue)) { + result = { + kind: "ternary", + cond: ownEntryTest(lowerer, o, shapeId, method, loc), + then: nodeThrowExpr(1, "", OTHER_ARM_FAILURE_MESSAGE, want, loc), + else_: builtin, + type: want, + loc, + }; } return { kind: "seqExpr", stmts, result, type: want, loc }; } diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8362-record-inherited-order.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8362-record-inherited-order.ts.json new file mode 100644 index 000000000..e080ae4d1 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8362-record-inherited-order.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/8362-record-inherited-order.ts" + ], + "diags": [] +} diff --git a/tests/corpus/8362-record-inherited-order.ts b/tests/corpus/8362-record-inherited-order.ts new file mode 100644 index 000000000..33a72eb91 --- /dev/null +++ b/tests/corpus/8362-record-inherited-order.ts @@ -0,0 +1,40 @@ +// Object.prototype's methods on typed records keep JavaScript's evaluation +// order and key conversions. + +// A function-table key that reassigns the receiver reads the table +// evaluated before the key. +let table: Record string> = { a: () => "first" }; +const other: Record string> = { a: () => "second" }; +function key(): string { + table = other; + return "a"; +} +console.log(String(table[key()]())); + +// hasOwnProperty and propertyIsEnumerable with a symbol key on a record of +// functions test the symbol key. +const fns: Record number> = { double: (n) => n * 2 }; +const sym = Symbol("s"); +console.log(fns.hasOwnProperty(sym as unknown as string), fns.propertyIsEnumerable(sym as unknown as string)); + +// A call by name reads the method before its arguments run: an argument +// that adds, replaces or deletes the own method does not change the callee. +type Has = Record boolean>; +function add(r: Has): string { + r.hasOwnProperty = () => true; + return "zzz"; +} +function swap(r: Has): string { + r.hasOwnProperty = (key) => key === "b"; + return "a"; +} +function drop(r: Has): string { + delete r["hasOwnProperty"]; + return "zzz"; +} +const added: Has = {}; +console.log(added.hasOwnProperty(add(added)), added.hasOwnProperty("zzz")); +const swapped: Has = { hasOwnProperty: (key) => key === "a" }; +console.log(swapped.hasOwnProperty(swap(swapped)), swapped.hasOwnProperty("a")); +const removed: Has = { hasOwnProperty: () => true }; +console.log(removed.hasOwnProperty(drop(removed)), removed.hasOwnProperty("zzz")); diff --git a/tests/fixtures/record-inherited/member-signature.ts b/tests/fixtures/record-inherited/member-signature.ts new file mode 100644 index 000000000..c9c65f8ad --- /dev/null +++ b/tests/fixtures/record-inherited/member-signature.ts @@ -0,0 +1,12 @@ +// A dot read of a prototype-named member whose own entry has another +// signature than the member's (tests/harness/record-inherited-member.test.ts). +const plain: Record number> = { double: (x) => x * 2 }; +const builtin = plain.toString; +console.log(typeof builtin); +const numeric: Record number> = { toString: (x: number) => x + 1 }; +try { + const own = numeric.toString; + console.log(typeof own); +} catch (e) { + console.log(e instanceof TypeError, (e as Error).message); +} diff --git a/tests/harness/corpus-costs/8362-record-inherited-order.ts.json b/tests/harness/corpus-costs/8362-record-inherited-order.ts.json new file mode 100644 index 000000000..8bcdf4a18 --- /dev/null +++ b/tests/harness/corpus-costs/8362-record-inherited-order.ts.json @@ -0,0 +1 @@ +{ "plain": 2000, "san": 2000 } diff --git a/tests/harness/record-inherited-member.test.ts b/tests/harness/record-inherited-member.test.ts new file mode 100644 index 000000000..ed14524d2 --- /dev/null +++ b/tests/harness/record-inherited-member.test.ts @@ -0,0 +1,32 @@ +/* Object.prototype's methods on typed records (frontend/lowering/ + * record-inherited.ts): a dot read of a prototype-named member answers the + * built-in when the record has no own entry, and throws the checked + * conversion's TypeError when its own entry is a function of another + * signature than the member's, which the program's types claimed. Node reads + * the own function there, so the program cannot be a corpus case. */ +import { execFile } from "node:child_process"; +import { mkdirSync } from "node:fs"; +import { join } from "node:path"; +import { promisify } from "node:util"; +import { expect, test } from "vitest"; +import { compile } from "@scriptc/compiler"; +import { OTHER_ARM_FAILURE_MESSAGE } from "../../packages/compiler/src/frontend/lowering/coercions/unions.js"; + +const execFileAsync = promisify(execFile); +const repoRoot = join(import.meta.dirname, "../.."); +const entry = join(repoRoot, "tests/fixtures/record-inherited/member-signature.ts"); +const outDir = join(repoRoot, "node_modules/.cache/scriptc-tests/record-inherited-member"); +const sanitize = process.env["SCRIPTC_SAN"] === "1"; + +test( + "a prototype-named member read with an own function of another signature throws", + { timeout: 300_000 }, + async () => { + mkdirSync(outDir, { recursive: true }); + const result = await compile(entry, { outPath: join(outDir, "main"), outDir, sanitize }); + if (!result.ok) + throw new Error(result.diagnostics.map((d) => `${d.code}: ${d.message}`).join("\n")); + const { stdout } = await execFileAsync(result.binaryPath, [], { encoding: "utf8" }); + expect(stdout).toBe(`function\ntrue ${OTHER_ARM_FAILURE_MESSAGE}\n`); + }, +); From c21fc0120180fb89bf6ff2251bba30d6739f3c7b Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Sun, 25 Oct 2026 14:49:55 -0500 Subject: [PATCH 29/44] Key bound alias instances by the declared alias The compiler compiles itself, and a Map keyed by another Map is outside the key domains it compiles (SC2009 in the self-hosted frontend). The Lowerer's table is now keyed by the declared alias reference: each entry lists the instances that instantiate it with their ts-level bindings maps, and the resolver picks the current instance's by identity. --- .../frontend/lowering/generic-functions.ts | 7 ++-- .../compiler/src/frontend/lowering/lowerer.ts | 32 ++++++++++++------- 2 files changed, 24 insertions(+), 15 deletions(-) diff --git a/packages/compiler/src/frontend/lowering/generic-functions.ts b/packages/compiler/src/frontend/lowering/generic-functions.ts index a2eda839a..1769e8562 100644 --- a/packages/compiler/src/frontend/lowering/generic-functions.ts +++ b/packages/compiler/src/frontend/lowering/generic-functions.ts @@ -1041,11 +1041,12 @@ function unifySignatureBindings( iAliasArgs.length === dAliasArgs.length ) { if (tsBindings && declared !== inst) { - let pairs = lowerer.boundAliasInstances.get(tsBindings); - if (!pairs) lowerer.boundAliasInstances.set(tsBindings, (pairs = new Map())); + let entries = lowerer.boundAliasInstances.get(declared); + if (!entries) lowerer.boundAliasInstances.set(declared, (entries = [])); // A body forwarding its own alias (`inner(g)` returning `Catalog` inside an instance) resolves through the enclosing one. - if (!pairs.has(declared)) pairs.set(declared, lowerer.boundAliasTsResolver(inst) ?? inst); + if (!entries.some((entry) => entry.tsBindings === tsBindings)) + entries.push({ tsBindings, instance: lowerer.boundAliasTsResolver(inst) ?? inst }); } dAliasArgs.forEach((da, i) => { const ia = iAliasArgs[i]; diff --git a/packages/compiler/src/frontend/lowering/lowerer.ts b/packages/compiler/src/frontend/lowering/lowerer.ts index 216851c9c..c77741997 100644 --- a/packages/compiler/src/frontend/lowering/lowerer.ts +++ b/packages/compiler/src/frontend/lowering/lowerer.ts @@ -2233,16 +2233,19 @@ export class Lowerer { * the body needs — indexed accesses (`T[K]` needs K's literal key) and * keyed record reads (`o[k]` where k's type is a literal-bound K). */ typeParamTsBindings: Map | null = null; - /** Generic alias references of a call-keyed instance's declared signature - * (`Catalog`), each paired with the same alias as the call's - * resolved signature instantiates it (`Catalog<{ g1: … }, "acme">`), - * keyed by the instance's ts-level bindings (typeParamTsBindings). The - * checker keeps the alias symbolic inside the generic body and offers no - * way to instantiate it with the bound arguments, but alias - * instantiations are interned: the body's `x as Catalog` is the - * very type the signature declares, so the pair recorded during binding - * unification is that re-instantiation. */ - readonly boundAliasInstances = new Map, Map>(); + /** Generic alias references of call-keyed instances' declared signatures + * (`Catalog`), each with the same alias as every call's resolved + * signature instantiates it (`Catalog<{ g1: … }, "acme">`), together with + * that instance's ts-level bindings map (typeParamTsBindings), which every + * scope that lowers the instance installs. The checker keeps the alias + * symbolic inside the generic body and offers no way to instantiate it + * with the bound arguments, but alias instantiations are interned: the + * body's `x as Catalog` is the very type the signature declares, so + * the pair recorded during unification is that re-instantiation. */ + readonly boundAliasInstances = new Map< + ts.Type, + { tsBindings: Map; instance: ts.Type }[] + >(); /** Non-null while an IMPLICIT-ANY instance body lowers (npm-static JS; * see generic-functions.ts): bound param symbol → * the call site's checker type, consulted by typeOf for identifier @@ -9563,10 +9566,15 @@ export class Lowerer { /** The concrete instantiation of a generic alias reference (`Catalog`) the current call-keyed instance's signature declares - * (boundAliasInstances). Inert outside generic function instantiation. */ + * (boundAliasInstances), or null when its signature never instantiated + * that reference. Inert outside generic function instantiation. */ readonly boundAliasTsResolver = (t: ts.Type): ts.Type | null => { if (!this.typeParamTsBindings || t.getAliasTypeArguments().length === 0) return null; - return this.boundAliasInstances.get(this.typeParamTsBindings)?.get(t) ?? null; + const tsBindings = this.typeParamTsBindings; + return ( + this.boundAliasInstances.get(t)?.find((entry) => entry.tsBindings === tsBindings)?.instance ?? + null + ); }; irTypeOf(node: ts.Node): IrType { From aead30fc2635a67d62764fbd1a61147cec16e592 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Sun, 25 Oct 2026 15:10:32 -0500 Subject: [PATCH 30/44] Mark emitted functions sanitize_address only in sanitized builds The sanitized lane's -fsanitize=address link needs the attribute to instrument the emitted functions. Without a sanitizer pass it is not inert: LLVM does not shrink-wrap functions that carry it, so every call saved and restored all of a function's callee-saved registers on entry, including on paths that return before any call, and it blocks speculative loads. The LLVM helper already dropped it for builds that import runtime bitcode (speed); release builds kept it. - The new sanitizeAttributes option (default true for direct emission) is set from --sanitize wherever objectAudit is. - Without it and without a target CPU level, attribute group #0 holds a string attribute LLVM ignores, since a group cannot be empty. --- packages/cli/test/target-cpu.test.ts | 4 +++- .../compiler/src/backend/llvm/emit-module.ts | 20 ++++++++++--------- .../compiler/src/backend/llvm/emitter.test.ts | 13 ++++++++++++ packages/compiler/src/backend/llvm/emitter.ts | 9 +++++++++ packages/compiler/src/backend/llvm/shapes.ts | 5 +++-- packages/compiler/src/index.ts | 3 +++ packages/compiler/src/native/prepare.ts | 2 ++ .../compiler/test/generator-emission.test.ts | 2 +- 8 files changed, 45 insertions(+), 13 deletions(-) diff --git a/packages/cli/test/target-cpu.test.ts b/packages/cli/test/target-cpu.test.ts index ca5243564..5bc6717c3 100644 --- a/packages/cli/test/target-cpu.test.ts +++ b/packages/cli/test/target-cpu.test.ts @@ -50,9 +50,11 @@ test.runIf(process.platform === "linux" && process.arch === "x64")( const expected = `${(await execFileAsync(process.execPath, [entry])).stdout}`; const baseline = await build([]); expect(baseline).not.toContain('"target-cpu"'); + // Only --sanitize builds mark the functions for ASan instrumentation. + expect(baseline).not.toContain("sanitize_address"); expect(await build(["--target-cpu=baseline"])).toBe(baseline); const v3 = await build(["--target-cpu=x86-64-v3"]); - expect(v3).toMatch(/^attributes #0 = \{ sanitize_address "target-cpu"="x86-64-v3" \}$/m); + expect(v3).toMatch(/^attributes #0 = \{ "target-cpu"="x86-64-v3" \}$/m); if (runsV3) expect((await execFileAsync(outPath)).stdout).toBe(expected); // The bootstrap's cached route builds without a level. expect(await build([], true)).toBe(baseline); diff --git a/packages/compiler/src/backend/llvm/emit-module.ts b/packages/compiler/src/backend/llvm/emit-module.ts index af00ebc2c..8145429f1 100644 --- a/packages/compiler/src/backend/llvm/emit-module.ts +++ b/packages/compiler/src/backend/llvm/emit-module.ts @@ -196,15 +196,17 @@ function readRange(dir: string, range: number): { result: ChunkResult; texts: st /** The attribute groups of the emitted functions: #0 for every function, * #1 for coroutine bodies and #2 for out-of-line record clones. Each carries - * the build's `--target-cpu` level. */ + * the build's `--target-cpu` level, and `sanitize_address` in sanitized + * builds. A group needs one attribute: without either, #0 holds a string + * attribute that LLVM ignores. */ function functionAttributes(emitter: LlEmitter, hasNoInlineRecordClone: boolean): string[] { const cpu = targetCpuAttribute(emitter.targetCpu); + const asan = emitter.sanitizeAttributes ? " sanitize_address" : ""; + const base = `${asan}${cpu}` || ` "scriptc-function"`; return [ - `attributes #0 = { sanitize_address${cpu} }`, - ...(emitter.usesCoroutines - ? [`attributes #1 = { sanitize_address presplitcoroutine${cpu} }`] - : []), - ...(hasNoInlineRecordClone ? [`attributes #2 = { noinline sanitize_address${cpu} }`] : []), + `attributes #0 = {${base} }`, + ...(emitter.usesCoroutines ? [`attributes #1 = {${asan} presplitcoroutine${cpu} }`] : []), + ...(hasNoInlineRecordClone ? [`attributes #2 = { noinline${asan}${cpu} }`] : []), ]; } @@ -1179,9 +1181,9 @@ export function emitModuleParts(emitter: LlEmitter): string[] { ); for (const line of emitter.inlineRcTail(flushedDecls)) out.push(line); out.push( - // sanitize_address is inert under the plain pipeline; the sanitized - // lane's -fsanitize=address link activates instrumentation over the - // emitted functions too (the runtime TUs get theirs from clang). + // The sanitized lane's -fsanitize=address link instruments the + // functions marked sanitize_address (the runtime TUs get theirs from + // clang). ...functionAttributes(emitter, hasNoInlineRecordClone), ``, ); diff --git a/packages/compiler/src/backend/llvm/emitter.test.ts b/packages/compiler/src/backend/llvm/emitter.test.ts index 876476cc1..880685bf2 100644 --- a/packages/compiler/src/backend/llvm/emitter.test.ts +++ b/packages/compiler/src/backend/llvm/emitter.test.ts @@ -211,6 +211,19 @@ test("a target CPU level reaches every function attribute group and nothing else expect(emitLlvmModule(module, { targetCpu: "baseline" })).toBe(plain); }); +test("only sanitized builds mark the emitted functions sanitize_address", () => { + const module = moduleFor([call()]); + const sanitized = emitLlvmModule(module); + expect(emitLlvmModule(module, { sanitizeAttributes: true })).toBe(sanitized); + expect(sanitized).toMatch(/^attributes #0 = \{ sanitize_address \}$/m); + const plain = emitLlvmModule(module, { sanitizeAttributes: false }); + expect(plain).not.toContain("sanitize_address"); + // A group needs an attribute: an ignored string attribute stands in. + expect(plain).toMatch(/^attributes #0 = \{ "scriptc-function" \}$/m); + const v3 = emitLlvmModule(module, { sanitizeAttributes: false, targetCpu: "x86-64-v3" }); + expect(v3).toMatch(/^attributes #0 = \{ "target-cpu"="x86-64-v3" \}$/m); +}); + test("inline RC inlines every fast path at speed and keeps box releases as calls in release", () => { for (const boxed of [false, true]) { const module = moduleFor(Array.from({ length: 4 }, call), { boxed }); diff --git a/packages/compiler/src/backend/llvm/emitter.ts b/packages/compiler/src/backend/llvm/emitter.ts index 5569599ee..f1f1fa7d6 100644 --- a/packages/compiler/src/backend/llvm/emitter.ts +++ b/packages/compiler/src/backend/llvm/emitter.ts @@ -511,6 +511,12 @@ export interface LlvmTargetOptions { inlineRc?: InlineRcMode; /** Define the inline RC fast-path helpers `noinline` (sanitized builds). */ rcHelpersNoinline?: boolean; + /** Mark every emitted function `sanitize_address`, which the sanitized + * lane's `-fsanitize=address` link needs to instrument it. Without a + * sanitizer pass the attribute still turns off shrink-wrapping and + * speculative loads, so plain builds pass false. Defaults to true for + * direct emission. */ + sanitizeAttributes?: boolean; /** Define `scr_detached_exit` so the Linux runtime hands the address-space * teardown after `process.exit` to a helper process (`--detached-exit`). * Executables only; the runtime's weak default keeps it off. */ @@ -1269,6 +1275,8 @@ export class LlEmitter { readonly runtimeAbiMarker: boolean; readonly detachedExit: boolean; readonly targetCpu: TargetCpu | undefined; + /** Emitted functions carry `sanitize_address` (LlvmTargetOptions). */ + readonly sanitizeAttributes: boolean; readonly wasmBoundsChecks: boolean; /** Interned string literals: UTF-8 text → { symbol, byte length } — * first-use order, the runtime ABI’s determinism discipline. */ @@ -1971,6 +1979,7 @@ export class LlEmitter { this.runtimeAbiMarker = options.runtimeAbiMarker === true; this.detachedExit = options.detachedExit === true; this.targetCpu = options.targetCpu; + this.sanitizeAttributes = options.sanitizeAttributes !== false; this.wasmBoundsChecks = options.wasmBoundsChecks === true || wasmBoundsChecks({ diff --git a/packages/compiler/src/backend/llvm/shapes.ts b/packages/compiler/src/backend/llvm/shapes.ts index 7498235b0..6d03db222 100644 --- a/packages/compiler/src/backend/llvm/shapes.ts +++ b/packages/compiler/src/backend/llvm/shapes.ts @@ -216,8 +216,9 @@ export interface RegionHooks { classDefinitions(cls: IrClassDef, sizeOf: string): string[]; } -/** Every emitted function/helper carries #0 = { sanitize_address } — see - * the emitter header. */ +/** Every emitted function/helper carries attribute group #0: the build's + * target CPU level, and `sanitize_address` in sanitized builds (see + * emit-module.ts functionAttributes). */ export const FN_ATTRS = "#0"; export { computeTraced } from "../cycle-analysis.js"; diff --git a/packages/compiler/src/index.ts b/packages/compiler/src/index.ts index ffd56a1bf..11b0f6d13 100644 --- a/packages/compiler/src/index.ts +++ b/packages/compiler/src/index.ts @@ -854,6 +854,7 @@ async function prepareExecutableInput( // the pending flag, since their link may not add an unwinder. unwindAbi: buildPlatform === "linux" && outputKind !== "obj", objectAudit: opts.sanitize === true, + sanitizeAttributes: opts.sanitize === true, lowRegion: lowRegionTarget(buildPlatform, opts.sanitize === true), ...(opts.sanitize === true ? { wasmBoundsChecks: true } : {}), ...inlineRcOption(opts.optimization, opts.sanitize === true), @@ -925,6 +926,7 @@ async function prepareExecutableInput( runtimeAbiMarker: useRuntimePack, unwindAbi: buildPlatform === "linux", objectAudit: opts.sanitize === true, + sanitizeAttributes: opts.sanitize === true, lowRegion: lowRegionTarget(buildPlatform, opts.sanitize === true), ...(opts.sanitize === true ? { wasmBoundsChecks: true } : {}), ...inlineRcOption(opts.optimization, opts.sanitize === true), @@ -2357,6 +2359,7 @@ async function compileLibraryTracked( pointerBits: buildPlatform === "wasi" ? 32 : 64, wasi: buildPlatform === "wasi", objectAudit: opts.sanitize === true, + sanitizeAttributes: opts.sanitize === true, ...inlineRcOption(profile.optimization, opts.sanitize === true), }); timing("llvm-emit", { diff --git a/packages/compiler/src/native/prepare.ts b/packages/compiler/src/native/prepare.ts index 11b8f9dd1..749027b80 100644 --- a/packages/compiler/src/native/prepare.ts +++ b/packages/compiler/src/native/prepare.ts @@ -431,6 +431,7 @@ export function prepareNativeExecutable( (outputKind === "exe" || outputKind === "llvm" || outputKind === "asm") && toolchain.target.platform === "linux", objectAudit: options.sanitize === true, + sanitizeAttributes: options.sanitize === true, lowRegion: lowRegionTarget(toolchain.target.platform, options.sanitize === true), functionWorkers: emitWorkers(), forkPrimitives: emitForkPrimitives(), @@ -545,6 +546,7 @@ export function prepareNativeLibrary( pointerBits: toolchain.target.pointerBits, wasi: toolchain.target.platform === "wasi", objectAudit: options.sanitize === true, + sanitizeAttributes: options.sanitize === true, ...inlineRcOption(profile.optimization, options.sanitize === true), }); timing("llvm-emit"); diff --git a/packages/compiler/test/generator-emission.test.ts b/packages/compiler/test/generator-emission.test.ts index 8c43925ea..47e5f16c5 100644 --- a/packages/compiler/test/generator-emission.test.ts +++ b/packages/compiler/test/generator-emission.test.ts @@ -50,7 +50,7 @@ console.log(sum); expect(definition(llvm, "sc_f_range")).not.toContain("@scr_coro_yield_f64("); expect(llvm).not.toContain("@scr_gen_take_out_f64("); expect(llvm).toContain("%ScrGenHead = type { i64, ptr, ptr, i32, %ScrGenSlot }"); - expect(llvm).toContain("attributes #1 = { sanitize_address presplitcoroutine }"); + expect(llvm).toContain("attributes #1 = { presplitcoroutine }"); expect(definition(llvm, "sc_gs_range")).toContain("call ptr @scr_gen_new_stackless("); expect(llvm).not.toContain("@scr_gen_yield_f64("); }); From a4ae5f9e27f89e238ca24d6dd91273674a7ba092 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Sun, 25 Oct 2026 15:17:20 -0500 Subject: [PATCH 31/44] Run the linker job directly for host Linux links with lld A Linux executable link runs `clang -fuse-ld=lld ...`, and the driver only starts ld.lld with a job it derives from its arguments, its environment and the installed toolchain. Starting the driver costs about 10 ms of a small program's 29 ms link. LinkerJobs (native/linker-job.ts) records that job in the native cache, with the link's own inputs and output as positions. A traced link (the Linux executable cache's `-v -Wl,-t` link) records it from the job line the driver prints with -v; a link the executable cache does not trace lists it with -v -### first. The entry also keeps the identities of the files and directories the job and the driver's -v lines name (ld.lld, startup objects, library directories, the GCC installation, the driver's directory): the set the Linux executable cache records from its traced link. It is keyed by the driver's identity, the link's arguments, the toolchain environment, PATH and the working directory. A later link with the same key whose recorded paths still have those identities runs ld.lld with the job directly, and the executable cache traces it with the job's -t output and the recorded identities. Every other link runs through the driver as before: cross-target links, links without lld, a driver that is not clang, SCRIPTC_LINKER, sanitizer links, a toolchain environment that names mutable inputs, a listing without exactly one ld.lld job naming each of the link's paths once, a recorded path whose identity changed, and a job that fails (the driver then reports the error). The linker gets the same arguments, so the executables are identical. --- packages/compiler/src/native/cache.ts | 1 + packages/compiler/src/native/compiler.ts | 8 +- .../src/native/executable-cache.test.ts | 42 +- .../compiler/src/native/executable-cache.ts | 28 +- .../compiler/src/native/linker-job.test.ts | 408 ++++++++++++++++++ packages/compiler/src/native/linker-job.ts | 286 ++++++++++++ 6 files changed, 760 insertions(+), 13 deletions(-) create mode 100644 packages/compiler/src/native/linker-job.test.ts create mode 100644 packages/compiler/src/native/linker-job.ts diff --git a/packages/compiler/src/native/cache.ts b/packages/compiler/src/native/cache.ts index e204b8a6b..0250e9618 100644 --- a/packages/compiler/src/native/cache.ts +++ b/packages/compiler/src/native/cache.ts @@ -168,6 +168,7 @@ export class NativeCache { "sanitizer", "executable", "kept", + "linker-job", "runtime-pack", "binary", "dsym", diff --git a/packages/compiler/src/native/compiler.ts b/packages/compiler/src/native/compiler.ts index ff49e2ac9..0b56b0a53 100644 --- a/packages/compiler/src/native/compiler.ts +++ b/packages/compiler/src/native/compiler.ts @@ -70,6 +70,7 @@ import { prepareNativeExecutable, prepareNativeLibrary } from "./prepare.js"; import { buildSanitizedRuntime, sanitizerDriver, sanitizerFlags } from "./sanitizer.js"; import { lldLinkerArgs } from "../backend/lld.js"; import { openNativeExecutableCache } from "./executable-cache.js"; +import { LinkerJobs } from "./linker-job.js"; import { nativeFileIdentity } from "./file-identity.js"; /** The first partition keeps the whole-object key used before partitioning. */ @@ -530,11 +531,14 @@ export class NativeCompiler { "-o", stagedOutput, ]; - const cacheable = executableCache?.trace(linkArgs, stage) ?? false; + // Host Linux links with lld run the linker job the driver would. + const jobs = LinkerJobs.open(this.cache, toolchain, lld, options.sanitize ?? false); + const job = jobs?.link([...plan.inputs, stagedOutput], stage); + const cacheable = executableCache?.trace(linkArgs, stage, job) ?? false; timing("link-inputs"); // A successful dependency trace already produced this executable. // Keep its verified output instead of asking the linker to repeat it. - if (!cacheable) { + if (!cacheable && (job?.run(linkArgs, true) ?? null) === null) { const driver = options.sanitize ? sanitizerDriver(toolchain) : toolchain.linker; try { runNativeTool(driver, linkArgs); diff --git a/packages/compiler/src/native/executable-cache.test.ts b/packages/compiler/src/native/executable-cache.test.ts index 8bec9916e..4f950e5b5 100644 --- a/packages/compiler/src/native/executable-cache.test.ts +++ b/packages/compiler/src/native/executable-cache.test.ts @@ -15,7 +15,7 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, expect, test, vi } from "vitest"; import { contentDigest, NativeCache } from "./cache.js"; -import { NativeExecutableCache } from "./executable-cache.js"; +import { identity, NativeExecutableCache } from "./executable-cache.js"; vi.mock("node:child_process", async (original) => ({ ...(await original()), @@ -331,3 +331,43 @@ test("a Linux build that keeps its output does not read the deferred inputs", () expect(open().restore(f.destination)).toBe(true); expect(deferred).toHaveBeenCalledTimes(3); }); + +test("a Linux link that runs as its linker job is traced from the job's run, and otherwise records it", () => { + const f = fixture(); + const lib = join(f.root, "lib"); + mkdirSync(lib); + const crt = join(lib, "Scrt1.o"); + writeFileSync(crt, "startup object"); + const program = join(f.stage, "program.o"); + writeFileSync(program, "program"); + const job = { + run: vi.fn((_args: string[], _list: boolean) => ({ + output: [crt, program].join("\n") + "\n", + driverInputs: [identity(crt), identity(lib)], + })), + record: vi.fn(), + }; + const open = () => new NativeExecutableCache(f.cache, f.key, "clang", false, [f.runtime], true); + const entry = open(); + expect(entry.trace(["-o", f.output], f.stage, job)).toBe(true); + // The job runs the traced link, and the driver does not. + expect(job.run.mock.calls[0]).toEqual([["-o", f.output, "-v", "-Wl,-t"], false]); + expect(spawnSync).not.toHaveBeenCalled(); + expect(job.record).not.toHaveBeenCalled(); + entry.recordOutput(f.output); + const recorded = ( + JSON.parse(f.cache.read("kept", f.key)!.toString()) as { inputs: { path: string }[] } + ).inputs.map((saved) => saved.path); + expect(recorded).toEqual(expect.arrayContaining([crt, lib])); + expect(recorded).not.toContain(program); + expect(open().keep(f.output)).toBe(true); + // A link without a job runs through the driver, and records the job from + // the driver's output. + const record = vi.fn(); + expect(open().trace(["-o", f.output], f.stage, { run: () => null, record })).toBe(true); + expect(spawnSync).toHaveBeenCalledTimes(1); + expect(record).toHaveBeenCalledWith(["-o", f.output, "-v", "-Wl,-t"], f.library + "\n\n", [ + identity(f.sdk), + identity(f.library), + ]); +}); diff --git a/packages/compiler/src/native/executable-cache.ts b/packages/compiler/src/native/executable-cache.ts index bafd2c020..a26bc1d87 100644 --- a/packages/compiler/src/native/executable-cache.ts +++ b/packages/compiler/src/native/executable-cache.ts @@ -14,8 +14,9 @@ import type { NativeOptimization } from "../backend/optimization.js"; import type { NativeRuntimeSelection } from "../backend/runtime-pack-native.js"; import type { NativeToolchain } from "./toolchain.js"; import { contentDigest, NativeCache } from "./cache.js"; +import type { LinkerJobLink } from "./linker-job.js"; -interface InputIdentity { +export interface InputIdentity { path: string; canonical: string; kind: "file" | "directory"; @@ -33,7 +34,7 @@ interface ExecutableEntry { symbols: string | null; } -function identity(path: string): InputIdentity { +export function identity(path: string): InputIdentity { path = resolve(path); const info = statSync(path); if (!info.isFile() && !info.isDirectory()) @@ -63,7 +64,7 @@ function sameIdentity(a: InputIdentity, b: InputIdentity): boolean { ); } -function stillMatches(inputs: InputIdentity[]): boolean { +export function stillMatches(inputs: InputIdentity[]): boolean { try { return inputs.every((input) => sameIdentity(identity(input.path), input)); } catch { @@ -71,7 +72,7 @@ function stillMatches(inputs: InputIdentity[]): boolean { } } -function commandPath(command: string): string { +export function commandPath(command: string): string { if (command.includes("/") || command.includes("\\")) return resolve(command); for (const directory of (process.env["PATH"] ?? "/usr/bin:/bin").split(delimiter)) { const candidate = join(directory || process.cwd(), command); @@ -92,7 +93,7 @@ function toolOutput(executable: string, args: string[]): string { return result.stdout + "\n" + result.stderr; } -function tracePaths(output: string, driver: boolean, roots: string[]): string[] { +export function tracePaths(output: string, driver: boolean, roots: string[]): string[] { const paths = new Set(); for (const line of output.split(/\r?\n/)) { for (const candidate of driver ? driverTraceCandidates(line) : linkTraceCandidate(line)) { @@ -134,7 +135,7 @@ interface KeptEntry { inputs: InputIdentity[]; } -function validIdentity(input: InputIdentity): boolean { +export function validIdentity(input: InputIdentity): boolean { return ( input !== null && typeof input === "object" && @@ -290,8 +291,10 @@ export class NativeExecutableCache { * Private build inputs already belong to the LLVM/runtime cache identity. * A true result means the requested output was linked successfully and its * inputs remained stable. The caller can publish that output directly. On - * false, the caller links the output itself. */ - trace(args: string[], stage: string): boolean { + * false, the caller links the output itself. On Linux, `job` runs the + * traced link as its recorded linker job when it can, and otherwise records + * the job the driver ran (see LinkerJobs). */ + trace(args: string[], stage: string, job?: LinkerJobLink): boolean { this.traced = false; try { this.addDeferredInputs(); @@ -300,8 +303,13 @@ export class NativeExecutableCache { ? [] : tracePaths(toolOutput(this.linker, [...args, "-###"]), true, roots).map(identity); // On Linux, -v prints the jobs that -### would, while the link runs. - const linked = toolOutput(this.linker, [...args, ...(this.linux ? ["-v"] : []), "-Wl,-t"]); - if (this.linux) driverInputs = tracePaths(linked, true, roots).map(identity); + const traced = [...args, ...(this.linux ? ["-v"] : []), "-Wl,-t"]; + const direct = this.linux ? (job?.run(traced, false) ?? null) : null; + const linked = direct?.output ?? toolOutput(this.linker, traced); + if (this.linux) { + driverInputs = direct?.driverInputs ?? tracePaths(linked, true, roots).map(identity); + if (direct === null) job?.record(traced, linked, driverInputs); + } const paths = tracePaths(linked, false, roots); if (paths.length === 0 || !stillMatches(this.inputs) || !stillMatches(driverInputs)) return false; diff --git a/packages/compiler/src/native/linker-job.test.ts b/packages/compiler/src/native/linker-job.test.ts new file mode 100644 index 000000000..b9d718c03 --- /dev/null +++ b/packages/compiler/src/native/linker-job.test.ts @@ -0,0 +1,408 @@ +import { spawnSync } from "node:child_process"; +import { + chmodSync, + mkdirSync, + mkdtempSync, + readFileSync, + renameSync, + rmSync, + statSync, + utimesSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, beforeEach, expect, test } from "vitest"; +import { + LINUX_ARM64_GNU_TARGET, + LINUX_X64_GNU_TARGET, + MACOS_ARM64_TARGET, + type NativeTargetSpec, +} from "../backend/targets.js"; +import { NativeCache } from "./cache.js"; +import { LinkerJobs } from "./linker-job.js"; +import type { NativeToolchain } from "./toolchain.js"; + +const directories: string[] = []; +const saved = new Map(); +const variables = [ + "SCRIPTC_LINKER", + "COMPILER_PATH", + "LOG", + "LD", + "CRT", + "LIB", + "FAIL", + "BROKEN", + "DOUBLE", +]; +beforeEach(() => { + for (const name of variables) { + saved.set(name, process.env[name]); + delete process.env[name]; + } +}); +afterEach(() => { + for (const [name, value] of saved) { + if (value === undefined) delete process.env[name]; + else process.env[name] = value; + } + saved.clear(); + for (const path of directories.splice(0)) rmSync(path, { recursive: true, force: true }); +}); + +function script(path: string, text: string): string { + writeFileSync(path, text); + chmodSync(path, 0o755); + return path; +} + +/** A driver named `name` and an ld.lld that log their arguments to $LOG. The + * driver's job runs $LD with `-pie`, the startup object $CRT, `-L$LIB` and + * its own arguments (without `-fuse-ld=lld` and `-v`, `-Wl,` removed). With + * -### it prints that job as clang does, every argument quoted; with -v it + * prints the job it runs as clang does, only the program quoted. With + * $BROKEN, -### prints the job twice; with $DOUBLE, it names the path $DOUBLE + * once more. The linker writes its inputs to its output, prints them with + * `-t`, and fails with $FAIL. */ +function fixture(name = "clang") { + const root = mkdtempSync(join(tmpdir(), "scriptc-linker-job-")); + directories.push(root); + const bin = join(root, "bin"); + const lib = join(root, "lib"); + // The recorded inputs include the parents of the toolchain's paths (here + // `root`); stages are elsewhere, as a build's are. + const stages = join(root, "stages"); + for (const directory of [bin, lib, stages]) mkdirSync(directory); + const log = join(root, "log"); + writeFileSync(log, ""); + const driver = script( + join(bin, name), + `#!/bin/sh +echo "driver $*" >> "$LOG" +job=" \\"$LD\\" \\"-pie\\" \\"$CRT\\" \\"-L$LIB\\"" +raw=" \\"$LD\\" -pie $CRT -L$LIB" +listing=0; verbose=0 +for a; do + case "$a" in + -'###') listing=1 ;; + -v) verbose=1 ;; + -fuse-ld=lld) ;; + -Wl,*) job="$job \\"\${a#-Wl,}\\""; raw="$raw \${a#-Wl,}" ;; + *) job="$job \\"$a\\""; raw="$raw $a" ;; + esac +done +if [ $listing = 1 ]; then + echo "InstalledDir: $(dirname "$0")" >&2 + [ -z "$DOUBLE" ] && echo "$job" >&2 + [ -n "$BROKEN" ] && echo "$job" >&2 + [ -n "$DOUBLE" ] && echo "$job \\"$DOUBLE\\"" >&2 && exit 0 + exit 0 +fi +[ $verbose = 1 ] && echo "$raw" >&2 +eval "exec $job" +`, + ); + const ld = script( + join(bin, "ld.lld"), + `#!/bin/sh +echo "ld.lld $*" >> "$LOG" +[ -n "$FAIL" ] && exit 1 +out=""; prev="" +for a; do [ "$prev" = "-o" ] && out="$a"; prev="$a"; done +printf '%s\\n' "$@" > "$out" +for a; do [ "$a" = "-t" ] && printf '%s\\n' "$@" | grep '^/'; done +exit 0 +`, + ); + const crt = join(lib, "Scrt1.o"); + writeFileSync(crt, "startup object"); + Object.assign(process.env, { LOG: log, LD: ld, CRT: crt, LIB: lib }); + // A directory's identity is its stat: an old time makes a later change visible. + utimesSync(lib, 0, 0); + const cache = new NativeCache(join(root, "cache")); + const toolchain = (target: NativeTargetSpec = LINUX_X64_GNU_TARGET, helper = target) => + ({ + target, + helper: helper.helper, + linker: driver, + }) as unknown as NativeToolchain; + let count = 0; + /** A stage with a program object, the link's paths and its driver arguments. */ + const stage = () => { + const directory = join(stages, `stage-${++count}`); + mkdirSync(directory); + const program = join(directory, "program.o"); + const output = join(directory, "out"); + writeFileSync(program, "program"); + return { + directory, + output, + paths: [program, output], + args: ["-fuse-ld=lld", "-pthread", program, "-lm", "-o", output], + }; + }; + const calls = () => + readFileSync(log, "utf8") + .split("\n") + .filter((line) => line !== ""); + return { root, bin, lib, crt, ld, driver, cache, toolchain, stage, calls }; +} + +test.skipIf(process.platform === "win32")( + "a link records the driver's job once, and later links run ld.lld with their own paths", + () => { + const f = fixture(); + const jobs = LinkerJobs.open(f.cache, f.toolchain(), ["-fuse-ld=lld"], false)!; + expect(jobs).not.toBeNull(); + const first = f.stage(); + const run = jobs.run(first.args, first.paths, first.directory, true); + expect(run).not.toBeNull(); + // The driver only listed the job; ld.lld linked. + expect(f.calls()).toEqual([ + `driver ${first.args.join(" ")} -v -###`, + `ld.lld -pie ${f.crt} -L${f.lib} -pthread ${first.paths[0]} -lm -o ${first.output}`, + ]); + expect(readFileSync(first.output, "utf8")).toContain(first.paths[0]); + const recorded = run!.driverInputs.map((input) => input.path); + expect(recorded).toEqual(expect.arrayContaining([f.ld, f.bin, f.crt, f.lib])); + expect(recorded.some((path) => path.startsWith(first.directory))).toBe(false); + // Another build, with other paths, runs the recorded job without the driver. + const second = f.stage(); + const again = LinkerJobs.open(f.cache, f.toolchain(), ["-fuse-ld=lld"], false)!; + expect(again.run(second.args, second.paths, second.directory, true)).not.toBeNull(); + expect(f.calls().slice(2)).toEqual([ + `ld.lld -pie ${f.crt} -L${f.lib} -pthread ${second.paths[0]} -lm -o ${second.output}`, + ]); + // Other arguments are another job. + const third = f.stage(); + expect( + again.run([...third.args, "-Wl,-t"], third.paths, third.directory, true)?.output, + ).toContain(third.paths[0]); + expect(f.calls().slice(3)[0]).toBe(`driver ${third.args.join(" ")} -Wl,-t -v -###`); + }, +); + +test.skipIf(process.platform === "win32")( + "a driver link records its job from its -v output, and later links run it", + () => { + const f = fixture(); + const open = () => LinkerJobs.open(f.cache, f.toolchain(), ["-fuse-ld=lld"], false)!; + const first = f.stage(); + const traced = [...first.args, "-v", "-Wl,-t"]; + // Without an entry, a link that is not listed runs through the driver. + const link = open().link(first.paths, first.directory); + expect(link.run(traced, false)).toBeNull(); + expect(f.calls()).toEqual([]); + const linked = spawnSync(f.driver, traced, { encoding: "utf8" }); + expect(linked.status).toBe(0); + link.record(traced, linked.stdout + "\n" + linked.stderr); + const second = f.stage(); + const run = open() + .link(second.paths, second.directory) + .run([...second.args, "-v", "-Wl,-t"], false); + expect(run?.output).toContain(second.paths[0]); + expect(run!.driverInputs.map((input) => input.path)).toEqual( + expect.arrayContaining([f.ld, f.bin, f.crt, f.lib]), + ); + expect(f.calls().slice(2)).toEqual([ + `ld.lld -pie ${f.crt} -L${f.lib} -pthread ${second.paths[0]} -lm -o ${second.output} -t`, + ]); + }, +); + +test.skipIf(process.platform === "win32")( + "a changed startup object, library directory or linker sends the link to the driver", + () => { + for (const change of ["crt", "lib", "ld"] as const) { + const f = fixture(); + const open = () => LinkerJobs.open(f.cache, f.toolchain(), ["-fuse-ld=lld"], false)!; + const first = f.stage(); + expect(open().run(first.args, first.paths, first.directory, true)).not.toBeNull(); + if (change === "lib") writeFileSync(join(f.lib, "libm.so"), "new library"); + else { + const path = change === "crt" ? f.crt : f.ld; + const before = statSync(path); + writeFileSync(path + ".new", readFileSync(path)); + chmodSync(path + ".new", 0o755); + utimesSync(path + ".new", before.atime, before.mtime); + renameSync(path + ".new", path); + } + const second = f.stage(); + const count = f.calls().length; + expect(open().run(second.args, second.paths, second.directory, true)).toBeNull(); + expect(f.calls().length).toBe(count); + } + }, +); + +test.skipIf(process.platform === "win32")("a changed driver records the job again", () => { + const f = fixture(); + const first = f.stage(); + expect( + LinkerJobs.open(f.cache, f.toolchain(), ["-fuse-ld=lld"], false)!.run( + first.args, + first.paths, + first.directory, + true, + ), + ).not.toBeNull(); + writeFileSync(f.driver, readFileSync(f.driver, "utf8") + "\n"); + const second = f.stage(); + expect( + LinkerJobs.open(f.cache, f.toolchain(), ["-fuse-ld=lld"], false)!.run( + second.args, + second.paths, + second.directory, + true, + ), + ).not.toBeNull(); + expect(f.calls()[2]).toBe(`driver ${second.args.join(" ")} -v -###`); +}); + +test.skipIf(process.platform === "win32")( + "a job that fails sends this and the build's later links to the driver", + () => { + const f = fixture(); + const first = f.stage(); + const open = () => LinkerJobs.open(f.cache, f.toolchain(), ["-fuse-ld=lld"], false)!; + expect(open().run(first.args, first.paths, first.directory, true)).not.toBeNull(); + process.env["FAIL"] = "1"; + const jobs = open(); + const second = f.stage(); + expect(jobs.run(second.args, second.paths, second.directory, true)).toBeNull(); + const count = f.calls().length; + expect(jobs.run(second.args, second.paths, second.directory, true)).toBeNull(); + expect(f.calls().length).toBe(count); + }, +); + +test.skipIf(process.platform === "win32")( + "a listing without one ld.lld job naming each path once keeps the driver, and is not asked again", + () => { + for (const listing of ["two jobs", "another linker", "a path twice"] as const) { + const f = fixture(); + if (listing === "two jobs") process.env["BROKEN"] = "1"; + if (listing === "another linker") + process.env["LD"] = script(join(f.bin, "ld.bfd"), readFileSync(f.ld, "utf8")); + const open = () => LinkerJobs.open(f.cache, f.toolchain(), ["-fuse-ld=lld"], false)!; + const first = f.stage(); + if (listing === "a path twice") process.env["DOUBLE"] = first.paths[0]; + expect(open().run(first.args, first.paths, first.directory, true)).toBeNull(); + expect(f.calls()).toEqual([`driver ${first.args.join(" ")} -v -###`]); + // The answer is stored: the next build does not list the job again. + expect(open().run(first.args, first.paths, first.directory, true)).toBeNull(); + expect(f.calls().length).toBe(1); + delete process.env["BROKEN"]; + delete process.env["DOUBLE"]; + } + }, +); + +test.skipIf(process.platform === "win32")( + "arguments that name the stage besides the link's paths keep the driver", + () => { + const f = fixture(); + const first = f.stage(); + const jobs = LinkerJobs.open(f.cache, f.toolchain(), ["-fuse-ld=lld"], false)!; + const script = `-Wl,--version-script=${join(first.directory, "exports")}`; + expect(jobs.run([...first.args, script], first.paths, first.directory, true)).toBeNull(); + expect(f.calls()).toEqual([]); + }, +); + +test.skipIf(process.platform === "win32")( + "only host Linux links through a clang driver with lld run the job", + () => { + const f = fixture(); + const lld = ["-fuse-ld=lld"]; + expect(LinkerJobs.open(f.cache, f.toolchain(), lld, false)).not.toBeNull(); + // A cross-target link: the arm64 target with the x64 host's helper. + expect( + LinkerJobs.open( + f.cache, + f.toolchain(LINUX_ARM64_GNU_TARGET, LINUX_X64_GNU_TARGET), + lld, + false, + ), + ).toBeNull(); + expect( + LinkerJobs.open( + f.cache, + f.toolchain(LINUX_X64_GNU_TARGET, LINUX_ARM64_GNU_TARGET), + lld, + false, + ), + ).toBeNull(); + expect(LinkerJobs.open(f.cache, f.toolchain(MACOS_ARM64_TARGET), lld, false)).toBeNull(); + // The driver's default linker, a sanitizer link, no cache. + expect(LinkerJobs.open(f.cache, f.toolchain(), [], false)).toBeNull(); + expect(LinkerJobs.open(f.cache, f.toolchain(), lld, true)).toBeNull(); + expect(LinkerJobs.open(null, f.toolchain(), lld, false)).toBeNull(); + // A linker that is not a clang driver. + const gcc = fixture("gcc"); + expect(LinkerJobs.open(gcc.cache, gcc.toolchain(), lld, false)).toBeNull(); + // An explicit linker, or an environment that names mutable toolchain inputs. + process.env["SCRIPTC_LINKER"] = f.driver; + expect(LinkerJobs.open(f.cache, f.toolchain(), lld, false)).toBeNull(); + delete process.env["SCRIPTC_LINKER"]; + process.env["COMPILER_PATH"] = f.bin; + expect(LinkerJobs.open(f.cache, f.toolchain(), lld, false)).toBeNull(); + }, +); + +const hostTarget = + process.arch === "x64" + ? LINUX_X64_GNU_TARGET + : process.arch === "arm64" + ? LINUX_ARM64_GNU_TARGET + : null; +const hasLld = + process.platform === "linux" && + spawnSync("clang", ["-fuse-ld=lld", "-###", "-x", "c", "/dev/null"], { + encoding: "utf8", + }).stderr?.includes("ld.lld") === true; + +test.skipIf(hostTarget === null || !hasLld)("the job links the executable the driver links", () => { + const root = mkdtempSync(join(tmpdir(), "scriptc-linker-job-clang-")); + directories.push(root); + const stage = join(root, "stage"); + mkdirSync(stage); + const program = join(stage, "program.o"); + const compiled = spawnSync("clang", ["-c", "-x", "c", "-", "-o", program], { + input: '#include \nint main(void) { puts("linked"); return 0; }\n', + }); + expect(compiled.status).toBe(0); + const toolchain = { + target: hostTarget, + helper: hostTarget!.helper, + linker: "clang", + } as unknown as NativeToolchain; + const args = (output: string) => ["-fuse-ld=lld", "-pthread", program, "-lm", "-o", output]; + const viaDriver = join(stage, "driver"); + expect(spawnSync("clang", args(viaDriver)).status).toBe(0); + const cache = new NativeCache(join(root, "cache")); + // Recorded from the -v output of a traced link, and listed with -###. + const traced = (output: string) => [...args(output), "-v", "-Wl,-t"]; + const tracedOutput = join(stage, "traced"); + const linked = spawnSync("clang", traced(tracedOutput), { encoding: "utf8" }); + expect(linked.status).toBe(0); + LinkerJobs.open(cache, toolchain, ["-fuse-ld=lld"], false)! + .link([program, tracedOutput], stage) + .record(traced(tracedOutput), linked.stdout + "\n" + linked.stderr); + for (const [name, list] of [ + ["first", false], + ["second", true], + ["third", true], + ] as const) { + const output = join(stage, name); + const link = LinkerJobs.open(cache, toolchain, ["-fuse-ld=lld"], false)!.link( + [program, output], + stage, + ); + const run = list ? link.run(args(output), true) : link.run(traced(output), false); + expect(run).not.toBeNull(); + expect(readFileSync(output).equals(readFileSync(viaDriver))).toBe(true); + expect(spawnSync(output, { encoding: "utf8" }).stdout).toBe("linked\n"); + } +}); diff --git a/packages/compiler/src/native/linker-job.ts b/packages/compiler/src/native/linker-job.ts new file mode 100644 index 000000000..e1195e4c4 --- /dev/null +++ b/packages/compiler/src/native/linker-job.ts @@ -0,0 +1,286 @@ +import { spawnSync } from "node:child_process"; +import { realpathSync } from "node:fs"; +import { basename, extname, isAbsolute, resolve } from "node:path"; +import { + toolchainEnvironmentCachePolicy, + toolchainEnvironmentFingerprint, +} from "../backend/toolchain-environment.js"; +import type { NativeToolchain } from "./toolchain.js"; +import { contentDigest, type NativeCache } from "./cache.js"; +import { + commandPath, + identity, + type InputIdentity, + stillMatches, + tracePaths, + validIdentity, +} from "./executable-cache.js"; + +/** A recorded linker job. In `job`, a number is the index of one of the link's + * own paths (its inputs and output); `inputs` are the identities of the + * other files and directories the job and the driver's `-v` lines name. */ +interface LinkerJobEntry { + schema: "scriptc.linker-job.v1"; + /** null when the driver's job could not be recorded: the link runs through + * the driver. */ + job: (string | number)[] | null; + inputs: InputIdentity[]; +} + +/** A link that ran as its recorded job: the linker's output, and the + * identities the job depends on besides the link's own paths. */ +export interface LinkerJobRun { + output: string; + driverInputs: InputIdentity[]; +} + +/** One link's use of LinkerJobs, with its own paths and stage. */ +export interface LinkerJobLink { + run(args: string[], list: boolean): LinkerJobRun | null; + record(args: string[], output: string, driverInputs?: InputIdentity[]): void; +} + +const MAX_ENTRY_BYTES = 1024 * 1024; + +function validEntry(value: unknown, paths: number): value is LinkerJobEntry { + if (value === null || typeof value !== "object") return false; + const entry = value as Partial; + if (entry.schema !== "scriptc.linker-job.v1" || !Array.isArray(entry.inputs)) return false; + if (!entry.inputs.every(validIdentity)) return false; + if (entry.job === null) return true; + if (!Array.isArray(entry.job) || typeof entry.job[0] !== "string") return false; + const used = new Set(); + for (const token of entry.job) { + if (typeof token === "string") continue; + if (!Number.isInteger(token) || token < 0 || token >= paths || used.has(token)) return false; + used.add(token); + } + return used.size === paths; +} + +function underRoots(text: string, roots: string[]): boolean { + return roots.some((root) => text.includes(root)); +} + +/** The arguments of one job line as the driver prints them: with `-###` + * each argument is quoted, with `-v` only the program and the arguments that + * contain a space, `"`, `\` or `$`; a quoted argument escapes `"`, `\` and + * `$` with `\`. Null for any other line. */ +function jobArguments(line: string): string[] | null { + const pattern = /\s*(?:"((?:\\.|[^"\\])*)"|([^\s"\\]+))/y; + const args: string[] = []; + let end = 0; + let match: RegExpExecArray | null; + while ((match = pattern.exec(line)) !== null) { + args.push(match[1] === undefined ? match[2]! : match[1].replace(/\\(.)/g, "$1")); + end = pattern.lastIndex; + } + return args.length === 0 || line.slice(end).trim() !== "" ? null : args; +} + +/** + * Linux links through a clang driver with lld. The driver only starts ld.lld + * with a job it derives from its arguments, its environment, its own + * installation and the system's (startup objects, the GCC installation, + * library directories). The job is recorded once per driver identity, + * arguments and environment, with the identities of the files and + * directories it and the driver's `-v` lines name. A later link whose recorded + * paths still have those identities runs ld.lld with the job directly; every + * other link runs through the driver. + */ +export class LinkerJobs { + /** A job run that failed in this build: later links run through the + * driver, which reports the error. */ + private failed = false; + + private constructor( + private readonly cache: NativeCache, + private readonly driver: string, + private readonly driverIdentity: InputIdentity, + ) {} + + /** Null unless the link is a host Linux link through a clang driver with + * `-fuse-ld=lld` (`lld`, the link's lld arguments), outside the sanitizer, + * with a native cache and a toolchain environment that allows complete + * artifacts. */ + static open( + cache: NativeCache | null, + toolchain: NativeToolchain, + lld: readonly string[], + sanitize: boolean, + ): LinkerJobs | null { + const target = toolchain.target; + if ( + cache === null || + sanitize || + target.platform !== "linux" || + // A cross-target link keeps the driver. + target.llvmTriple !== toolchain.helper.defaultTarget || + !lld.includes("-fuse-ld=lld") || + process.env["SCRIPTC_LINKER"] !== undefined || + !toolchainEnvironmentCachePolicy().completeArtifacts + ) + return null; + try { + const driver = commandPath(toolchain.linker); + if (!/^clang(-\d+)?$/.test(basename(realpathSync(driver)))) return null; + return new LinkerJobs(cache, driver, identity(driver)); + } catch { + return null; + } + } + + /** The link whose own inputs and output are `paths`, in `stage`. */ + link(paths: readonly string[], stage: string): LinkerJobLink { + return { + run: (args, list) => this.run(args, paths, stage, list), + record: (args, output, driverInputs) => this.record(args, paths, stage, output, driverInputs), + }; + } + + /** The entry key of the link `args` with its own `paths`, or null when + * the arguments name the stage besides those paths. */ + private key(args: readonly string[], paths: readonly string[], roots: string[]): string | null { + const index = new Map(paths.map((path, i) => [path, i])); + if (index.size !== paths.length || !paths.every((path) => args.includes(path))) return null; + if (args.some((arg) => !index.has(arg) && underRoots(arg, roots))) return null; + const shape = args.map((arg) => { + const at = index.get(arg); + return at === undefined ? arg : { path: at, extension: extname(arg) }; + }); + return contentDigest( + JSON.stringify({ + schema: 1, + driver: this.driverIdentity, + args: shape, + environment: toolchainEnvironmentFingerprint(), + cwd: process.cwd(), + path: process.env["PATH"] ?? null, + }), + ); + } + + /** + * Runs the link that the driver arguments `args` describe as its recorded + * job. `paths` are the link's own inputs and output among `args`, which may + * differ from build to build; `stage` is the directory they are in. Without + * an entry, `list` records the job first with the driver's `-v -###`; + * otherwise the caller links through the driver and passes its `-v` output + * to record(). Null when the link has to run through the driver: no job is + * recorded, a recorded path changed, or the job failed. + */ + run( + args: readonly string[], + paths: readonly string[], + stage: string, + list: boolean, + ): LinkerJobRun | null { + if (this.failed) return null; + try { + const roots = [resolve(stage), realpathSync(stage)]; + const key = this.key(args, paths, roots); + if (key === null) return null; + let entry = this.read(key, paths.length); + if (entry === null && list) { + const result = spawnSync(this.driver, [...args, "-v", "-###"], { + encoding: "utf8", + stdio: "pipe", + }); + if (result.error !== undefined || result.status !== 0 || result.signal !== null) + return null; + entry = this.store(key, result.stdout + "\n" + result.stderr, paths, roots); + } + if (entry === null || entry.job === null || !stillMatches(entry.inputs)) return null; + const job = entry.job.map((token) => (typeof token === "number" ? paths[token]! : token)); + const result = spawnSync(job[0]!, job.slice(1), { + encoding: "utf8", + stdio: "pipe", + }); + if (result.error !== undefined || result.status !== 0 || result.signal !== null) { + this.failed = true; + return null; + } + return { output: result.stdout + "\n" + result.stderr, driverInputs: entry.inputs }; + } catch { + return null; + } + } + + private read(key: string, paths: number): LinkerJobEntry | null { + const bytes = this.cache.read("linker-job", key, MAX_ENTRY_BYTES); + if (bytes === null) return null; + try { + const entry: unknown = JSON.parse(bytes.toString("utf8")); + return validEntry(entry, paths) ? entry : null; + } catch { + return null; + } + } + + /** Records the job of the link `args` from `output`, the `-v` output of + * the driver's successful link, unless an entry exists. `driverInputs` + * are the identities of the paths `output` names, when the caller has + * them. */ + record( + args: readonly string[], + paths: readonly string[], + stage: string, + output: string, + driverInputs?: InputIdentity[], + ): void { + try { + const roots = [resolve(stage), realpathSync(stage)]; + const key = this.key(args, paths, roots); + if (key !== null && this.read(key, paths.length) === null) + this.store(key, output, paths, roots, driverInputs); + } catch { + /* The next link records it. */ + } + } + + /** Stores the entry for `output`, the driver's `-v` lines and job. */ + private store( + key: string, + output: string, + paths: readonly string[], + roots: string[], + driverInputs?: InputIdentity[], + ): LinkerJobEntry { + const lines = output.split(/\r?\n/).filter((line) => line.startsWith(' "')); + const job = lines.length === 1 ? this.jobOf(jobArguments(lines[0]!), paths, roots) : null; + const entry: LinkerJobEntry = { + schema: "scriptc.linker-job.v1", + job, + inputs: job === null ? [] : (driverInputs ?? tracePaths(output, true, roots).map(identity)), + }; + this.cache.write("linker-job", key, JSON.stringify(entry)); + return entry; + } + + /** `job` with each of `paths` as its index: null unless the job runs + * ld.lld and names each of the paths once, as one whole argument, and names + * nothing else in the stage. */ + private jobOf( + job: string[] | null, + paths: readonly string[], + roots: string[], + ): (string | number)[] | null { + if (job === null || job.length === 0) return null; + if (!isAbsolute(job[0]!) || basename(job[0]!) !== "ld.lld") return null; + const index = new Map(paths.map((path, i) => [path, i])); + const used = new Set(); + const tokens: (string | number)[] = []; + for (const arg of job) { + const at = index.get(arg); + if (at === undefined) { + if (underRoots(arg, roots)) return null; + tokens.push(arg); + } else { + if (used.has(at)) return null; + used.add(at); + tokens.push(at); + } + } + return used.size === paths.length ? tokens : null; + } +} From e842da2ceb7bcb8e7c914e2d20203624497ff06a Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Sun, 25 Oct 2026 15:24:38 -0500 Subject: [PATCH 32/44] Batch the program's identifier symbols before collection --- .../compiler/src/frontend/lowering/lowerer.ts | 23 +++++++++++++++---- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/packages/compiler/src/frontend/lowering/lowerer.ts b/packages/compiler/src/frontend/lowering/lowerer.ts index 2a04b33c3..d272b7014 100644 --- a/packages/compiler/src/frontend/lowering/lowerer.ts +++ b/packages/compiler/src/frontend/lowering/lowerer.ts @@ -5221,11 +5221,11 @@ export class Lowerer { const sourceFiles = parts.map((p) => p.sf); const fnDecls = parts.flatMap((p) => p.fnDecls); // This fixed-point scan intentionally sees deferred bodies before - // reachability is known. Batch their binding queries through the - // facade's panic fence, then keep lookup free of lowering side effects: - // resolveValueSymbol also flushes deferred diagnostics and applies - // merged-namespace fences, whose authority remains ordinary lowering. - this.checker.prefetchSymbolRoots(sourceFiles, true); + // reachability is known. Its binding queries were batched through the + // facade's panic fence before collection (prefetchProgramSymbols); keep + // lookup free of lowering side effects: resolveValueSymbol also flushes + // deferred diagnostics and applies merged-namespace fences, whose + // authority remains ordinary lowering. // The facts each fixed-point site reads, by the symbols, nodes and // element keys they belong to (created with the site list below). let siteDependencies: SiteDependencies | null = null; @@ -8027,6 +8027,7 @@ export class Lowerer { if (this.remainder) { for (const fp of parts) this.checker.prefetchSourceFile(fp.sf); } + this.prefetchProgramSymbols(parts); this.collectProgram(parts); this.speculativeTypeQueries(() => { extendAssertedRecordMembers( @@ -8486,6 +8487,7 @@ export class Lowerer { this.onEdge = (name) => { collectionEdges.add(name); }; + this.prefetchProgramSymbols(parts); this.collectProgram(parts); this.speculativeTypeQueries(() => { extendAssertedRecordMembers( @@ -12183,6 +12185,17 @@ export class Lowerer { return lowerClassExpressionInfo(this, expr); } + /** Batch-prefetches the symbols of every runtime identifier of the + * program's files. Collection's early-binding scan and the runtime- + * optional analysis look them up across every body, reached or not; + * one batch answers both, instead of one request per identifier. */ + private prefetchProgramSymbols(parts: FileParts[]): void { + this.checker.prefetchSymbolRoots( + parts.map((fp) => fp.sf), + true, + ); + } + /** Depth of the whole-program analyses that query types across every * file before reachability is known (speculativeTypeQueries). */ private speculativeTypeQueryDepth = 0; From 11de08b93116e4758c4d66d114fd859d4ca2f606 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Sun, 25 Oct 2026 15:46:27 -0500 Subject: [PATCH 33/44] Skip worker cancellation polls in short loops inside another loop In a worker program, a loop inside another loop of the same function needs less than its own budget, because the enclosing loop polls at least once every 64 of its iterations: - A loop whose counters' proven ranges bound its runs, and those of the loops in its body, to 64 x 64 emits no poll. This covers loops with constant bounds, such as mandelbrot's lane and iteration loops. The bound uses the counted and two-counter loop ranges, or the range of a counter that the update alone steps by a constant where the condition reads it. Loops whose counted form needs a run-time guard don't count. - An innermost counted loop is strip-mined like a counted loop over typed arrays, whatever its body. A run of at most 64 iterations never polls, and the budget's countdown leaves the body. A run of at most 16384 iterations is one block and polls once as it leaves. A longer run polls between blocks of 64, but not after its last block. Between two polls there are then at most 64 iterations of the enclosing loop, and in each of them at most 16384 unpolled iterations of each strip-mined inner loop and 64 x 64 runs of each bounded nest. A loop that no other loop of its function encloses polls as before. New corpus program: native-worker-terminate-bounded-loops.ts. --- .../src/backend/llvm/emit-statements.ts | 142 +++++++++++++++++- packages/compiler/src/backend/loop-polls.ts | 4 +- packages/compiler/src/ir/integer-loops.ts | 22 +++ .../test/counted-loop-emission.test.ts | 91 +++++++++++ ...ive-worker-terminate-bounded-loops.ts.json | 6 + .../native-worker-terminate-bounded-loops.ts | 91 +++++++++++ ...ive-worker-terminate-bounded-loops.ts.json | 1 + 7 files changed, 349 insertions(+), 8 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/native-worker-terminate-bounded-loops.ts.json create mode 100644 tests/corpus/native-worker-terminate-bounded-loops.ts create mode 100644 tests/harness/corpus-costs/native-worker-terminate-bounded-loops.ts.json diff --git a/packages/compiler/src/backend/llvm/emit-statements.ts b/packages/compiler/src/backend/llvm/emit-statements.ts index a43cf3534..e0db764e8 100644 --- a/packages/compiler/src/backend/llvm/emit-statements.ts +++ b/packages/compiler/src/backend/llvm/emit-statements.ts @@ -50,6 +50,7 @@ import { matchIntegerBytesForLoop, matchIntegerCountedForLoop, matchIntegerPairedForLoop, + steppedLoopCounter, } from "../../ir/integer-loops.js"; import { emitByteWindowGuard, matchByteWindow } from "./byte-windows.js"; import { @@ -1218,7 +1219,24 @@ export function emitStmtBody(emitter: LlEmitter, s: IrStmt): void { // has no exit besides its own bound, which LLVM can vectorize. A typed // array's length is fixed, so each block can take it as its limit when // the loop does not assign the array's binding. + // + // Inside another loop of this function, which polls at least once + // every WORKER_POLL_BLOCK of its own iterations, a loop needs less: + // - a loop whose counters' ranges bound its runs, and those of the + // loops in its body, to WORKER_POLL_BLOCK^2 needs no poll; + // - an innermost counted loop is strip-mined, typed arrays or not, + // and does not poll after its last block. A run of at most + // WORKER_POLL_BLOCK iterations then never polls, and the countdown + // leaves its body. A run of at most WORKER_UNPOLLED_RUN iterations + // is one block, polled once as it leaves. + const enclosed = + emitter.mod.workers === true && + emitter.jumpTargets.some((target) => target.kind === "loop"); + const bounded = + enclosed && boundedLoopRuns(emitter, s) <= WORKER_POLL_BLOCK * WORKER_POLL_BLOCK; + const innermost = enclosed && !containsLoop(s.body); const stripMined = + !bounded && capturedLets.length === 0 && integerSlot !== null && ((countedLoop !== null && @@ -1226,16 +1244,21 @@ export function emitStmtBody(emitter: LlEmitter, s: IrStmt): void { Math.abs(countedLoop.step) * WORKER_POLL_BLOCK <= 2 ** 52) || (integerLoop !== null && integerArrayLoop === null && integerLoop.stableReceiver)) && emitter.workerLoopPolls(s) && - loopAccessesTypedArrays(s); - const workerBudget = stripMined ? null : emitter.workerLoopBudget(s); + (loopAccessesTypedArrays(s) || (countedLoop !== null && innermost)); + const workerBudget = stripMined || bounded ? null : emitter.workerLoopBudget(s); // One slot for every copy of the loop: a speculative copy resumes an // iteration in another copy's body, which then reads this bound. let blockBound: string | null = null; + let runSlot: string | null = null; if (stripMined) { blockBound = B.slot(); B.entryAllocas.push( `${blockBound} = alloca ${countedLoop ? "i64" : emitter.sizeType} ; cancellation poll block bound`, ); + if (innermost) { + runSlot = B.slot(); + B.entryAllocas.push(`${runSlot} = alloca i1 ; cancellation poll after the run`); + } } freshenBindings(); emitter.emitLoopCopies(s, capturedLets.length === 0, le, (exit) => { @@ -1271,7 +1294,7 @@ export function emitStmtBody(emitter: LlEmitter, s: IrStmt): void { return len; }, }; - emitStripMinedCountedLoop(emitter, s, counter, blockBound!, exit); + emitStripMinedCountedLoop(emitter, s, counter, blockBound!, exit, runSlot); return; } const lc = B.newLabel("loop.c"); @@ -2628,6 +2651,51 @@ function emitInt32KindLoop( /** Iterations between cancellation polls of a strip-mined counted loop. */ const WORKER_POLL_BLOCK = 64; +/** The longest run of a strip-mined loop inside another loop that polls + * only as it leaves. */ +const WORKER_UNPOLLED_RUN = 16384; + +/** At most how many times a `for` loop and the loops in its body run their + * bodies per entry, from the counters' proven ranges: its iterations times + * one plus the nested loops' runs. Infinity when a count has no bound, or + * depends on a run-time guard (the guard's slow copy steps a double). */ +function boundedLoopRuns(emitter: LlEmitter, loop: IrStmt): number { + if (loop.kind !== "for") return Infinity; + const counted = matchIntegerCountedForLoop( + loop, + emitter.numericLocals, + emitter.integerRanges, + LOOPS.of(emitter).unaliasedArrays, + ); + let trips = Infinity; + if (counted && !counted.guarded) + trips = Math.floor((counted.range.max - counted.range.min) / Math.abs(counted.step)) + 1; + else if (!counted) { + // Each iteration closes the gap between the two counters by at least two. + const paired = matchIntegerPairedForLoop(loop, emitter.numericLocals, emitter.integerRanges); + if (paired && !paired.guarded) + trips = Math.floor((paired.range.max - paired.range.min) / 2) + 1; + } + if (trips === Infinity) { + // A counter with a proven range where the condition reads it, such as + // one compared with a module constant. + const stepped = steppedLoopCounter(loop, emitter.currentLocals); + const range = stepped && emitter.integerRanges.get(stepped.ref); + if (stepped && range) trips = Math.floor((range.max - range.min) / Math.abs(stepped.step)) + 1; + } + if (!(trips <= WORKER_POLL_BLOCK * WORKER_POLL_BLOCK)) return Infinity; + let nested = 0; + const visit = (s: IrStmt): boolean => { + if (s.kind === "while" || s.kind === "doWhile" || s.kind === "for" || s.kind === "forOf") { + nested += boundedLoopRuns(emitter, s); + return nested !== Infinity; + } + return everyStmtChild(s, () => true, visit); + }; + if (!loop.body.every(visit)) return Infinity; + return trips * (1 + nested); +} + /** The counter of a strip-mined loop: its slot, the LLVM integer type and * signedness of its values, its step, the counters stepped beside it, and * the limit, emitted in the outer header before each block. */ @@ -2645,13 +2713,20 @@ interface StripMinedCounter { * tests the counter against the loop's limit, each block runs the body up * to the nearer of the limit and WORKER_POLL_BLOCK steps on, and the poll * follows the block. It polls as often as the budget it replaces, and the - * inner loop's only exit besides `break` and `return` is its bound. */ + * inner loop's only exit besides `break` and `return` is its bound. + * + * With `runSlot` (for a loop that another loop of the function encloses), a + * block that ends at the limit leaves the loop without polling, and a run + * of at most WORKER_UNPOLLED_RUN iterations is one block, polled once as + * it leaves when it took more than WORKER_POLL_BLOCK iterations. `runSlot` + * holds whether the current block is such a run. */ function emitStripMinedCountedLoop( emitter: LlEmitter, s: Extract, counter: StripMinedCounter, boundSlot: string, exit: string, + runSlot: string | null, ): void { const B = emitter.B; const { slot, type, signed, step } = counter; @@ -2684,7 +2759,27 @@ function emitStripMinedCountedLoop( `${stepped} = call ${type} @${saturating}(${type} ${start}, ${type} ${step * WORKER_POLL_BLOCK})`, ); B.line(`${bound} = call ${type} @${nearer}(${type} ${stepped}, ${type} ${limit})`); - B.line(`store ${type} ${bound}, ptr ${boundSlot}`); + if (runSlot === null) B.line(`store ${type} ${bound}, ptr ${boundSlot}`); + else { + // The steps left to the limit: positive, as the header tested. + const left = B.tmp(), + whole = B.tmp(), + long = B.tmp(), + polled = B.tmp(), + end = B.tmp(); + const stride = Math.abs(step); + B.line( + ascending + ? `${left} = sub ${wrap} ${type} ${limit}, ${start}` + : `${left} = sub ${wrap} ${type} ${start}, ${limit}`, + ); + B.line(`${whole} = icmp ule ${type} ${left}, ${stride * WORKER_UNPOLLED_RUN}`); + B.line(`${long} = icmp ugt ${type} ${left}, ${stride * WORKER_POLL_BLOCK}`); + B.line(`${polled} = and i1 ${whole}, ${long}`); + B.line(`${end} = select i1 ${whole}, ${type} ${limit}, ${type} ${bound}`); + B.line(`store ${type} ${end}, ptr ${boundSlot}`); + B.line(`store i1 ${polled}, ptr ${runSlot}`); + } B.br(bc); B.startBlock(bc); const index = B.tmp(); @@ -2693,7 +2788,8 @@ function emitStripMinedCountedLoop( B.line(`${index} = load ${type}, ptr ${slot}`); B.line(`${end} = load ${type}, ptr ${boundSlot}`); B.line(`${inBlock} = icmp ${compare} ${type} ${index}, ${end}`); - B.condBr(inBlock, lb, poll); + const after = runSlot === null ? poll : B.newLabel("loop.end"); + B.condBr(inBlock, lb, after); B.startBlock(lb); emitter.jumpTargets.push({ kind: "loop", @@ -2716,11 +2812,45 @@ function emitStripMinedCountedLoop( B.line(`store ${type} ${next}, ptr ${stepSlot}`); } B.br(bc); + if (runSlot !== null) { + // A speculative copy may enter the body without passing the header, so + // the limit is read again here. + B.startBlock(after); + const at = B.tmp(); + const more = B.tmp(); + const run = B.tmp(); + const done = B.newLabel("loop.done"); + const last = B.newLabel("loop.poll.run"); + B.line(`${at} = load ${type}, ptr ${slot}`); + B.line(`${more} = icmp ${compare} ${type} ${at}, ${counter.limit()}`); + B.condBr(more, poll, done); + B.startBlock(done); + B.line(`${run} = load i1, ptr ${runSlot}`); + B.condBr(run, last, exit); + B.startBlock(last); + emitter.emitWorkerPoll(); + B.br(exit); + } B.startBlock(poll); emitter.emitWorkerPoll(); B.br(lc); } +/** Whether `body` holds a loop outside its closures. */ +function containsLoop(body: IrStmt[]): boolean { + const inStmt = (s: IrStmt): boolean => + s.kind === "while" || + s.kind === "doWhile" || + s.kind === "for" || + s.kind === "forOf" || + !everyStmtChild( + s, + () => true, + (child) => !inStmt(child), + ); + return body.some(inStmt); +} + function enterLoopHeader(emitter: LlEmitter, header: string): void { const dispatch = emitter.speculationHeader; emitter.speculationHeader = null; diff --git a/packages/compiler/src/backend/loop-polls.ts b/packages/compiler/src/backend/loop-polls.ts index c582a4974..866711fb7 100644 --- a/packages/compiler/src/backend/loop-polls.ts +++ b/packages/compiler/src/backend/loop-polls.ts @@ -110,8 +110,8 @@ export function loopTestsEveryIteration(loop: Loop, tested: (call: IrExpr) => bo /** True when the body of `loop`, outside its nested loops and closures, * reads or writes a typed-array element: the loops whose cancellation poll * the emitter moves between blocks of iterations, so that LLVM can - * vectorize them. Other loops keep the budget in their body, which costs - * less in a loop of a few iterations. */ + * vectorize them. Outside another loop, other loops keep the budget in + * their body, which costs less in a loop of a few iterations. */ export function loopAccessesTypedArrays(loop: Loop): boolean { const inExpr = (e: IrExpr): boolean => { if (e.kind === "bytesIntrinsic" && e.method === "get") return true; diff --git a/packages/compiler/src/ir/integer-loops.ts b/packages/compiler/src/ir/integer-loops.ts index ea866a134..427f6879c 100644 --- a/packages/compiler/src/ir/integer-loops.ts +++ b/packages/compiler/src/ir/integer-loops.ts @@ -618,6 +618,28 @@ function writesLocal(body: IrStmt[], localId: string): boolean { }); } +/** The counter of a `for` loop that its condition reads and its update + * steps by a constant, when nothing else in the loop writes it: each test + * of the condition then sees a different value. The local must not be + * boxed (a closure could write it). */ +export function steppedLoopCounter( + stmt: IrStmt & { kind: "for" }, + locals: ReadonlyMap, +): { ref: IrExpr & { kind: "varRef" }; step: number } | null { + const cond = stmt.cond; + if (cond?.kind !== "bin") return null; + for (const side of [cond.left, cond.right]) { + if (side.kind !== "varRef") continue; + const local = locals.get(side.localId); + if (local === undefined || local.boxed) continue; + const step = constantStep(stmt.update, side.localId); + if (step === null || writesLocal(stmt.body, side.localId)) continue; + if (writesLocal([{ kind: "exprStmt", expr: cond, loc: cond.loc }], side.localId)) continue; + return { ref: side, step }; + } + return null; +} + function isUnitIncrement(update: IrStmt | null, localId: string): boolean { // The frontend normally lowers `i++` to `i = i + 1` before backend // emission. Accept the incDec form too so this analysis remains valid for diff --git a/packages/compiler/test/counted-loop-emission.test.ts b/packages/compiler/test/counted-loop-emission.test.ts index f617c6ab4..794edd1af 100644 --- a/packages/compiler/test/counted-loop-emission.test.ts +++ b/packages/compiler/test/counted-loop-emission.test.ts @@ -136,3 +136,94 @@ test("worker programs poll counted loops over typed arrays between blocks of ite await rm(dir, { recursive: true, force: true }); } }); + +test("worker programs poll no loop whose proven ranges bound its runs inside another loop", async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-bounded-loop-polls-")); + try { + const entry = join(dir, "bounded.ts"); + await writeFile( + entry, + [ + 'import { isMainThread, Worker } from "node:worker_threads";', + "const LIMIT = 50;", + "export function rows(cr: Float64Array, n: number): number {", + " let t = 0;", + " for (let r = 0; r < n; r++)", + " for (let i = 0; i < 8; i += 2) {", + " const c = cr[i]!;", + " for (let j = 0; j < LIMIT; j++) {", + " t = (t * 31 + j + c) % 1000003;", + " if (t === 7) break;", + " }", + " }", + " return t;", + "}", + "export function open(cr: Float64Array, n: number, k: number): number {", + " let t = 0;", + " for (let r = 0; r < n; r++)", + " for (let i = 0; i < 8; i += 2) {", + " const c = cr[i]!;", + " for (let j = 0; j < k; j++) {", + " t = (t * 31 + j + c) % 1000003;", + " if (t === 7) break;", + " }", + " }", + " return t;", + "}", + "export function wide(n: number): number {", + " let t = 0;", + " for (let r = 0; r < n; r++) for (let j = 0; j < 5000; j++) t = (t * 31 + j) % 1000003;", + " return t;", + "}", + "export function single(): number {", + " let t = 0;", + " for (let j = 0; j < 50; j++) t = (t * 31 + j) % 1000003;", + " return t;", + "}", + "const cr = new Float64Array(8).fill(0.5);", + 'const k = Number(process.argv[2] ?? "50");', + "console.log(rows(cr, 3), open(cr, 3, k), wide(2), single());", + "if (!isMainThread) new Worker(new URL(import.meta.url));", + ].join("\n"), + ); + const outPath = join(dir, "bounded.ir.json"); + const result = await compile(entry, { outPath, outDir: dir, outputKind: "ir" }); + if (!result.ok) throw new Error(result.diagnostics.map((d) => d.message).join("\n")); + const ll = emitLlvmModule(deserializeModule(await readFile(outPath, "utf8"))); + const body = (name: string): string => + [ + ...ll.matchAll( + new RegExp( + `define internal [^\\n]+ @sc_b?f_${name}(?:\\.i32)?\\([^\\n]*\\n([\\s\\S]*?)\\n\\}`, + "g", + ), + ), + ] + .map((match) => match[1]!) + .join("\n"); + const polls = (text: string): number => text.match(/@scr_exc_flag/g)?.length ?? 0; + // The i and j loops of `rows` run at most 4 × (1 + 50) bodies: only the + // r loop polls, and the i loop over the typed array is not strip-mined. + const rows = body("rows"); + expect(rows).not.toMatch(/^loop\.poll\d+:$/m); + expect(polls(rows)).toBeGreaterThan(0); + // With a limit known only at run time the inner loops keep their polls. + // The innermost one is strip-mined, and a block that ends at the limit + // leaves it without polling: only a block that ends short of it polls. + const open = body("open"); + expect(polls(open)).toBeGreaterThan(polls(rows)); + expect(open).toMatch(/^loop\.poll\d+:$/m); + const ends = [...open.matchAll(/^loop\.end\d+:$[\s\S]*?^\S/gm)].map((match) => match[0]); + expect(ends.length).toBeGreaterThan(0); + for (const end of ends) expect(end).toMatch(/br i1 %\w+, label %loop\.poll\d+, label %/); + // A run of up to 16384 iterations is one block, polled as it leaves when + // it took more than 64. + expect(open).toMatch(/^loop\.poll\.run\d+:$/m); + // A nest over the bound, or a loop no other loop encloses, polls. + expect(polls(body("wide"))).toBeGreaterThan(0); + expect(body("wide")).toMatch(/^loop\.poll\d+:$/m); + expect(polls(body("single"))).toBeGreaterThan(0); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/native-worker-terminate-bounded-loops.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/native-worker-terminate-bounded-loops.ts.json new file mode 100644 index 000000000..2e52a519c --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/native-worker-terminate-bounded-loops.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/native-worker-terminate-bounded-loops.ts" + ], + "diags": [] +} diff --git a/tests/corpus/native-worker-terminate-bounded-loops.ts b/tests/corpus/native-worker-terminate-bounded-loops.ts new file mode 100644 index 000000000..d244e021d --- /dev/null +++ b/tests/corpus/native-worker-terminate-bounded-loops.ts @@ -0,0 +1,91 @@ +// worker.terminate() stops a worker whose loop runs short inner loops: +// counted loops of a few iterations, a two-counter loop, an inner loop +// whose length grows past 64 iterations, an inner loop too long to finish, +// and a nest of loops with constant bounds. +import { Worker, isMainThread, parentPort, workerData } from "node:worker_threads"; + +function counted(values: number[]): number { + let acc = 0; + for (let round = 0; ; round++) { + for (let i = 0; i < values.length; i++) acc = (acc + values[i]! * i) % 1000003; + for (let i = values.length - 1; i >= 0; i -= 2) acc = (acc * 7 + values[i]!) % 1000003; + if (round === 200) parentPort!.postMessage("ready"); + } + return acc; +} + +function paired(values: number[]): number { + let flips = 0; + for (let round = 0; ; round++) { + for (let i = 0, j = values.length - 1; i < j; i++, j--) { + const t = values[i]!; + values[i] = values[j]!; + values[j] = t; + flips++; + } + if (round === 200) parentPort!.postMessage("ready"); + } + return flips; +} + +function growing(): number { + let acc = 0; + for (let n = 1; ; n++) { + const limit = n % 200; + for (let i = 0; i < limit; i++) acc = (acc + i) % 1000003; + if (n === 200) parentPort!.postMessage("ready"); + } + return acc; +} + +function long(limit: number): number { + let acc = 0; + for (let round = 0; ; round++) { + for (let i = 0; i < limit; i++) { + acc = (acc * 7 + i) % 1000003; + if (i === 100000) parentPort!.postMessage("ready"); + } + } + return acc; +} + +const ROUNDS = 50; + +function constant(row: Float64Array): number { + let acc = 0; + for (let y = 0; ; y++) { + for (let i = 0; i < 8; i += 2) { + const c = row[i]!; + for (let j = 0; j < ROUNDS; j++) { + acc = (acc * 3 + j + c) % 1000003; + if (acc === -1) break; + } + } + if (y === 200) parentPort!.postMessage("ready"); + } + return acc; +} + +if (isMainThread) { + for (const kind of ["counted", "paired", "growing", "long", "constant"]) { + await new Promise((resolve) => { + const worker = new Worker(new URL(import.meta.url), { workerData: kind }); + worker.once("message", () => { + void worker.terminate().then((code: number) => console.log(kind, "terminate", code)); + }); + worker.once("exit", (code: number) => { + console.log(kind, "exit", code); + resolve(); + }); + }); + } +} else { + const values = [3, 1, 4, 1, 5, 9, 2, 6, 5, 3, 5, 8]; + let result = 0; + if (workerData === "counted") result = counted(values); + else if (workerData === "paired") result = paired(values); + else if (workerData === "growing") result = growing(); + else if (workerData === "long") result = long(1e15); + else result = constant(new Float64Array(8).fill(0.25)); + console.log("unexpected result", result); +} diff --git a/tests/harness/corpus-costs/native-worker-terminate-bounded-loops.ts.json b/tests/harness/corpus-costs/native-worker-terminate-bounded-loops.ts.json new file mode 100644 index 000000000..bb744e4ed --- /dev/null +++ b/tests/harness/corpus-costs/native-worker-terminate-bounded-loops.ts.json @@ -0,0 +1 @@ +{ "plain": 4034, "san": 5956 } From 829f35bf851bd691078b1fa3ed8226e56c7543a0 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Sun, 25 Oct 2026 15:55:57 -0500 Subject: [PATCH 34/44] Pass sanitizeAttributes where the serialized-IR test emits directly The test emits each corpus program from its serialized IR with its own emitter options and compares the text with compile()'s. compile() now marks functions sanitize_address only in sanitized builds, so the test passes the same setting as objectAudit. --- .../compiler/src/backend/llvm/serialized-ir-emission.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/compiler/src/backend/llvm/serialized-ir-emission.test.ts b/packages/compiler/src/backend/llvm/serialized-ir-emission.test.ts index 6d2457c57..1d35caa05 100644 --- a/packages/compiler/src/backend/llvm/serialized-ir-emission.test.ts +++ b/packages/compiler/src/backend/llvm/serialized-ir-emission.test.ts @@ -44,6 +44,7 @@ test("sort-heavy programs emit equal LLVM from memory and from serialized IR", a // compile() emits Linux textual LLVM with the unwinding ABI. unwindAbi: process.platform === "linux", objectAudit: sanitize, + sanitizeAttributes: sanitize, ...inlineRcOption(options.optimization, sanitize), }); const text = typeof emitted === "string" ? emitted : emitted.join(""); From d1f0113e49ce58cb2ea00c38b6521041c4d2a3bd Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Sun, 25 Oct 2026 15:58:14 -0500 Subject: [PATCH 35/44] Check only the files whose diagnostics the tsc gate keeps The tsc gate (preflight's errorsOf, and the original program of a JSX or declaration-projection build) asked for every file's semantic diagnostics, then dropped all of those in an opted-in package's files, in unselected package JavaScript and in package shims. With --npm-static that is nearly the whole check: on three.js's nodes fixture the 82 package files took 1.17 s and reported 13,022 diagnostics, the program's own files 0.1 s. getPreEmitDiagnostics takes the files whose diagnostics its caller keeps. When some file is left out, the server checks only the kept files (getDiagnosticsOfCheckedFiles), and their semantic diagnostics come in the full check's order. When a stored answer for the supplied declaration files applies to the program, it stands in for checking those files, as in the full check; the storage rule is unchanged. A global diagnostic can come from checking any file, so when the partial check reports one, the full check answers as before. At load time, before workspace packages are known, the gate leaves out only package and shim files. The kept diagnostics are unchanged. --- packages/compiler/src/frontend/program.ts | 21 +++++- .../src/frontend/ts7/lib-diagnostics.test.ts | 40 ++++++++++ .../src/frontend/ts7/lib-diagnostics.ts | 37 +++++++++- .../src/frontend/ts7/program-host.test.ts | 55 +++++++++++++- .../compiler/src/frontend/ts7/program-host.ts | 74 ++++++++++++++++--- 5 files changed, 208 insertions(+), 19 deletions(-) diff --git a/packages/compiler/src/frontend/program.ts b/packages/compiler/src/frontend/program.ts index b93e3f217..284b2ce06 100644 --- a/packages/compiler/src/frontend/program.ts +++ b/packages/compiler/src/frontend/program.ts @@ -665,6 +665,14 @@ function createRequireProgramRoots7(program: ts.Program): string[] { return [...roots].sort(); } +/** Whether the tsc gate can keep a file's diagnostics when the program + * loads: it drops all of an opted-in package's files and of the package + * shims (keptErrors in preflight also drops unselected package JavaScript, + * which is not known yet here, so those files are still checked). */ +function tscGateFileKept(fileName: string): boolean { + return npmStaticPackageOfPath(fileName) === null && !isPackageShimFile(fileName); +} + /** Classify the explicitly selected installed package's own sources. */ export function entryPackageFilePredicate(entryPath: string): (file: string) => boolean { const entryFile = tsgoPath(entryPath); @@ -934,7 +942,7 @@ function loadProgram7( const original = program; // The project world must read the original text too, so it is built // before the rewritten files are served, and only when it is needed. - const diagnostics = ts.getPreEmitDiagnostics(original); + const diagnostics = ts.getPreEmitDiagnostics(original, tscGateFileKept); const errors = diagnostics.some((d) => d.category === ts.DiagnosticCategory.Error); tscWorld = { program: original, @@ -982,7 +990,7 @@ function loadProgram7( } else { tscWorld = { program: original, - diagnostics: ts.getPreEmitDiagnostics(original), + diagnostics: ts.getPreEmitDiagnostics(original, tscGateFileKept), projectWorld: null, declWorldFallback: true, }; @@ -5438,7 +5446,14 @@ function preflight7(load: LoadResult): { const sf = p.getSourceFile(d.fileName); return sf !== undefined && isCheckedJsSourceFile(sf, projectCheckJs); }; - const errorsOf = (p: ts.Program): ts.Diagnostic[] => keptErrors(p, ts.getPreEmitDiagnostics(p)); + // keptErrors drops every diagnostic of these files: the gate does not ask + // the checker for them. + const diagnosticsKept = (fileName: string): boolean => + npmStaticPackageOfPath(fileName) === null && + !unselectedPackageJsFile(fileName) && + !isPackageShimFile(fileName); + const errorsOf = (p: ts.Program): ts.Diagnostic[] => + keptErrors(p, ts.getPreEmitDiagnostics(p, diagnosticsKept)); const keptErrors = (p: ts.Program, all: readonly ts.Diagnostic[]): ts.Diagnostic[] => { // First pass: every comment-side 2300's (file, name) — the partners // the second pass forgives. diff --git a/packages/compiler/src/frontend/ts7/lib-diagnostics.test.ts b/packages/compiler/src/frontend/ts7/lib-diagnostics.test.ts index e87db5f1a..83abaeddc 100644 --- a/packages/compiler/src/frontend/ts7/lib-diagnostics.test.ts +++ b/packages/compiler/src/frontend/ts7/lib-diagnostics.test.ts @@ -4,6 +4,7 @@ import { join } from "node:path"; import { afterAll, afterEach, expect, test } from "vitest"; import type { SourceFile } from "./ast-types.js"; import { + reusedKeptLibDiagnostics, reusedLibDiagnostics, setLibDiagnosticsStore, type LibDiagnosticsStore, @@ -118,6 +119,45 @@ test("a stored answer stands in for checking the supplied files, merged by file expect(full?.semantic).toEqual(reused?.semantic); }); +test("with kept files, a stored answer needs checking only the kept other files", () => { + const store = memoryStore(); + setLibDiagnosticsStore(store, "checker"); + const pkg = join(dir, "node_modules", "pkg", "index.d.ts"); + const semantic = new Map([ + [lib, [diagnostic(lib, 1)]], + [fallback, []], + [main, [diagnostic(main, 5)]], + [pkg, [diagnostic(pkg, 3)]], + ]); + const withPackage = files({ + [main]: "export const n: number = 'x';\n", + [pkg]: "export declare const p: number;\n", + }); + const all = [lib, fallback, main, pkg]; + const keep = new Set([lib, fallback, main]); + // Without a stored answer nothing is checked or stored. + const cold = sessionProgram({ semantic }); + expect(reusedKeptLibDiagnostics(cold, all, supplied, withPackage, keep)).toBeNull(); + expect(cold.asked).toEqual([]); + expect(store.entries.size).toBe(0); + reusedLibDiagnostics(sessionProgram({ semantic }), all, supplied, withPackage); + expect(store.entries.size).toBe(1); + const warm = sessionProgram({ semantic }); + expect(reusedKeptLibDiagnostics(warm, all, supplied, withPackage, keep)).toEqual({ + semantic: [diagnostic(lib, 1), diagnostic(main, 5)], + global: [], + }); + expect(warm.asked).toEqual([`checked ${main}`]); + // A left-out file that can declare something global still prevents reuse. + const augmenting = files({ + [main]: "export const n: number = 'x';\n", + [pkg]: "export {};\ndeclare global {\n var NaN: number;\n}\n", + }); + const blocked = sessionProgram({ semantic }); + expect(reusedKeptLibDiagnostics(blocked, all, supplied, augmenting, keep)).toBeNull(); + expect(blocked.asked).toEqual([]); +}); + test("files merge in code point order, which differs from UTF-16 order", () => { const store = memoryStore(); setLibDiagnosticsStore(store, "checker"); diff --git a/packages/compiler/src/frontend/ts7/lib-diagnostics.ts b/packages/compiler/src/frontend/ts7/lib-diagnostics.ts index 30c6e81f0..c3e563fef 100644 --- a/packages/compiler/src/frontend/ts7/lib-diagnostics.ts +++ b/packages/compiler/src/frontend/ts7/lib-diagnostics.ts @@ -64,6 +64,31 @@ export function reusedLibDiagnostics( fileNames: readonly string[], supplied: (fileName: string) => boolean, sourceFile: (fileName: string) => SourceFile | undefined, +): { semantic: Ts7DiagnosticData[]; global: Ts7DiagnosticData[] } | null { + return reuse(program, fileNames, supplied, sourceFile, () => true, false); +} + +/** reusedLibDiagnostics for a caller that keeps only the diagnostics of the + * `kept` files: a stored answer then needs checking only the kept other + * files, and without a stored answer the result is null (nothing is checked + * or stored). */ +export function reusedKeptLibDiagnostics( + program: Ts7SessionProgram, + fileNames: readonly string[], + supplied: (fileName: string) => boolean, + sourceFile: (fileName: string) => SourceFile | undefined, + kept: ReadonlySet, +): { semantic: Ts7DiagnosticData[]; global: Ts7DiagnosticData[] } | null { + return reuse(program, fileNames, supplied, sourceFile, (name) => kept.has(name), true); +} + +function reuse( + program: Ts7SessionProgram, + fileNames: readonly string[], + supplied: (fileName: string) => boolean, + sourceFile: (fileName: string) => SourceFile | undefined, + keep: (fileName: string) => boolean, + storedOnly: boolean, ): { semantic: Ts7DiagnosticData[]; global: Ts7DiagnosticData[] } | null { if (configured === null || process.env["SCRIPTC_LIB_DIAGNOSTICS"] === "0") return null; const { store, checker } = configured; @@ -104,12 +129,16 @@ export function reusedLibDiagnostics( if (globals !== null) writeEntry(store, key, globals, Object.fromEntries(stored.files)); } if (stored !== null && (scripts.length === 0 || (globals !== null && !collides(globals)))) { - const checked = program.checkedFileDiagnosticData(own); + const checkedNames = storedOnly ? own.filter(keep) : own; + const checked = program.checkedFileDiagnosticData(checkedNames); if (checked === null) return null; - const byFile = new Map(stored.files); - own.forEach((name, index) => byFile.set(name, checked.files[index] ?? [])); + const byFile = new Map( + storedOnly ? [...stored.files].filter(([name]) => keep(name)) : stored.files, + ); + checkedNames.forEach((name, index) => byFile.set(name, checked.files[index] ?? [])); return { semantic: inFileOrder(byFile), global: checked.global }; } + if (storedOnly) return null; // The full check, as before. Its answers for the supplied files are stored // when it reports no global diagnostics and the program's scripts declare // none of the supplied files' globals (so nothing of the program reached @@ -339,7 +368,7 @@ function readEntry( /** Each file's diagnostics (already in the checker's order) in the order the * checker sorts files: by file name, compared by code point. */ -function inFileOrder(byFile: ReadonlyMap): Ts7DiagnosticData[] { +export function inFileOrder(byFile: ReadonlyMap): Ts7DiagnosticData[] { const names = [...byFile.keys()].sort(compareCodePoints); const out: Ts7DiagnosticData[] = []; for (const name of names) for (const d of byFile.get(name)!) out.push(d); diff --git a/packages/compiler/src/frontend/ts7/program-host.test.ts b/packages/compiler/src/frontend/ts7/program-host.test.ts index 80c4d2bcf..2ab6a0293 100644 --- a/packages/compiler/src/frontend/ts7/program-host.test.ts +++ b/packages/compiler/src/frontend/ts7/program-host.test.ts @@ -3,7 +3,12 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, expect, test, vi } from "vitest"; import { Ts7Api } from "./rpc-api.js"; -import { Ts7Host, type Ts7ApiOptions, type Ts7CompilerOptions } from "./program-host.js"; +import { + getPreEmitDiagnostics, + Ts7Host, + type Ts7ApiOptions, + type Ts7CompilerOptions, +} from "./program-host.js"; import { ModuleDetectionKind, ModuleKind, ModuleResolutionKind, ScriptTarget } from "./enums.js"; import { tsgoPath } from "./session-path.js"; @@ -70,6 +75,54 @@ test("implementation discovery leaves declaration trees lazy without skipping th expect(() => program.getSourceFileNames()).toThrow("disposed"); }); +test("pre-emit diagnostics for kept files check only those, unless a global diagnostic appears", () => { + const { directory, entry } = fixture(); + const other = join(directory, "other.ts"); + const later = join(directory, "later.ts"); + writeFileSync( + entry, + 'import "./other.js";\nimport "./later.js";\nexport const n: number = "x";\n', + ); + writeFileSync(other, "export const s: string = 1;\n"); + writeFileSync(later, "export const b: boolean = 0;\n"); + const { host } = makeHost(directory); + const program = host.createProgram([entry], options); + try { + const full = getPreEmitDiagnostics(program); + const keep = (name: string): boolean => name !== tsgoPath(other); + const checked = vi.spyOn(program.project.program, "checkedFileDiagnosticData"); + const kept = getPreEmitDiagnostics(program, keep); + // The full answer without the left-out file's diagnostics, in the same order. + expect(full.some((diagnostic) => diagnostic.fileName === tsgoPath(other))).toBe(true); + expect(kept.map((diagnostic) => [diagnostic.fileName, diagnostic.code])).toEqual( + full + .filter((diagnostic) => diagnostic.fileName === undefined || keep(diagnostic.fileName)) + .map((diagnostic) => [diagnostic.fileName, diagnostic.code]), + ); + expect(kept).toHaveLength(2); + expect(checked).toHaveBeenCalledTimes(1); + expect(checked.mock.calls[0]![0]).not.toContain(tsgoPath(other)); + // Keeping every file is the full check. + expect(getPreEmitDiagnostics(program, () => true)).toEqual(full); + expect(checked).toHaveBeenCalledTimes(1); + // A global diagnostic from the partial check, or a server without it: the full check. + for (const answer of [ + { + files: [], + global: [{ fileName: undefined, pos: 0, end: 0, code: 2318, category: 1, text: "global" }], + }, + null, + ]) { + checked.mockReturnValueOnce( + answer as ReturnType, + ); + expect(getPreEmitDiagnostics(program, keep)).toEqual(full); + } + } finally { + program.dispose(); + } +}); + test("the factory receives one connection with the host's filesystem and timing", () => { const { directory, entry } = fixture(); const factory = vi.fn((connection: Ts7ApiOptions) => new Ts7Api(connection)); diff --git a/packages/compiler/src/frontend/ts7/program-host.ts b/packages/compiler/src/frontend/ts7/program-host.ts index 9e8f51bb3..bd6921afc 100644 --- a/packages/compiler/src/frontend/ts7/program-host.ts +++ b/packages/compiler/src/frontend/ts7/program-host.ts @@ -48,7 +48,7 @@ import { import { packageShimDeclarationText } from "../package-shims.js"; import { typePatchedText } from "../type-patches/layer.js"; import { ambientDtsPath, fallbackDtsPath, overridesDtsPath } from "../dts-paths.js"; -import { reusedLibDiagnostics } from "./lib-diagnostics.js"; +import { inFileOrder, reusedKeptLibDiagnostics, reusedLibDiagnostics } from "./lib-diagnostics.js"; import { ts7Diagnostic } from "./diagnostic.js"; import type { Ts7DiagnosticData } from "./session-schema.generated.js"; @@ -423,17 +423,31 @@ export class Ts7Program { /** The semantic and global diagnostics with the stored diagnostics of the * declaration files scriptc supplies (lib-diagnostics.ts), or null. */ reusedLibDiagnostics(): { semantic: Ts7DiagnosticData[]; global: Ts7DiagnosticData[] } | null { + return this.libDiagnostics(null); + } + + /** reusedLibDiagnostics for the `kept` files only: null without a stored + * answer. */ + reusedKeptLibDiagnostics( + kept: ReadonlySet, + ): { semantic: Ts7DiagnosticData[]; global: Ts7DiagnosticData[] } | null { + return this.libDiagnostics(kept); + } + + private libDiagnostics( + kept: ReadonlySet | null, + ): { semantic: Ts7DiagnosticData[]; global: Ts7DiagnosticData[] } | null { this.snapshot.ensureActive(); const program = this.project.program; const declarations = new Set([ambientDtsPath(), overridesDtsPath(), fallbackDtsPath()]); - return reusedLibDiagnostics( - program, - this.getSourceFileNames(), - (name) => - declarations.has(tsgoPath(name)) || - program.getSourceFileMetadataByPath(name)?.isDefaultLibrary === true, - (name) => this.getSourceFile(name), - ); + const names = this.getSourceFileNames(); + const supplied = (name: string): boolean => + declarations.has(tsgoPath(name)) || + program.getSourceFileMetadataByPath(name)?.isDefaultLibrary === true; + const sourceFile = (name: string): SourceFile | undefined => this.getSourceFile(name); + return kept === null + ? reusedLibDiagnostics(program, names, supplied, sourceFile) + : reusedKeptLibDiagnostics(program, names, supplied, sourceFile, kept); } isSourceFileDefaultLibrary(file: SourceFile): boolean { @@ -464,10 +478,35 @@ export class Ts7Program { /** 5.9.3's getPreEmitDiagnostics, composed from 7.0.2's split surface (the * package dropped the aggregate). Order mirrors 5.9.3: config, options/ - * program, global, syntactic, semantic. */ -export function getPreEmitDiagnostics(program: Ts7Program): readonly Diagnostic[] { + * program, global, syntactic, semantic. + * + * With `keep`, a caller that discards every diagnostic of the other files + * asks for semantic diagnostics of the kept files only: the server checks + * only those (getDiagnosticsOfCheckedFiles), or only those not answered by + * stored diagnostics of the supplied declaration files, and their + * diagnostics come in the full check's order. A global diagnostic can come + * from checking any file, so when that partial check reports one, the full + * check answers. */ +export function getPreEmitDiagnostics( + program: Ts7Program, + keep?: (fileName: string) => boolean, +): readonly Diagnostic[] { const config = program.getConfigFileParsingDiagnostics(); const options = program.getProgramDiagnostics(); + if (keep !== undefined) { + const kept = program.getSourceFileNames().filter((name) => keep(name)); + if (kept.length < program.getSourceFileNames().length) { + const partial = + program.reusedKeptLibDiagnostics(new Set(kept)) ?? keptFileDiagnostics(program, kept); + if (partial !== null && partial.global.length === 0) + return [ + ...config, + ...options, + ...program.getSyntacticDiagnostics(), + ...partial.semantic.map(ts7Diagnostic), + ]; + } + } const reused = program.reusedLibDiagnostics(); if (reused !== null) return [ @@ -486,6 +525,19 @@ export function getPreEmitDiagnostics(program: Ts7Program): readonly Diagnostic[ ]; } +/** The semantic diagnostics of the kept files, in the full check's order, + * and the global diagnostics of checking them; null when the server cannot + * check only some files. */ +function keptFileDiagnostics( + program: Ts7Program, + kept: readonly string[], +): { semantic: Ts7DiagnosticData[]; global: Ts7DiagnosticData[] } | null { + const checked = program.project.program.checkedFileDiagnosticData(kept); + if (checked === null) return null; + const byFile = new Map(kept.map((name, index) => [name, checked.files[index] ?? []])); + return { semantic: inFileOrder(byFile), global: checked.global }; +} + /** ts.findConfigFile: nearest configName walking up from searchPath. */ export function findConfigFile( searchPath: string, From a96fbcfb641ec841f1ff5e98f5b8cd855782734e Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Sun, 25 Oct 2026 16:14:48 -0500 Subject: [PATCH 36/44] Refuse static member reads of MIMEType and MIMEParams in JavaScript JavaScript sources hold node:util's MIMEType and MIMEParams as identity tokens, so a read such as MIMEType.prototype or MIMEType.name answered undefined where Node returns the class's member. Those reads now report SC2020, as the TypeScript spelling does at compile time; in JavaScript the refusal throws when the read runs. The node24-mime-errors fixture reads MIMEType.prototype to take the getters' descriptors, so its expected refusal is now SC2020. --- .../compiler/src/frontend/lowering/builtins/mime.ts | 11 +++++++++++ .../frontend/lowering/expressions/property-access.ts | 2 ++ tests/harness/npm.test.ts | 2 +- 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/packages/compiler/src/frontend/lowering/builtins/mime.ts b/packages/compiler/src/frontend/lowering/builtins/mime.ts index 27f95226c..4c36d6cc5 100644 --- a/packages/compiler/src/frontend/lowering/builtins/mime.ts +++ b/packages/compiler/src/frontend/lowering/builtins/mime.ts @@ -134,6 +134,17 @@ export function lowerMimeCall(lowerer: Lowerer, call: ts.CallExpression): IrExpr }; } +/** Reports SC2020 when `access` reads a static member of MIMEType or + * MIMEParams (`MIMEType.prototype`, `MIMEType.name`); null otherwise. + * JavaScript sources hold either class as an identity token, which has no + * members, so the read would answer undefined. MIMEType.parse calls lower + * in lowerMimeCall and never reach the member read. */ +export function fenceMimeStaticRead(lowerer: Lowerer, access: ts.PropertyAccessExpression): null { + const cls = mimeClassOf(lowerer, access.expression); + if (cls !== null) lowerer.noLowering(`util.${cls}.${access.name.text}`, access); + return null; +} + /** `v instanceof MIMEType` / `v instanceof MIMEParams`. */ export function lowerMimeInstanceof(lowerer: Lowerer, expr: ts.BinaryExpression): IrExpr | null { const cls = mimeClassOf(lowerer, expr.right); diff --git a/packages/compiler/src/frontend/lowering/expressions/property-access.ts b/packages/compiler/src/frontend/lowering/expressions/property-access.ts index df62da2c8..cfd7ae297 100644 --- a/packages/compiler/src/frontend/lowering/expressions/property-access.ts +++ b/packages/compiler/src/frontend/lowering/expressions/property-access.ts @@ -63,6 +63,7 @@ import { } from "../../../ir/ir.js"; import { lowerNavigatorMember } from "../builtins/navigator.js"; import { lowerX509PublicKey } from "../builtins/x509.js"; +import { fenceMimeStaticRead } from "../builtins/mime.js"; import { NODE_COMPAT_VERSION } from "../builtins/process.js"; import { inlineNamespaceRequireOf, @@ -942,6 +943,7 @@ export function lowerPropertyAccessExpr( // prototype that no lowering above claimed keeps the SC2020 fence // rather than reading undefined through the field and dynamic reads. fenceBuiltinConstructorStaticRead(lowerer, expr) ?? + fenceMimeStaticRead(lowerer, expr) ?? lowerer.lowerFieldRead(expr); // A union-typed field read narrows like an identifier when the // checker has narrowed this use to one arm. diff --git a/tests/harness/npm.test.ts b/tests/harness/npm.test.ts index d51a2e242..8e133f6e2 100644 --- a/tests/harness/npm.test.ts +++ b/tests/harness/npm.test.ts @@ -74,7 +74,7 @@ const KNOWN: Record = { "node24-file-url-conversion": { build: "SC1090" }, "node24-fs-descriptors": { run: "SC2020" }, "node24-mime": { run: "SC2020" }, - "node24-mime-errors": { run: "SC1090" }, + "node24-mime-errors": { run: "SC2020" }, "node24-no-error-assertions": { run: "SC2020" }, "node24-parseargs": { run: "SC1090" }, "node24-promise-timers": { run: "SC2020" }, From a9ea71df7ddec01a9e0c9984fe3981f4d4b68cf9 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Sun, 25 Oct 2026 16:22:25 -0500 Subject: [PATCH 37/44] Compare unions that hold promises as untyped values A promise converted into a union's promise arm is a new promise, so two optional slots set to one promise each hold their own conversion, and a value narrowed out of such a slot may be one too. Union equality compared those arm payloads by pointer and answered false where Node answers true. Any comparison between a union with a promise arm and a promise or another such union now compares both sides as untyped values, where a converted promise answers with its origin's identity. --- .../frontend/lowering/expressions/binary.ts | 27 ++++++++++--------- .../union-promise-function-arm-widening.ts | 9 +++++++ 2 files changed, 24 insertions(+), 12 deletions(-) diff --git a/packages/compiler/src/frontend/lowering/expressions/binary.ts b/packages/compiler/src/frontend/lowering/expressions/binary.ts index 01b48554e..4e46b4d5f 100644 --- a/packages/compiler/src/frontend/lowering/expressions/binary.ts +++ b/packages/compiler/src/frontend/lowering/expressions/binary.ts @@ -2388,19 +2388,22 @@ export function lowerStrictEquality( loc, }; } - // A plain promise against a union whose promise arm has a different - // payload (`chains.get(id) === tail` with a `Map>`): - // wrapping would convert the promise into that arm, and the converted - // promise is a new object. Promises are objects to ===, so compare both - // sides as untyped values, where a converted promise answers with its - // origin's identity (the plain promise-vs-promise rule below). - const plainPromise = - left.type.kind === "promise" ? left : right.type.kind === "promise" ? right : null; + // A promise against a union with a promise arm, or two such unions: + // converting a promise into a union's promise arm of another payload + // (`chains.get(id) === tail` with a `Map>`, or + // `a.pending === b.pending` after both were set to one promise) builds a + // new promise, so either side may hold a conversion of the other's + // promise, and a value narrowed out of the union may be one too. + // Promises are objects to ===, so compare both sides as untyped values, + // where a converted promise answers with its origin's identity (the plain + // promise-vs-promise rule below). + const holdsPromise = (type: IrType): boolean => + type.kind === "promise" || + (type.kind === "union" && + (lowerer.unions.get(type.unionId)?.arms ?? []).some((arm) => arm.kind === "promise")); if ( - !bothUnion && - plainPromise !== null && - lowerer.armTag(ut.unionId, plainPromise.type) < 0 && - unionArms.some((arm) => arm.kind === "promise") && + holdsPromise(left.type) && + holdsPromise(right.type) && lowerer.dynConvertible(left.type) && lowerer.dynConvertible(right.type) ) { diff --git a/tests/corpus/union-promise-function-arm-widening.ts b/tests/corpus/union-promise-function-arm-widening.ts index 751af6f41..d3b8f618d 100644 --- a/tests/corpus/union-promise-function-arm-widening.ts +++ b/tests/corpus/union-promise-function-arm-widening.ts @@ -58,6 +58,15 @@ async function main(): Promise { console.log(store.pending === other); console.log(await store.pending); + // One promise in two optional slots: each holds its own conversion. + const second: Store = {}; + second.pending = work; + console.log(store.pending === second.pending, store.pending !== second.pending); + const slots: Store[] = [store, second, { pending: other }]; + console.log(slots.map((slot) => slot.pending === slots[0]!.pending).join(",")); + second.lookup = findName; + console.log(store.lookup === second.lookup); + const lookup = store.lookup; if (lookup) console.log(await lookup("abcd"), await lookup("ab")); console.log(store.scale?.(21)); From e3f02c73c9cde475b83fae2db150f2cc5f0af083 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Sun, 25 Oct 2026 16:32:27 -0500 Subject: [PATCH 38/44] Probe hashed number-key maps without normalizing integer keys A number-key Map or Set read whose key has an integer view (sc_map_entry_f64_int) fell back to the same hashed probe as any double key, which tests for NaN and -0 before hashing. The fallback is now sc_map_entry_f64_hashed_int, which takes the integer and converts it to the key word directly: an integer is never NaN, and zero converts to +0. Both hashed probes test the absent-key filter only before scanning a map that keeps no bucket table. The filter keeps two of its 64 bits per key, so it rejects few absent keys once a map has a bucket table, and the probe's first bucket read answers those. The map layout and the runtime lookup are unchanged. Corpus 8331 covers integers the compiler tracks as integers on hashed maps and absent keys on small bucket tables, and matches Node's output. --- .../compiler/src/backend/llvm/emit-module.ts | 254 ++++++++++-------- .../backend/llvm/map-number-lookup.test.ts | 22 +- .../src/backend/llvm/map-number-lookup.ts | 7 +- ...331-map-number-hashed-integer-keys.ts.json | 6 + .../8331-map-number-hashed-integer-keys.ts | 102 +++++++ ...331-map-number-hashed-integer-keys.ts.json | 1 + 6 files changed, 276 insertions(+), 116 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8331-map-number-hashed-integer-keys.ts.json create mode 100644 tests/corpus/8331-map-number-hashed-integer-keys.ts create mode 100644 tests/harness/corpus-costs/8331-map-number-hashed-integer-keys.ts.json diff --git a/packages/compiler/src/backend/llvm/emit-module.ts b/packages/compiler/src/backend/llvm/emit-module.ts index af00ebc2c..66e419fb9 100644 --- a/packages/compiler/src/backend/llvm/emit-module.ts +++ b/packages/compiler/src/backend/llvm/emit-module.ts @@ -1210,6 +1210,126 @@ export function emitModuleParts(emitter: LlEmitter): string[] { return out; } +/** The out-of-line hashed probe of a number-key map (scr_map_find_f64): + * `keyLines` define the normalized key word %kb from the parameter, and + * `runtimeKey` is the double passed to the runtime lookup. A map with at + * most SCR_MAP_LINEAR_LIMIT entries keeps no bucket table: the absent-key + * filter is tested before its entries are scanned. A bucket table is + * probed without the filter: the filter keeps two of its 64 bits per key, + * so past a few dozen keys it rejects few absent keys, and the probe's + * first bucket read answers those. Compact tables tag each 24-bit entry + * index (+1) with the hash's top byte; wider tables take the runtime + * lookup. */ +function hashedNumberProbe( + name: string, + param: string, + sz: string, + keyLines: string[], + runtimeKey: string, +): string[] { + return [ + `define internal ptr @${name}(ptr %m, ${param}) noinline ${FN_ATTRS} {`, + `entry:`, + ...keyLines, + ` %x1s = lshr i64 %kb, 30`, + ` %x1 = xor i64 %kb, %x1s`, + ` %x2 = mul i64 %x1, -4658895280553007687`, + ` %x3s = lshr i64 %x2, 27`, + ` %x3 = xor i64 %x2, %x3s`, + ` %x4 = mul i64 %x3, -7723592293110705685`, + ` %x5s = lshr i64 %x4, 31`, + ` %x5 = xor i64 %x4, %x5s`, + ` %hz = icmp eq i64 %x5, 0`, + ` %h = select i1 %hz, i64 1, i64 %x5`, + ` %nbp = getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 13`, + ...(sz === "i64" + ? [` %nb32 = load i32, ptr %nbp`, ` %nb = zext i32 %nb32 to i64`] + : [` %nb = load i32, ptr %nbp`]), + ` %eps = getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 11`, + ` %ents = load ptr, ptr %eps`, + ` %linear = icmp eq ${sz} %nb, 0`, + ` br i1 %linear, label %filter, label %buckets`, + `filter:`, + ` %f1s = lshr i64 %h, 44`, + ` %f1a = and i64 %f1s, 63`, + ` %f1 = shl i64 1, %f1a`, + ` %f2s = lshr i64 %h, 50`, + ` %f2a = and i64 %f2s, 63`, + ` %f2 = shl i64 1, %f2a`, + ` %fb = or i64 %f1, %f2`, + ` %fp = getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 10`, + ` %filter.bits = load i64, ptr %fp`, + ` %fm = and i64 %filter.bits, %fb`, + ` %fin = icmp eq i64 %fm, %fb`, + ` br i1 %fin, label %scan, label %miss`, + // A number-key entry has no cached hash; a dead entry's key matches no key. + `scan:`, + ` %nep = getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 8`, + ...(sz === "i64" + ? [` %ne32 = load i32, ptr %nep`, ` %ne = zext i32 %ne32 to i64`] + : [` %ne = load i32, ptr %nep`]), + ` br label %scan.loop`, + `scan.loop:`, + ` %si = phi ${sz} [ 0, %scan ], [ %si.next, %scan.next ]`, + ` %sdone = icmp eq ${sz} %si, %ne`, + ` br i1 %sdone, label %miss, label %scan.entry`, + `scan.entry:`, + ` %se = getelementptr inbounds %ScrMapNumEntry, ptr %ents, ${sz} %si`, + ` %sk = load i64, ptr %se`, + ` %skq = icmp eq i64 %sk, %kb`, + ` br i1 %skq, label %scan.found, label %scan.next`, + `scan.found:`, + ` ret ptr %se`, + `scan.next:`, + ` %si.next = add ${sz} %si, 1`, + ` br label %scan.loop`, + `buckets:`, + ` %wide = icmp ugt ${sz} %nb, 16777216`, + ` br i1 %wide, label %runtime, label %probe`, + `probe:`, + ` %bp = getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 14`, + ` %bk = load ptr, ptr %bp`, + ` %mask = sub ${sz} %nb, 1`, + ` %ht = lshr i64 %h, 32`, + ` %ht32 = trunc i64 %ht to i32`, + ` %tag = and i32 %ht32, -16777216`, + sz === "i64" ? ` %hi = add i64 %h, 0` : ` %hi = trunc i64 %h to ${sz}`, + ` %i0 = and ${sz} %hi, %mask`, + ` br label %probe.loop`, + `probe.loop:`, + ` %pi = phi ${sz} [ %i0, %probe ], [ %pi.next, %probe.next ]`, + ` %bep = getelementptr inbounds i32, ptr %bk, ${sz} %pi`, + ` %b = load i32, ptr %bep`, + ` %bempty = icmp eq i32 %b, 0`, + ` br i1 %bempty, label %miss, label %probe.tag`, + `probe.tag:`, + ` %bt = and i32 %b, -16777216`, + ` %btq = icmp eq i32 %bt, %tag`, + ` br i1 %btq, label %probe.entry, label %probe.next`, + `probe.entry:`, + ` %bix = and i32 %b, 16777215`, + ` %bix1 = sub i32 %bix, 1`, + sz === "i32" ? ` %bi = add i32 %bix1, 0` : ` %bi = zext i32 %bix1 to ${sz}`, + ` %pe = getelementptr inbounds %ScrMapNumEntry, ptr %ents, ${sz} %bi`, + ` %pk = load i64, ptr %pe`, + ` %pkq = icmp eq i64 %pk, %kb`, + ` br i1 %pkq, label %probe.found, label %probe.next`, + `probe.found:`, + ` ret ptr %pe`, + `probe.next:`, + ` %pi.inc = add ${sz} %pi, 1`, + ` %pi.next = and ${sz} %pi.inc, %mask`, + ` br label %probe.loop`, + `miss:`, + ` ret ptr null`, + `runtime:`, + ` %r = call ptr @scr_map_entry_f64(ptr %m, double ${runtimeKey}) memory(read) nounwind willreturn`, + ` ret ptr %r`, + `}`, + ``, + ]; +} + /** The shared abort helpers (emitted only when referenced): the OOM * abort of untraced shape allocation and the invalid-union-tag abort — * both print the runtime ABI’s exact message on fd 2 and abort. */ @@ -1349,118 +1469,34 @@ export function helperDefs(emitter: LlEmitter): string[] { ` %dense.any = icmp ne i64 %nd, 0`, ` br i1 %dense.any, label %miss, label %slow`, `slow:`, - ` %hashed = call ptr @sc_map_entry_f64_hashed(ptr %m, double %k)`, + ` %hashed = call ptr @sc_map_entry_f64_hashed_int(ptr %m, i64 %ki)`, ` ret ptr %hashed`, `}`, ``, - // Hashing (scr_map_find_f64): the SameValueZero key word, its - // scr_key_mix hash (zero remapped to one) and the absent-key filter. - `define internal ptr @sc_map_entry_f64_hashed(ptr %m, double %k) noinline ${FN_ATTRS} {`, - `entry:`, - ` %nan = fcmp uno double %k, %k`, - ` %zero = fcmp oeq double %k, 0.0`, - ` %kz = select i1 %zero, double 0.0, double %k`, - ` %kzb = bitcast double %kz to i64`, - ` %kb = select i1 %nan, i64 9221120237041090560, i64 %kzb`, - ` %x1s = lshr i64 %kb, 30`, - ` %x1 = xor i64 %kb, %x1s`, - ` %x2 = mul i64 %x1, -4658895280553007687`, - ` %x3s = lshr i64 %x2, 27`, - ` %x3 = xor i64 %x2, %x3s`, - ` %x4 = mul i64 %x3, -7723592293110705685`, - ` %x5s = lshr i64 %x4, 31`, - ` %x5 = xor i64 %x4, %x5s`, - ` %hz = icmp eq i64 %x5, 0`, - ` %h = select i1 %hz, i64 1, i64 %x5`, - ` %f1s = lshr i64 %h, 44`, - ` %f1a = and i64 %f1s, 63`, - ` %f1 = shl i64 1, %f1a`, - ` %f2s = lshr i64 %h, 50`, - ` %f2a = and i64 %f2s, 63`, - ` %f2 = shl i64 1, %f2a`, - ` %fb = or i64 %f1, %f2`, - ` %fp = getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 10`, - ` %filter = load i64, ptr %fp`, - ` %fm = and i64 %filter, %fb`, - ` %fin = icmp eq i64 %fm, %fb`, - ` br i1 %fin, label %table, label %miss`, - `table:`, - ` %nbp = getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 13`, - ...(sz === "i64" - ? [` %nb32 = load i32, ptr %nbp`, ` %nb = zext i32 %nb32 to i64`] - : [` %nb = load i32, ptr %nbp`]), - ` %eps = getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 11`, - ` %ents = load ptr, ptr %eps`, - ` %linear = icmp eq ${sz} %nb, 0`, - ` br i1 %linear, label %scan, label %buckets`, - // Small maps keep no bucket table: scan the entries' keys. A number-key - // entry has no cached hash; a dead entry's key matches no key. - `scan:`, - ` %nep = getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 8`, - ...(sz === "i64" - ? [` %ne32 = load i32, ptr %nep`, ` %ne = zext i32 %ne32 to i64`] - : [` %ne = load i32, ptr %nep`]), - ` br label %scan.loop`, - `scan.loop:`, - ` %si = phi ${sz} [ 0, %scan ], [ %si.next, %scan.next ]`, - ` %sdone = icmp eq ${sz} %si, %ne`, - ` br i1 %sdone, label %miss, label %scan.entry`, - `scan.entry:`, - ` %se = getelementptr inbounds %ScrMapNumEntry, ptr %ents, ${sz} %si`, - ` %sk = load i64, ptr %se`, - ` %skq = icmp eq i64 %sk, %kb`, - ` br i1 %skq, label %scan.found, label %scan.next`, - `scan.found:`, - ` ret ptr %se`, - `scan.next:`, - ` %si.next = add ${sz} %si, 1`, - ` br label %scan.loop`, - // Compact tables tag each 24-bit entry index (+1) with the hash's - // top byte; wider tables take the runtime lookup. - `buckets:`, - ` %wide = icmp ugt ${sz} %nb, 16777216`, - ` br i1 %wide, label %runtime, label %probe`, - `probe:`, - ` %bp = getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 14`, - ` %bk = load ptr, ptr %bp`, - ` %mask = sub ${sz} %nb, 1`, - ` %ht = lshr i64 %h, 32`, - ` %ht32 = trunc i64 %ht to i32`, - ` %tag = and i32 %ht32, -16777216`, - sz === "i64" ? ` %hi = add i64 %h, 0` : ` %hi = trunc i64 %h to ${sz}`, - ` %i0 = and ${sz} %hi, %mask`, - ` br label %probe.loop`, - `probe.loop:`, - ` %pi = phi ${sz} [ %i0, %probe ], [ %pi.next, %probe.next ]`, - ` %bep = getelementptr inbounds i32, ptr %bk, ${sz} %pi`, - ` %b = load i32, ptr %bep`, - ` %bempty = icmp eq i32 %b, 0`, - ` br i1 %bempty, label %miss, label %probe.tag`, - `probe.tag:`, - ` %bt = and i32 %b, -16777216`, - ` %btq = icmp eq i32 %bt, %tag`, - ` br i1 %btq, label %probe.entry, label %probe.next`, - `probe.entry:`, - ` %bix = and i32 %b, 16777215`, - ` %bix1 = sub i32 %bix, 1`, - sz === "i32" ? ` %bi = add i32 %bix1, 0` : ` %bi = zext i32 %bix1 to ${sz}`, - ` %pe = getelementptr inbounds %ScrMapNumEntry, ptr %ents, ${sz} %bi`, - ` %pk = load i64, ptr %pe`, - ` %pkq = icmp eq i64 %pk, %kb`, - ` br i1 %pkq, label %probe.found, label %probe.next`, - `probe.found:`, - ` ret ptr %pe`, - `probe.next:`, - ` %pi.inc = add ${sz} %pi, 1`, - ` %pi.next = and ${sz} %pi.inc, %mask`, - ` br label %probe.loop`, - `miss:`, - ` ret ptr null`, - `runtime:`, - ` %r = call ptr @scr_map_entry_f64(ptr %m, double %k) memory(read) nounwind willreturn`, - ` ret ptr %r`, - `}`, - ``, + // Hashing (scr_map_find_f64): the SameValueZero key word and its + // scr_key_mix hash. sc_map_entry_f64_int's probe converts the integer + // view to the key word, which is never NaN or -0, and so skips the + // normalization. + ...hashedNumberProbe( + "sc_map_entry_f64_hashed", + "double %k", + sz, + [ + ` %nan = fcmp uno double %k, %k`, + ` %zero = fcmp oeq double %k, 0.0`, + ` %kz = select i1 %zero, double 0.0, double %k`, + ` %kzb = bitcast double %kz to i64`, + ` %kb = select i1 %nan, i64 9221120237041090560, i64 %kzb`, + ], + "%k", + ), + ...hashedNumberProbe( + "sc_map_entry_f64_hashed_int", + "i64 %ki", + sz, + [` %kf = sitofp i64 %ki to double`, ` %kb = bitcast double %kf to i64`], + "%kf", + ), ); } // Map/Set iteration steps read the entries array inline: an index is diff --git a/packages/compiler/src/backend/llvm/map-number-lookup.test.ts b/packages/compiler/src/backend/llvm/map-number-lookup.test.ts index 7457db42d..f7bd87af1 100644 --- a/packages/compiler/src/backend/llvm/map-number-lookup.test.ts +++ b/packages/compiler/src/backend/llvm/map-number-lookup.test.ts @@ -168,8 +168,10 @@ test("hashed number-key lookups probe inline and call the runtime only for wide expect(hashed).toContain("select i1 %nan, i64 9221120237041090560, i64 %kzb"); expect(hashed).toContain("mul i64 %x1, -4658895280553007687"); expect(hashed).toContain("mul i64 %x3, -7723592293110705685"); - // Filter (field 10), small-map scan of the entry count (field 8), and - // compact buckets (fields 13 and 14) tagged with the hash's top byte. + // Filter (field 10) before the small-map scan of the entry count (field + // 8) only, and compact buckets (fields 13 and 14) tagged with the hash's + // top byte. + expect(hashed).toContain("br i1 %linear, label %filter, label %buckets"); expect(hashed).toContain("getelementptr inbounds %ScrMapIx, ptr %m, i32 0, i32 10"); expect(hashed).toContain(`load i32, ptr %nep`); expect(hashed).toContain(`load i32, ptr %nbp`); @@ -192,11 +194,23 @@ test("a key with an integer view tests the direct index on the integer", () => { integral.module, )![0]; // An unsigned compare also rejects negative keys; the hashed probe takes - // the double. + // the integer. expect(helper).toContain("icmp ult i64 %ki, %nd"); expect(helper).toContain(`getelementptr inbounds i32, ptr %dense, ${sz} %i`); - expect(helper).toContain("call ptr @sc_map_entry_f64_hashed(ptr %m, double %k)"); + expect(helper).toContain("call ptr @sc_map_entry_f64_hashed_int(ptr %m, i64 %ki)"); expect(helper).not.toContain("fptosi"); + // Its key word is the integer's double, which needs no NaN or -0 + // normalization; the runtime lookup gets that double. + const hashed = + /^define internal ptr @sc_map_entry_f64_hashed_int\(ptr %m, i64 %ki\) noinline [^]*?^}/m.exec( + integral.module, + )![0]; + expect(hashed).toContain("sitofp i64 %ki to double"); + expect(hashed).not.toContain("fcmp"); + expect(hashed).toContain("mul i64 %x1, -4658895280553007687"); + expect(hashed).toContain( + "call ptr @scr_map_entry_f64(ptr %m, double %kf) memory(read) nounwind willreturn", + ); // A fractional key and a key without a proof keep the double lookup. for (const key of [{ kind: "numLit", value: 0.5, type: F64, loc } as IrExpr, ref("k", F64)]) { const { work } = emitted(lookupFixture(key), bits); diff --git a/packages/compiler/src/backend/llvm/map-number-lookup.ts b/packages/compiler/src/backend/llvm/map-number-lookup.ts index afc8ec606..17a1563c3 100644 --- a/packages/compiler/src/backend/llvm/map-number-lookup.ts +++ b/packages/compiler/src/backend/llvm/map-number-lookup.ts @@ -4,9 +4,10 @@ * direct index in the runtime (scr_map.c). Reads go through the internal * sc_map_entry_f64 helper (emitter.ts helperDefs), which LLVM inlines: it * resolves directly indexed maps without a call and other maps through - * the internal sc_map_entry_f64_hashed probe, which hashes the key as the - * runtime does and defers to scr_map_entry_f64 only for wide bucket - * tables. Both only read memory. The generated code then reads the entry's + * the internal sc_map_entry_f64_hashed probe (sc_map_entry_f64_hashed_int + * for a key with an integer view), which hashes the key as the runtime + * does and defers to scr_map_entry_f64 only for wide bucket tables. Both + * only read memory. The generated code then reads the entry's * value slot itself and retains a reference value with the value type's own * retain, which is the adapter the collection was constructed with. * String-keyed reads use the same entry contract through the internal diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8331-map-number-hashed-integer-keys.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8331-map-number-hashed-integer-keys.ts.json new file mode 100644 index 000000000..86822b641 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8331-map-number-hashed-integer-keys.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/8331-map-number-hashed-integer-keys.ts" + ], + "diags": [] +} diff --git a/tests/corpus/8331-map-number-hashed-integer-keys.ts b/tests/corpus/8331-map-number-hashed-integer-keys.ts new file mode 100644 index 000000000..8af7293e9 --- /dev/null +++ b/tests/corpus/8331-map-number-hashed-integer-keys.ts @@ -0,0 +1,102 @@ +// Number-keyed Map and Set lookups whose keys are integers the compiler +// tracks as integers (loop counters, literals, integer arithmetic) on +// hashed storage, and lookups of absent keys on small bucket tables. + +// Keys spread past the direct index's range, so the maps hash them. +const spread = new Map(); +for (let i = 0; i < 500; i++) spread.set(i * 104729 - 26000000, i); +let found = 0, + absent = 0, + total = 0; +for (let i = -10; i < 520; i++) { + const v = spread.get(i * 104729 - 26000000); + if (v === undefined) absent++; + else { + found++; + total += v; + } +} +console.log("spread", spread.size, found, absent, total); +console.log("spread literals", spread.get(-26000000), spread.get(-25895271), spread.get(0), spread.has(-1)); + +// Zero stored through -0 and through an integer; read back through both. +const zeros = new Map(); +for (let i = 1; i <= 20; i++) zeros.set(i * 1000003, "k" + i); +zeros.set(-0, "minus zero"); +let z = 0; +console.log("zeros", zeros.size, zeros.get(z), zeros.get(0), zeros.get(-0), zeros.get(z * -1)); +zeros.set(z, "zero"); +console.log("zeros overwrite", zeros.size, zeros.get(-0), [...zeros.keys()].includes(0)); + +// Large and negative integer keys. +const wide = new Map(); +for (let i = 0; i < 64; i++) wide.set(2 ** 40 + i * 3, i); +for (let i = 0; i < 64; i++) wide.set(-(2 ** 33) - i, -i); +let wideUp = 0, + wideDown = 0, + wideMissing = 0; +for (let i = 0; i < 200; i++) { + const a = wide.get(2 ** 40 + i); + const b = wide.get(-(2 ** 33) - i); + if (a === undefined) wideMissing++; + else wideUp += a; + if (b === undefined) wideMissing++; + else wideDown += b; +} +console.log("wide", wide.size, wideUp, wideDown, wideMissing, wide.get(1099511627776), wide.get(-8589934592)); + +// Absent keys on bucket tables just past the linear scan, and as they grow. +for (const size of [9, 12, 16, 33, 64, 100, 257, 400]) { + const m = new Map(); + for (let i = 0; i < size; i++) m.set(i * 7919 + 1000000, i); + let hits = 0, + misses = 0; + for (let i = 0; i < size * 4; i++) { + if (m.has(i * 7919 + 1000000)) hits++; + else misses++; + if (m.get(i * 7919 + 1000001) !== undefined) hits += 1000; + } + for (let i = 0; i < size; i += 2) m.delete(i * 7919 + 1000000); + let after = 0; + for (let i = 0; i < size; i++) if (m.has(i * 7919 + 1000000)) after++; + console.log("size", size, hits, misses, m.size, after, m.get(1000000), m.get(1007919)); +} + +// Small maps that scan their entries, with integer keys. +const tiny = new Map(); +for (let i = 0; i < 8; i++) tiny.set(i * 5000011, "t" + i); +let tinyHits = 0; +for (let i = 0; i < 40; i++) if (tiny.has(i * 5000011)) tinyHits++; +console.log("tiny", tiny.size, tinyHits, tiny.get(0), tiny.get(35000077), tiny.has(5000012)); + +// Sets with integer keys. +const seen = new Set(); +let dupes = 0; +for (let i = 0; i < 3000; i++) { + const k = ((i * 2654435761) % 1000003) - 500000; + if (seen.has(k)) dupes++; + else seen.add(k); +} +let present = 0; +for (let i = -500000; i < 500003; i += 997) if (seen.has(i)) present++; +console.log("set", seen.size, dupes, present, seen.has(-500000), seen.has(500002)); + +// A counting loop like a k-mer histogram: get, then update the found entry. +class Count { + n = 0; +} +const counts = new Map(); +let key = 0; +for (let i = 0; i < 20000; i++) { + key = ((key * 4) % 4194304) + (i % 7 === 0 ? 3 : i % 3); + const c = counts.get(key); + if (c === undefined) { + const fresh = new Count(); + fresh.n = 1; + counts.set(key, fresh); + } else c.n++; +} +let most = 0, + mostKey = -1; +for (const [k, c] of counts) if (c.n > most) [most, mostKey] = [c.n, k]; +console.log("counts", counts.size, most, mostKey); diff --git a/tests/harness/corpus-costs/8331-map-number-hashed-integer-keys.ts.json b/tests/harness/corpus-costs/8331-map-number-hashed-integer-keys.ts.json new file mode 100644 index 000000000..961fe0dad --- /dev/null +++ b/tests/harness/corpus-costs/8331-map-number-hashed-integer-keys.ts.json @@ -0,0 +1 @@ +{ "plain": 3441, "san": 1752 } From 3a900ed9b6d8b43cdb7b764228dcd134df7f76d7 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Sun, 25 Oct 2026 16:38:27 -0500 Subject: [PATCH 39/44] Keep the poll in nested loops whose counter can stop changing at 2^53 The range analysis caps a counter that steps by 1 at 2^53, where `j + 1 === j`, so a loop such as `for (let j = 2 ** 53 - 2; j < 2 ** 53 + 10; j++)` has a range of three values at its condition and was taken for a loop of three runs. It never ends, and without its poll worker.terminate() could not stop it. A loop's runs now count as bounded only when a step from every value in its counter's range gives a different value. --- .../src/backend/llvm/emit-statements.ts | 15 ++++++++++++--- .../test/counted-loop-emission.test.ts | 19 ++++++++++++++++++- .../native-worker-terminate-bounded-loops.ts | 15 +++++++++++++-- 3 files changed, 43 insertions(+), 6 deletions(-) diff --git a/packages/compiler/src/backend/llvm/emit-statements.ts b/packages/compiler/src/backend/llvm/emit-statements.ts index e0db764e8..cc0af8ebb 100644 --- a/packages/compiler/src/backend/llvm/emit-statements.ts +++ b/packages/compiler/src/backend/llvm/emit-statements.ts @@ -2655,6 +2655,14 @@ const WORKER_POLL_BLOCK = 64; * only as it leaves. */ const WORKER_UNPOLLED_RUN = 16384; +/** Whether a step of `step` from every value in `range` gives a different + * value. Beyond 2^53 a double can stop changing (`j + 1 === j` from 2^53), + * so a counter there can run without end inside a range of a few values. */ +function exactSteps(range: { min: number; max: number }, step: number): boolean { + const size = Math.abs(step); + return range.min >= size - 2 ** 53 && range.max <= 2 ** 53 - size; +} + /** At most how many times a `for` loop and the loops in its body run their * bodies per entry, from the counters' proven ranges: its iterations times * one plus the nested loops' runs. Infinity when a count has no bound, or @@ -2668,12 +2676,12 @@ function boundedLoopRuns(emitter: LlEmitter, loop: IrStmt): number { LOOPS.of(emitter).unaliasedArrays, ); let trips = Infinity; - if (counted && !counted.guarded) + if (counted && !counted.guarded && exactSteps(counted.range, counted.step)) trips = Math.floor((counted.range.max - counted.range.min) / Math.abs(counted.step)) + 1; else if (!counted) { // Each iteration closes the gap between the two counters by at least two. const paired = matchIntegerPairedForLoop(loop, emitter.numericLocals, emitter.integerRanges); - if (paired && !paired.guarded) + if (paired && !paired.guarded && exactSteps(paired.range, 1)) trips = Math.floor((paired.range.max - paired.range.min) / 2) + 1; } if (trips === Infinity) { @@ -2681,7 +2689,8 @@ function boundedLoopRuns(emitter: LlEmitter, loop: IrStmt): number { // one compared with a module constant. const stepped = steppedLoopCounter(loop, emitter.currentLocals); const range = stepped && emitter.integerRanges.get(stepped.ref); - if (stepped && range) trips = Math.floor((range.max - range.min) / Math.abs(stepped.step)) + 1; + if (stepped && range && exactSteps(range, stepped.step)) + trips = Math.floor((range.max - range.min) / Math.abs(stepped.step)) + 1; } if (!(trips <= WORKER_POLL_BLOCK * WORKER_POLL_BLOCK)) return Infinity; let nested = 0; diff --git a/packages/compiler/test/counted-loop-emission.test.ts b/packages/compiler/test/counted-loop-emission.test.ts index 794edd1af..d7ebc97fa 100644 --- a/packages/compiler/test/counted-loop-emission.test.ts +++ b/packages/compiler/test/counted-loop-emission.test.ts @@ -175,6 +175,18 @@ test("worker programs poll no loop whose proven ranges bound its runs inside ano " for (let r = 0; r < n; r++) for (let j = 0; j < 5000; j++) t = (t * 31 + j) % 1000003;", " return t;", "}", + "export function exact(n: number): number {", + " let t = 0;", + " for (let r = 0; r < n; r++)", + " for (let j = 9007199254740900; j < 9007199254740950; j++) t = (t + (j % 7)) % 1000003;", + " return t;", + "}", + "export function stalled(n: number): number {", + " let t = 0;", + " for (let r = 0; r < n; r++)", + " for (let j = 9007199254740990; j < 9007199254741002; j++) t = (t + (j % 7)) % 1000003;", + " return t;", + "}", "export function single(): number {", " let t = 0;", " for (let j = 0; j < 50; j++) t = (t * 31 + j) % 1000003;", @@ -182,7 +194,7 @@ test("worker programs poll no loop whose proven ranges bound its runs inside ano "}", "const cr = new Float64Array(8).fill(0.5);", 'const k = Number(process.argv[2] ?? "50");', - "console.log(rows(cr, 3), open(cr, 3, k), wide(2), single());", + "console.log(rows(cr, 3), open(cr, 3, k), wide(2), exact(2), stalled(0), single());", "if (!isMainThread) new Worker(new URL(import.meta.url));", ].join("\n"), ); @@ -219,6 +231,11 @@ test("worker programs poll no loop whose proven ranges bound its runs inside ano // A run of up to 16384 iterations is one block, polled as it leaves when // it took more than 64. expect(open).toMatch(/^loop\.poll\.run\d+:$/m); + // Steps of 1 below 2^53 are exact, so 50 runs near it are bounded. From + // 2^53, `j + 1 === j`: a counter ranging up to it can run without end + // and keeps its poll. + expect(polls(body("exact"))).toBe(polls(body("single"))); + expect(polls(body("stalled"))).toBeGreaterThan(polls(body("exact"))); // A nest over the bound, or a loop no other loop encloses, polls. expect(polls(body("wide"))).toBeGreaterThan(0); expect(body("wide")).toMatch(/^loop\.poll\d+:$/m); diff --git a/tests/corpus/native-worker-terminate-bounded-loops.ts b/tests/corpus/native-worker-terminate-bounded-loops.ts index d244e021d..ec1e4c736 100644 --- a/tests/corpus/native-worker-terminate-bounded-loops.ts +++ b/tests/corpus/native-worker-terminate-bounded-loops.ts @@ -1,7 +1,8 @@ // worker.terminate() stops a worker whose loop runs short inner loops: // counted loops of a few iterations, a two-counter loop, an inner loop // whose length grows past 64 iterations, an inner loop too long to finish, -// and a nest of loops with constant bounds. +// a nest of loops with constant bounds, and an inner loop whose counter +// stops changing at 2 ** 53 (`j + 1 === j` there), so it never ends. import { Worker, isMainThread, parentPort, workerData } from "node:worker_threads"; function counted(values: number[]): number { @@ -66,8 +67,17 @@ function constant(row: Float64Array): number { return acc; } +function stalled(): number { + let acc = 0; + for (let round = 0; ; round++) { + if (round === 0) parentPort!.postMessage("ready"); + for (let j = 9007199254740990; j < 9007199254741002; j++) acc = (acc + (j % 7)) % 1000003; + } + return acc; +} + if (isMainThread) { - for (const kind of ["counted", "paired", "growing", "long", "constant"]) { + for (const kind of ["counted", "paired", "growing", "long", "constant", "stalled"]) { await new Promise((resolve) => { const worker = new Worker(new URL(import.meta.url), { workerData: kind }); worker.once("message", () => { @@ -86,6 +96,7 @@ if (isMainThread) { else if (workerData === "paired") result = paired(values); else if (workerData === "growing") result = growing(); else if (workerData === "long") result = long(1e15); + else if (workerData === "stalled") result = stalled(); else result = constant(new Float64Array(8).fill(0.25)); console.log("unexpected result", result); } From 5f76d41a7181348e2e14d60a2ec3377f825bc3d5 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Sun, 25 Oct 2026 16:57:11 -0500 Subject: [PATCH 40/44] Keep U+11A7 after a Hangul LV syllable in NFC and NFKC MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit libunicode composes an LV syllable with U+11A7 as if it were a trailing consonant (U+11A8..U+11C2), which drops it: "가ᆧ".normalize() gave "가" where Node gives "가ᆧ". U+11A7 is a class-0 vowel that composes with nothing, so for the composing forms the code points between occurrences of it normalize on their own and it is kept as it is. The guard sits in front of the vendored library (scr_unicode_normalize), for String.prototype.normalize and the NFC step of IDNA mapping. 8009-normalize-hangul-jungseong-o-yae prints the code points of all four forms for U+11A7 after LV and LVT syllables, conjoining jamo, other starters and combining marks. --- ...9-normalize-hangul-jungseong-o-yae.ts.json | 6 ++ packages/runtime/src/scr_regex.c | 55 ++++++++++++++++++- .../8009-normalize-hangul-jungseong-o-yae.ts | 28 ++++++++++ ...9-normalize-hangul-jungseong-o-yae.ts.json | 1 + 4 files changed, 88 insertions(+), 2 deletions(-) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8009-normalize-hangul-jungseong-o-yae.ts.json create mode 100644 tests/corpus/8009-normalize-hangul-jungseong-o-yae.ts create mode 100644 tests/harness/corpus-costs/8009-normalize-hangul-jungseong-o-yae.ts.json diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8009-normalize-hangul-jungseong-o-yae.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8009-normalize-hangul-jungseong-o-yae.ts.json new file mode 100644 index 000000000..872a85396 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/8009-normalize-hangul-jungseong-o-yae.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/8009-normalize-hangul-jungseong-o-yae.ts" + ], + "diags": [] +} diff --git a/packages/runtime/src/scr_regex.c b/packages/runtime/src/scr_regex.c index 2a98e92c6..37f94bd65 100644 --- a/packages/runtime/src/scr_regex.c +++ b/packages/runtime/src/scr_regex.c @@ -2418,6 +2418,57 @@ static void *scr_normalize_realloc(void *opaque, void *ptr, size_t size) { return realloc(ptr, size); } +/* unicode_normalize, except that U+11A7 never composes. libunicode composes + * an LV syllable with U+11A7 as a trailing consonant (those are U+11A8 to + * U+11C2), which drops it: "\uAC00\u11A7" became "\uAC00". U+11A7 is a + * class-0 vowel that composes with nothing, so for the composing forms the + * code points between occurrences of it normalize on their own and it is + * kept as it is, as in Node. */ +static int scr_unicode_normalize(uint32_t **pdst, const uint32_t *src, int len, + UnicodeNormalizationEnum mode) { + int at = 0; + if (mode == UNICODE_NFC || mode == UNICODE_NFKC) + while (at < len && src[at] != 0x11A7) at++; + else + at = len; + if (at == len) return unicode_normalize(pdst, src, len, mode, NULL, scr_normalize_realloc); + uint32_t *out = NULL; + size_t count = 0, cap = 0; + int start = 0; + for (int i = 0; i <= len; i++) { + if (i < len && src[i] != 0x11A7) continue; + uint32_t *part = NULL; + int n = 0; + if (i > start) { + n = unicode_normalize(&part, src + start, i - start, mode, NULL, scr_normalize_realloc); + if (n < 0) { + free(out); + return -1; + } + } + size_t need = count + (size_t)n + 1; + if (need > cap) { + size_t grown = cap ? cap * 2 : 16; + if (grown < need) grown = need; + uint32_t *bigger = realloc(out, grown * sizeof *out); + if (!bigger) { + free(part); + free(out); + return -1; + } + out = bigger; + cap = grown; + } + if (n > 0) memcpy(out + count, part, (size_t)n * sizeof *out); + count += (size_t)n; + free(part); + if (i < len) out[count++] = 0x11A7; + start = i + 1; + } + *pdst = out; + return (int)count; +} + ScrStr *scr_str_normalize(const ScrStr *s, const ScrStr *form) { UnicodeNormalizationEnum mode; if (form->len == 3 && !memcmp(form->data, "NFC", 3)) mode = UNICODE_NFC; @@ -2442,7 +2493,7 @@ ScrStr *scr_str_normalize(const ScrStr *s, const ScrStr *form) { offset += step; } uint32_t *output = NULL; - int count = unicode_normalize(&output, input, length, mode, NULL, scr_normalize_realloc); + int count = scr_unicode_normalize(&output, input, length, mode); free(input); if (count < 0) scr_regex_oom(); ScrJsonBuf buffer; @@ -2455,7 +2506,7 @@ ScrStr *scr_str_normalize(const ScrStr *s, const ScrStr *form) { uint32_t *scr_nfc_code_points(const uint32_t *input, size_t length, size_t *out_len) { if (length > INT32_MAX) scr_regex_oom(); uint32_t *output = NULL; - int count = unicode_normalize(&output, input, (int)length, UNICODE_NFC, NULL, scr_normalize_realloc); + int count = scr_unicode_normalize(&output, input, (int)length, UNICODE_NFC); if (count < 0) scr_regex_oom(); *out_len = (size_t)count; return output; diff --git a/tests/corpus/8009-normalize-hangul-jungseong-o-yae.ts b/tests/corpus/8009-normalize-hangul-jungseong-o-yae.ts new file mode 100644 index 000000000..b7cb5418a --- /dev/null +++ b/tests/corpus/8009-normalize-hangul-jungseong-o-yae.ts @@ -0,0 +1,28 @@ +// U+11A7 (HANGUL JUNGSEONG O-YAE) is a vowel that composes with nothing: +// after an LV syllable, after a leading consonant, and around trailing +// consonants it stays as it is in every normalization form, as in Node. + +const samples = [ + "가ᆧ", + "가ᆧ", + "가ᆧᆨ", + "각ᆧ", + "ᄀᆧ", + "ᆧ", + "ᆧᆧ", + "x가ᆧy각z", + "café 가ᆧ é", + "ｶﾞᆧ가", +]; + +function points(text: string): string { + return Array.from(text, (ch) => ch.codePointAt(0)!.toString(16)).join("."); +} + +for (const sample of samples) { + console.log( + points(sample), + ["NFC", "NFD", "NFKC", "NFKD"].map((form) => points(sample.normalize(form))).join(" "), + ); +} +console.log("가ᆧ".normalize() === "가ᆧ", "가ᆧ".normalize().length); diff --git a/tests/harness/corpus-costs/8009-normalize-hangul-jungseong-o-yae.ts.json b/tests/harness/corpus-costs/8009-normalize-hangul-jungseong-o-yae.ts.json new file mode 100644 index 000000000..6c20e6ad7 --- /dev/null +++ b/tests/harness/corpus-costs/8009-normalize-hangul-jungseong-o-yae.ts.json @@ -0,0 +1 @@ +{ "plain": 2503, "san": 2110 } From 605e312e788131d8a3067227a0abdd1a2fbfb0fe Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Sun, 25 Oct 2026 14:00:29 -0700 Subject: [PATCH 41/44] Let boxable function arms ride a checked-dynamic union A union with a checked-dynamic arm (`unknown`, fetch's AbortSignal) maps to the checked-dynamic value when every other arm is one the checked-dynamic tree holds faithfully. Function arms were excluded, so `Progress | AbortSignal | undefined` was refused (SC2009). The tree boxes a function whose parameters check in and whose result converts out by reference (canBoxFuncIntoDyn), and `typeof x === "function"` narrows it back out to the typed function. Such arms now join the collapse. The corpus program checks calls, identity (`===` against the original), and the other arm's members against Node. pi's SessionManager.listAll takes `onProgressOrSignal?: SessionListProgress | AbortSignal`. The refusal rejected the whole SessionManager class. Co-Authored-By: Claude Opus 5.5 --- packages/compiler/src/frontend/type-mapper.ts | 11 +++++ .../corpus/dyn-union-function-arms.ts.json | 6 +++ tests/corpus/dyn-union-function-arms.ts | 48 +++++++++++++++++++ .../dyn-union-function-arms.ts.json | 1 + 4 files changed, 66 insertions(+) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/dyn-union-function-arms.ts.json create mode 100644 tests/corpus/dyn-union-function-arms.ts create mode 100644 tests/harness/corpus-costs/dyn-union-function-arms.ts.json diff --git a/packages/compiler/src/frontend/type-mapper.ts b/packages/compiler/src/frontend/type-mapper.ts index 17f9d0d11..03d85ba7d 100644 --- a/packages/compiler/src/frontend/type-mapper.ts +++ b/packages/compiler/src/frontend/type-mapper.ts @@ -10,6 +10,7 @@ import { isDynTypedRefType, BOOL, bytesOf, + canBoxFuncIntoDyn, canConvertToDyn, canDynCheckTo, CHILD_T, @@ -8341,6 +8342,16 @@ function dynSubsumableUnionArm(arm: IrType, ctx: TypeMapperCtx): boolean { // `string | URL | Request`, fetch's own input type under @types/node. case "url": return true; + // A function the checked-dynamic tree can box (each parameter checks + // in, the result converts out) is held by reference there, and + // `typeof x === "function"` narrows it back out: `Progress | AbortSignal` + // beside fetch's handles. + case "func": + return canBoxFuncIntoDyn( + arm, + (id) => ctx.shapes.get(id), + (id) => ctx.unions.get(id), + ); case "record": case "array": // `T | Promise` with T bound to `unknown`: the checked-dynamic value diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/dyn-union-function-arms.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/dyn-union-function-arms.ts.json new file mode 100644 index 000000000..80675e5e6 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/dyn-union-function-arms.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/dyn-union-function-arms.ts" + ], + "diags": [] +} diff --git a/tests/corpus/dyn-union-function-arms.ts b/tests/corpus/dyn-union-function-arms.ts new file mode 100644 index 000000000..e85cc2e6b --- /dev/null +++ b/tests/corpus/dyn-union-function-arms.ts @@ -0,0 +1,48 @@ +// A union of a function type with a value the checked-dynamic tree holds +// (AbortSignal, `unknown`) rides that representation: the function is held +// by reference, `typeof x === "function"` narrows it back out, and calls, +// identity and the other arm's members behave as in Node. + +type Progress = (loaded: number, total: number) => void; + +function listAll( + dirOrProgress?: string | Progress, + progressOrSignal?: Progress | AbortSignal, + signal?: AbortSignal, +): string { + const dir = typeof dirOrProgress === "string" ? dirOrProgress : "/default"; + const progress = + typeof dirOrProgress === "function" + ? dirOrProgress + : typeof progressOrSignal === "function" + ? progressOrSignal + : undefined; + const abortSignal = + typeof dirOrProgress === "string" || typeof progressOrSignal === "function" + ? signal + : (progressOrSignal ?? signal); + progress?.(1, 3); + progress?.(3, 3); + return `${dir} aborted=${abortSignal?.aborted ?? "none"}`; +} + +const seen: string[] = []; +const report: Progress = (loaded, total) => seen.push(`${loaded}/${total}`); +const controller = new AbortController(); +console.log(listAll()); +console.log(listAll("/x", report)); +console.log(listAll(report, controller.signal)); +controller.abort(); +console.log(listAll("/y", undefined, controller.signal)); +console.log(listAll(report, controller.signal), seen.join(" ")); + +function same(handler: Progress | AbortSignal | undefined): boolean { + return handler === report; +} +console.log(same(report), same(controller.signal), same(undefined), same((a, b) => void (a + b))); + +function describe(value: unknown | ((n: number) => string)): string { + if (typeof value === "function") return `function -> ${(value as (n: number) => string)(7)}`; + return `value ${JSON.stringify(value)}`; +} +console.log(describe((n: number) => `n=${n}`), describe({ a: 1 }), describe("s")); diff --git a/tests/harness/corpus-costs/dyn-union-function-arms.ts.json b/tests/harness/corpus-costs/dyn-union-function-arms.ts.json new file mode 100644 index 000000000..1254aa6e4 --- /dev/null +++ b/tests/harness/corpus-costs/dyn-union-function-arms.ts.json @@ -0,0 +1 @@ +{ "plain": 4000 } From 3375304945e1b6e548d501b3c9d403f1c5d155fa Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Sun, 25 Oct 2026 14:40:32 -0700 Subject: [PATCH 42/44] Erase lambdas that implement a generic method through their context MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A lambda with no type parameters of its own, contextually typed by a generic signature (`send: (m) => …` for an interface's `send(m: Message): string`), takes that signature's type parameters: its parameter is `Message`. A generic lambda written with its own type parameters already lowers once, erased, with them bound to `unknown`, but this one tried to map `Message` with `T` unbound and was refused (SC2009). It now lowers the same way, erased, binding the contextual signature's type parameters to `unknown`. Parameters bound in the enclosing scope keep their bindings. Co-Authored-By: Claude Opus 5.5 --- .../src/frontend/lowering/lower-functions.ts | 42 +++++++++++++++++++ .../generic-member-lambda-impls.ts.json | 6 +++ tests/corpus/generic-member-lambda-impls.ts | 40 ++++++++++++++++++ .../generic-member-lambda-impls.ts.json | 1 + 4 files changed, 89 insertions(+) create mode 100644 packages/compiler/test/ts7/baselines/order-parity/tests/corpus/generic-member-lambda-impls.ts.json create mode 100644 tests/corpus/generic-member-lambda-impls.ts create mode 100644 tests/harness/corpus-costs/generic-member-lambda-impls.ts.json diff --git a/packages/compiler/src/frontend/lowering/lower-functions.ts b/packages/compiler/src/frontend/lowering/lower-functions.ts index bd9b5f55d..7f4c8dfb7 100644 --- a/packages/compiler/src/frontend/lowering/lower-functions.ts +++ b/packages/compiler/src/frontend/lowering/lower-functions.ts @@ -167,6 +167,23 @@ function declareFunctionArguments( } } +/** The type parameters a lambda without its own takes from a generic + * contextual signature, unbound in the current scope. */ +function contextualTypeParameterSymbols( + lowerer: Lowerer, + node: ts.ArrowFunction | ts.FunctionExpression, +): ts.Symbol[] { + const signatures = lowerer.checker.getCallSignatures(lowerer.typeOf(node)); + if (signatures.length !== 1) return []; + const symbols: ts.Symbol[] = []; + for (const parameter of signatures[0]!.getTypeParameters()) { + const symbol = parameter.getSymbol(); + if (!symbol || lowerer.typeParamBindings?.has(symbol)) return []; + symbols.push(symbol); + } + return symbols; +} + /** Lifts an arrow function / function expression / nested declaration / * object-literal shorthand method to a module-level function and yields * the `closure` expression creating it. */ @@ -202,6 +219,31 @@ export function lowerLambda( lowerer.typeParamBindings = previous; } } + // A lambda contextually typed by a GENERIC signature (`send: (m) => …` + // for `send(m: Msg): string`) takes that signature's type + // parameters as its own; it lowers once, erased, like a generic lambda. + if ( + (ts.isArrowFunction(node) || ts.isFunctionExpression(node)) && + !node.typeParameters?.nodes.length && + !lowerer.erasingLambdas.has(node) + ) { + const contextual = contextualTypeParameterSymbols(lowerer, node); + if (contextual.length > 0) { + const previous = lowerer.typeParamBindings; + const bindings = new Map(previous ?? []); + for (const symbol of contextual) bindings.set(symbol, DYN); + lowerer.typeParamBindings = bindings; + lowerer.erasingLambdas.add(node); + lowerer.erasingDepth++; + try { + return lowerLambda(lowerer, node, checkedReceiver); + } finally { + lowerer.erasingDepth--; + lowerer.erasingLambdas.delete(node); + lowerer.typeParamBindings = previous; + } + } + } const prepared = ts.isArrowFunction(node) || ts.isFunctionExpression(node) ? lowerer.preparedClassFactories.get(node) diff --git a/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/generic-member-lambda-impls.ts.json b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/generic-member-lambda-impls.ts.json new file mode 100644 index 000000000..bc87c6ea0 --- /dev/null +++ b/packages/compiler/test/ts7/baselines/order-parity/tests/corpus/generic-member-lambda-impls.ts.json @@ -0,0 +1,6 @@ +{ + "order": [ + "/tests/corpus/generic-member-lambda-impls.ts" + ], + "diags": [] +} diff --git a/tests/corpus/generic-member-lambda-impls.ts b/tests/corpus/generic-member-lambda-impls.ts new file mode 100644 index 000000000..fd25bae96 --- /dev/null +++ b/tests/corpus/generic-member-lambda-impls.ts @@ -0,0 +1,40 @@ +// An object literal implements an interface's generic method with a plain +// lambda (`send: (m) => …` for `send(m: Message): string`). The +// lambda takes the contextual signature's type parameters as its own and +// lowers once, erased, like a generic lambda: its parameters hold whatever +// each call passes. + +interface Message { + content: string; + details?: T; +} + +interface Sender { + name: string; + send(message: Message): string; + notify(message: Message): void; + pick(keys: K[], fallback: K): K; +} + +const log: string[] = []; + +function makeSender(name: string): Sender { + return { + name, + send: (message) => `${name} sent ${message.content} (${message.details === undefined ? "no details" : JSON.stringify(message.details)})`, + notify: (message) => { + log.push(`${name} notified ${message.content}`); + }, + pick: (keys, fallback) => keys[0] ?? fallback, + }; +} + +const senders: Sender[] = [makeSender("first"), makeSender("second")]; +for (const sender of senders) { + console.log(sender.send({ content: "hi", details: { n: 3 } })); + console.log(sender.send({ content: "bare" })); + console.log(sender.send({ content: "list", details: [1, 2] })); + sender.notify({ content: "ping", details: true }); + console.log(sender.pick(["a", "b"], "z"), sender.pick([], "z")); +} +console.log(log.join("; ")); diff --git a/tests/harness/corpus-costs/generic-member-lambda-impls.ts.json b/tests/harness/corpus-costs/generic-member-lambda-impls.ts.json new file mode 100644 index 000000000..1254aa6e4 --- /dev/null +++ b/tests/harness/corpus-costs/generic-member-lambda-impls.ts.json @@ -0,0 +1 @@ +{ "plain": 4000 } From 1733a4c84bf815fa283ddcb8672d4ef3df18f5fe Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Sun, 25 Oct 2026 15:17:09 -0700 Subject: [PATCH 43/44] Name new corpus programs instead of numbering them Parallel changes kept taking the same next corpus number (6992 twice, the Node island programs renumbered several times). The harness never needed the numbers: it discovers every entry by glob, and 818 programs already have descriptive names. AGENTS.md now makes a name, without a number, the convention for new programs and keeps the numbered ones as they are. The two scripts that sampled programs by number skipped named programs. They now key a named program by a stable hash of its name, so they sample it at the same rate; the numbered programs they pick are unchanged. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 2 ++ scripts/lowering-order.mjs | 24 ++++++++++++++++++++---- scripts/refresh-cli-profile.mjs | 29 ++++++++++++++++++++++------- 3 files changed, 44 insertions(+), 11 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 2350d7337..55ccb2eab 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -46,6 +46,8 @@ SCRIPTC_TEST_WORKERS=4 SCRIPTC_SAN=1 pnpm test # sanitized lane Corpus programs are differential tests against Node: every program runs under Node and as a compiled native binary, and stdout, stderr, and exit codes must match byte-for-byte. A new feature lands with corpus programs that pin its behavior both ways. +Name a new corpus program after what it tests, without a number: lowercase words joined by hyphens, such as `tests/corpus/intersection-restated-members.ts`, or a directory such as `tests/corpus//main.ts` for a multi-file program. Parallel changes kept taking the same next number; with names, two changes collide only when they add the same name, which git reports as a conflict. Leave existing numbered programs as they are: never rename or renumber them, and name a new program even when it extends a numbered family. + Record a new or remeasured program's CI cost (used to balance the test shards) in `tests/harness/corpus-costs/.json`, not in `tests/harness/corpus-costs.json`; see the README there. Before opening a PR that adds or changes corpus, diagnostic, or frontend fixtures, run `pnpm test:ts7 --baselines-only`. This fast completeness check uses the same fixture discovery as the parity suite; it does not type-check or replace behavioral parity. Generate missing preflight/order baselines, review their diagnostics and module order, and run the affected `order-parity.test.ts` batches before pushing. Keep existing baseline entries unchanged unless the corresponding behavior intentionally changes. The full `pnpm test:ts7` command also checks completeness before starting its sweep. diff --git a/scripts/lowering-order.mjs b/scripts/lowering-order.mjs index 81611e6a4..6d68530cb 100644 --- a/scripts/lowering-order.mjs +++ b/scripts/lowering-order.mjs @@ -16,13 +16,16 @@ * [--seeds 1,2,3] [--jobs 8] [--entry ]... [--out ] * [--fail-on names|content] * + * --stride and --offset sample the top-level corpus programs by number, or by + * a hash of the name of a program without one. + * * Needs a built compiler (pnpm build). Writes /results.json and prints * a ranking of fields by the programs whose IR they change. With --fail-on, * it exits with status 1 when some program's IR differs between orders: * in any way (names), or beyond names and ids (content). */ import { spawn } from "node:child_process"; import { createHash } from "node:crypto"; -import { mkdirSync, readFileSync, readdirSync, writeFileSync } from "node:fs"; +import { mkdirSync, readFileSync, readdirSync, statSync, writeFileSync } from "node:fs"; import { cpus, tmpdir } from "node:os"; import { basename, join, resolve } from "node:path"; import { fileURLToPath, pathToFileURL } from "node:url"; @@ -306,12 +309,25 @@ if (values["fail-on"] !== undefined && !["names", "content"].includes(values["fa const out = resolve(values.out); mkdirSync(out, { recursive: true }); const seeds = values.seeds.split(",").filter(Boolean); +/** A top-level corpus program's sampling key: its number, or, for a program + * named without one, a stable hash of its name (FNV-1a). */ +const programKey = (name) => + /^\d+-/.test(name) + ? Number.parseInt(name, 10) + : [...name].reduce((h, c) => Math.imul(h ^ c.charCodeAt(0), 16777619) >>> 0, 2166136261); +/** Numbered programs by number, then the named ones by name. */ +const programOrder = (a, b) => + /^\d+-/.test(a) !== /^\d+-/.test(b) + ? /^\d+-/.test(a) + ? -1 + : 1 + : programKey(a) - programKey(b) || (a < b ? -1 : a > b ? 1 : 0); const corpus = join(root, "tests/corpus"); const programs = [ ...readdirSync(corpus) - .filter((name) => /^\d+-.*\.ts$/.test(name)) - .filter((name) => Number.parseInt(name, 10) % Number(values.stride) === Number(values.offset)) - .sort((a, b) => Number.parseInt(a, 10) - Number.parseInt(b, 10)) + .filter((name) => /^[\w-]+\.ts$/.test(name) && statSync(join(corpus, name)).isFile()) + .filter((name) => programKey(name) % Number(values.stride) === Number(values.offset)) + .sort(programOrder) .slice(0, Number(values.limit)) .map((name) => join(corpus, name)), ...values.entry.map((entry) => resolve(entry)), diff --git a/scripts/refresh-cli-profile.mjs b/scripts/refresh-cli-profile.mjs index 1285b5503..dde4e21b9 100644 --- a/scripts/refresh-cli-profile.mjs +++ b/scripts/refresh-cli-profile.mjs @@ -7,8 +7,9 @@ * Builds an instrumented native compiler for this Linux host, runs it on the * training builds below, and merges the raw profiles it writes into one * sparse indexed profile. The training builds are tsc-ts (a cold `speed` - * build, then a rebuild after an edit), every seventh corpus program at the - * default optimization and a further slice in `dev`, and the runtime + * build, then a rebuild after an edit), every seventh top-level corpus + * program (by number, or by a hash of the name of a program without one) at + * the default optimization and a further slice in `dev`, and the runtime * benchmark workloads. Run it on a quiet Linux x64 host after changes that * move much compiler code; a stale profile only loses part of its gain. A * refresh takes about 11 minutes on 16 cores. @@ -126,16 +127,30 @@ async function pool(tasks) { return passed; } -const programNumber = (name) => Number.parseInt(name, 10); +/** A top-level corpus program's sampling key: its number, or, for a program + * named without one, a stable hash of its name (FNV-1a). */ +const programKey = (name) => + /^\d+-/.test(name) + ? Number.parseInt(name, 10) + : [...name].reduce((h, c) => Math.imul(h ^ c.charCodeAt(0), 16777619) >>> 0, 2166136261); +/** Numbered programs by number, then the named ones by name. */ +const programOrder = (a, b) => + /^\d+-/.test(a) !== /^\d+-/.test(b) + ? /^\d+-/.test(a) + ? -1 + : 1 + : programKey(a) - programKey(b) || (a < b ? -1 : a > b ? 1 : 0); const corpus = readdirSync(join(root, "tests/corpus")) - .filter((name) => /^\d+-.*\.ts$/.test(name)) - .sort((a, b) => programNumber(a) - programNumber(b)); + .filter( + (name) => /^[\w-]+\.ts$/.test(name) && statSync(join(root, "tests/corpus", name)).isFile(), + ) + .sort(programOrder); const training = [ ...corpus - .filter((name) => programNumber(name) % 7 === 0) + .filter((name) => programKey(name) % 7 === 0) .map((name) => ({ file: join(root, "tests/corpus", name), optimization: "release" })), ...corpus - .filter((name) => programNumber(name) % 14 === 7) + .filter((name) => programKey(name) % 14 === 7) .map((name) => ({ file: join(root, "tests/corpus", name), optimization: "dev" })), ...readdirSync(join(root, "benchmarks/runtime/workloads")) .filter((name) => name.endsWith(".ts")) From d33b083a3b4f49c258e4ff2a911235bcd3abda8a Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Sun, 25 Oct 2026 17:30:44 -0500 Subject: [PATCH 44/44] Cache the zone offset at UTC midnights for local-time conversions Local setters and the field constructor convert a local time value to UTC by bracketing it with the zone offset a day before and after, then checking the candidate instants: three to five localtime(3) calls per conversion. In glibc each call takes the zone lock and, for dates past the zone file's last transition, parses the zone's rule string again. The offsets now come from a per-thread cache of offsets at UTC midnights: a day whose two midnights agree has that offset throughout, given at most one transition per day, which the conversion already assumes. A day with a transition still asks localtime for the instant. A change of process.env.TZ clears the cache. --- packages/runtime/src/scr_lib.c | 58 ++++++++++++++++++++++++++++++++-- 1 file changed, 56 insertions(+), 2 deletions(-) diff --git a/packages/runtime/src/scr_lib.c b/packages/runtime/src/scr_lib.c index a9fd25915..1a625cdf1 100644 --- a/packages/runtime/src/scr_lib.c +++ b/packages/runtime/src/scr_lib.c @@ -11588,8 +11588,8 @@ double scr_date_get_timezone_offset(double ms) { * from local fields reads those fields back. */ /* The host zone's offset (local - UTC) in ms at UTC instant `ms` (an - * integral time value within about ±8.65e15). */ -static bool scr_date_offset_at(double ms, double *out) { + * integral time value within about ±8.65e15), from the host's localtime. */ +static bool scr_date_offset_at_uncached(double ms, double *out) { long long cms = (long long)ms; double secd = (double)(cms / 1000 - (cms % 1000 < 0)); struct tm tmv; @@ -11611,6 +11611,60 @@ static bool scr_date_offset_at(double ms, double *out) { return true; } +/* Offsets at UTC midnights, for scr_date_offset_at. A day whose two + * midnights have the same offset has that offset throughout, given at most + * one transition per day (scr_date_local_to_utc already takes transitions + * to be at least a day apart). Each local setter asks for three to five + * offsets around one time value, and consecutive ones land on the same or + * neighbouring days, so most answers come from here instead of a + * localtime(3) call: glibc takes its zone lock in each and, for dates past + * the zone file's last transition, parses the zone's rule string again. A + * change of process.env.TZ starts a new generation. */ +#define SCR_DATE_MIDNIGHTS 64 +typedef struct { + long long day; + double offset; + bool valid; +} ScrDateMidnight; +typedef struct { + bool valid; + unsigned generation; + ScrDateMidnight entries[SCR_DATE_MIDNIGHTS]; +} ScrDateMidnightCache; +static SCR_TL ScrDateMidnightCache scr_date_midnight_cache; + +static bool scr_date_midnight_offset(ScrDateMidnightCache *c, long long day, double *out) { + ScrDateMidnight *e = &c->entries[(unsigned long long)day % SCR_DATE_MIDNIGHTS]; + if (e->valid && e->day == day) { + *out = e->offset; + return true; + } + if (!scr_date_offset_at_uncached((double)day * 86400000.0, out)) return false; + e->valid = true; + e->day = day; + e->offset = *out; + return true; +} + +static bool scr_date_offset_at(double ms, double *out) { + ScrDateMidnightCache *c = &scr_date_midnight_cache; + unsigned generation = scr_date_tz_generation; + if (!c->valid || c->generation != generation) { + memset(c, 0, sizeof *c); + c->valid = true; + c->generation = generation; + } + long long cms = (long long)ms; + long long day = cms / 86400000 - (cms % 86400000 < 0); + double start, end; + if (scr_date_midnight_offset(c, day, &start) && scr_date_midnight_offset(c, day + 1, &end) && + start == end) { + *out = start; + return true; + } + return scr_date_offset_at_uncached(ms, out); +} + static volatile const double scr_date_day_ms = 86400000.0; static double scr_date_local_to_utc(double t) {